From f760e9070a8abc98600a69e2fd8ba3fbaab77dd3 Mon Sep 17 00:00:00 2001 From: Harwinder Sekhon Date: Tue, 28 Jul 2026 20:08:53 -0400 Subject: [PATCH] Research: Dukascopy real-spread study + Oanda/TD bar-divergence study Scripts + summary CSVs for the 2026-07-24 studies (real XAU spread ~$0.65 tradable hours; Oanda vs TD bars diverge ~11% of ATR with 10% direction flips). Large raw data files (parquet, per-boundary samples) gitignored. Claude-Session: https://claude.ai/code/session_01Mc7XKYyPySVXiUpcN82QEK --- .gitignore | 2 + backtest/research/duka_spread.py | 194 +++++++++++++++++ backtest/research/duka_spread_by_hour.csv | 25 +++ backtest/research/fetch_oanda_m15.py | 158 ++++++++++++++ backtest/research/oanda_td_divergence.py | 199 ++++++++++++++++++ .../research/oanda_td_divergence_by_hour.csv | 25 +++ 6 files changed, 603 insertions(+) create mode 100644 backtest/research/duka_spread.py create mode 100644 backtest/research/duka_spread_by_hour.csv create mode 100644 backtest/research/fetch_oanda_m15.py create mode 100644 backtest/research/oanda_td_divergence.py create mode 100644 backtest/research/oanda_td_divergence_by_hour.csv diff --git a/.gitignore b/.gitignore index 69b0ff9..d4dd623 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,8 @@ Thumbs.db # Backtesting data (keep scripts, ignore generated files) backtest/cache/ backtest/*.csv +backtest/research/*.parquet +backtest/research/duka_m5_boundary_spreads.csv backtest/trailgrid/ backtest/trailgrid5m/ backtest/__pycache__/ diff --git a/backtest/research/duka_spread.py b/backtest/research/duka_spread.py new file mode 100644 index 0000000..f5a29a9 --- /dev/null +++ b/backtest/research/duka_spread.py @@ -0,0 +1,194 @@ +"""Dukascopy XAU/USD tick download + bid/ask spread study. + +Validates ForexBot2's $0.30 round-trip cost assumption (config: +SignalConfig.round_trip_cost / PAIR_INFO default_spread). Round-trip +cost of a market entry+exit = one ask-bid spread width (buy at ask, +sell at bid), so the assumption holds iff typical spread ~ $0.30. + +bi5 format: LZMA 'alone' stream of 20-byte big-endian records +(ms_in_hour uint32, ask uint32, bid uint32, ask_vol f32, bid_vol f32); +XAUUSD prices scaled x1000. URL months are 0-indexed. +""" +import lzma +import struct +import sys +from concurrent.futures import ThreadPoolExecutor +from datetime import date, timedelta +from pathlib import Path + +import numpy as np +import pandas as pd +import requests + +SCALE = 1000.0 +CACHE = Path(__file__).parent / "tick_cache" +CACHE.mkdir(exist_ok=True) +URL = "https://datafeed.dukascopy.com/datafeed/XAUUSD/{y}/{m:02d}/{d:02d}/{h:02d}h_ticks.bi5" + +# Bot's tradable bar-close hours: session 03:00-23:00 minus XAU blocked +# hours [0,1,2,10,11,13,16,21] (M5 additionally blocks 9 — reported +# separately below). +BLOCKED = {0, 1, 2, 10, 11, 13, 16, 21} +TRADABLE = sorted(set(range(3, 23)) - BLOCKED) + +# ── Day sample ──────────────────────────────────────────────────────── +recent = [date(2026, 7, 6) + timedelta(days=i) for i in range(17)] +recent = [d for d in recent if d.weekday() < 5] # Mon-Fri Jul 6-22 +monthly = [date(2025, 8, 13), date(2025, 9, 10), date(2025, 10, 15), + date(2025, 11, 12), date(2025, 12, 10), date(2026, 1, 14), + date(2026, 2, 11), date(2026, 3, 11), date(2026, 4, 15), + date(2026, 5, 13), date(2026, 6, 10)] # mid-month Wednesdays +nfp = [date(2026, 6, 5), date(2026, 7, 3)] # NFP Fridays +sundays = [date(2026, 7, 12), date(2026, 7, 19)] # Sun 22-24 open +DAYS = sorted(set(recent + monthly + nfp + sundays)) +assert all(d.weekday() < 5 for d in recent + monthly + nfp) + + +_tls = __import__("threading").local() + + +def _session() -> requests.Session: + if not hasattr(_tls, "s"): + _tls.s = requests.Session() + _tls.s.headers["User-Agent"] = "Mozilla/5.0 (research; spread-study)" + return _tls.s + + +def fetch(day: date, hour: int) -> Path: + p = CACHE / f"{day.isoformat()}_{hour:02d}.bi5" + if p.exists(): + return p + u = URL.format(y=day.year, m=day.month - 1, d=day.day, h=hour) + last = None + for attempt in range(4): + try: + r = _session().get(u, timeout=30) + if r.status_code != 200: + raise RuntimeError(f"{u} -> HTTP {r.status_code}") + p.write_bytes(r.content) + return p + except (requests.ConnectionError, requests.Timeout, RuntimeError) as exc: + last = exc + __import__("time").sleep(1.5 * (attempt + 1)) + if hasattr(_tls, "s"): + _tls.s.close() + del _tls.s + raise RuntimeError(f"giving up on {u}: {last}") + + +def decode(p: Path, day: date, hour: int) -> pd.DataFrame: + blob = p.read_bytes() + if not blob: + return pd.DataFrame() + raw = lzma.decompress(blob) + n = len(raw) // 20 + a = np.frombuffer(raw[: n * 20], dtype=">u4").reshape(n, 5) + f = np.frombuffer(raw[: n * 20], dtype=">f4").reshape(n, 5) + df = pd.DataFrame({ + "ms": a[:, 0].astype(np.int64), + "ask": a[:, 1] / SCALE, + "bid": a[:, 2] / SCALE, + }) + df["spread"] = df["ask"] - df["bid"] + df["date"], df["hour"] = day.isoformat(), hour + df["weekday"] = day.weekday() + return df + + +def load_all() -> pd.DataFrame: + jobs = [(d, h) for d in DAYS for h in range(24)] + with ThreadPoolExecutor(max_workers=3) as ex: + paths = list(ex.map(lambda j: fetch(*j), jobs)) + frames = [decode(p, d, h) for p, (d, h) in zip(paths, jobs)] + df = pd.concat([f for f in frames if not f.empty], ignore_index=True) + return df + + +def wstats(g: pd.DataFrame) -> pd.Series: + """Tick stats + time-weighted mean (each tick's spread held until the + next tick; last tick held to hour end).""" + tw_num = tw_den = 0.0 + keys = [k for k in ("date", "hour") if k in g.columns] + for _, grp in g.groupby(keys, sort=False): + ms = grp["ms"].to_numpy() + sp = grp["spread"].to_numpy() + w = np.diff(np.append(ms, 3_600_000)).clip(min=0) + tw_num += float((sp * w).sum()) + tw_den += float(w.sum()) + s = g["spread"] + return pd.Series({ + "ticks": len(s), + "tw_mean": tw_num / tw_den if tw_den else np.nan, + "mean": s.mean(), "median": s.median(), + "p75": s.quantile(0.75), "p90": s.quantile(0.90), + "p99": s.quantile(0.99), "max": s.max(), + "pct_le_030": (s <= 0.30).mean() * 100, + "pct_ge_060": (s >= 0.60).mean() * 100, + }) + + +def m5_boundary_spreads(df: pd.DataFrame) -> pd.DataFrame: + """Spread of the FIRST tick at/after each M5 boundary — what a + next-open fill actually pays.""" + rows = [] + for (day, hour), grp in df.groupby(["date", "hour"], sort=False): + ms = grp["ms"].to_numpy() + sp = grp["spread"].to_numpy() + wd = grp["weekday"].iat[0] + for b in range(0, 3_600_000, 300_000): + i = np.searchsorted(ms, b) + if i < len(ms) and ms[i] - b <= 30_000: # tick within 30 s + rows.append((day, wd, hour, b // 60_000, sp[i])) + return pd.DataFrame(rows, columns=["date", "weekday", "hour", "minute", "spread"]) + + +def main() -> None: + df = load_all() + print(f"days={df['date'].nunique()} ticks={len(df):,} " + f"price range {df['bid'].min():.0f}-{df['ask'].max():.0f}") + + print("\n── Overall (all sampled hours) ──") + print(wstats(df).round(3).to_string()) + + trad = df[df["hour"].isin(TRADABLE) & (df["weekday"] < 5)] + print("\n── Bot-tradable hours (session 03-23 minus blocked, Mon-Fri) ──") + print(wstats(trad).round(3).to_string()) + + roll = df[df["hour"].isin([21, 22, 23])] + print("\n── Rollover window 21-24 UTC ──") + print(wstats(roll).round(3).to_string()) + + print("\n── Per UTC hour ──") + per_hour = df[df["weekday"] < 5].groupby("hour").apply(wstats, include_groups=False).round(3) + per_hour["tradable"] = ["Y" if h in TRADABLE else "-" for h in per_hour.index] + print(per_hour.to_string()) + per_hour.to_csv(Path(__file__).parent / "duka_spread_by_hour.csv") + + print("\n── M5 next-open boundary fills ──") + mb = m5_boundary_spreads(df) + mb_trad = mb[mb["hour"].isin(TRADABLE) & (mb["weekday"] < 5)] + for name, d in [("all hours", mb), ("tradable hours", mb_trad)]: + s = d["spread"] + print(f"{name}: n={len(s):,} mean={s.mean():.3f} median={s.median():.3f} " + f"p90={s.quantile(0.9):.3f} p99={s.quantile(0.99):.3f} " + f"%<=0.30={(s <= 0.30).mean()*100:.1f} %>=0.60={(s >= 0.60).mean()*100:.1f}") + mb.to_csv(Path(__file__).parent / "duka_m5_boundary_spreads.csv", index=False) + + print("\n── By month (tradable hours, tick median / tw_mean) ──") + trad2 = trad.copy() + trad2["month"] = trad2["date"].str[:7] + bym = trad2.groupby("month").apply(wstats, include_groups=False).round(3) + print(bym[["ticks", "tw_mean", "median", "p90", "pct_le_030", "pct_ge_060"]].to_string()) + + print("\n── NFP Fridays 12:00-15:00 UTC vs same hours other days ──") + nfp_days = {"2026-06-05", "2026-07-03"} + win = df[df["hour"].isin([12, 13, 14])] + for name, d in [("NFP", win[win["date"].isin(nfp_days)]), + ("non-NFP", win[~win["date"].isin(nfp_days) & (win["weekday"] < 5)])]: + s = d["spread"] + print(f"{name}: n={len(s):,} tick-median={s.median():.3f} p90={s.quantile(0.9):.3f} " + f"p99={s.quantile(0.99):.3f} max={s.max():.2f}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backtest/research/duka_spread_by_hour.csv b/backtest/research/duka_spread_by_hour.csv new file mode 100644 index 0000000..a96a3ea --- /dev/null +++ b/backtest/research/duka_spread_by_hour.csv @@ -0,0 +1,25 @@ +hour,ticks,tw_mean,mean,median,p75,p90,p99,max,pct_le_030,pct_ge_060,tradable +0,362893.0,0.749,0.768,0.79,0.82,0.86,1.1,7.69,0.053,94.561,- +1,540182.0,0.735,0.742,0.74,0.81,0.87,1.0,5.59,0.073,90.449,- +2,431589.0,0.716,0.722,0.73,0.8,0.86,0.95,2.96,0.1,85.27,- +3,343952.0,0.708,0.711,0.71,0.79,0.84,0.95,1.67,0.081,82.939,Y +4,246262.0,0.704,0.702,0.71,0.78,0.82,0.94,1.86,0.098,82.627,Y +5,388365.0,0.712,0.716,0.72,0.79,0.82,0.95,2.17,0.076,86.745,Y +6,368999.0,0.65,0.651,0.65,0.69,0.74,0.94,2.32,0.079,71.668,Y +7,313149.0,0.601,0.598,0.59,0.65,0.69,0.994,3.08,0.239,42.904,Y +8,347276.0,0.612,0.616,0.6,0.66,0.74,1.0,3.04,0.308,48.349,Y +9,321619.0,0.612,0.615,0.6,0.66,0.72,1.0,5.95,0.288,50.644,Y +10,295271.0,0.611,0.618,0.6,0.66,0.71,1.0,9.36,0.164,49.253,- +11,336963.0,0.619,0.624,0.6,0.67,0.79,1.0,4.97,0.203,50.178,- +12,476773.0,0.66,0.697,0.63,0.72,0.877,1.367,14.98,0.257,59.633,Y +13,651425.0,0.641,0.648,0.63,0.69,0.787,1.487,11.407,0.296,60.297,- +14,673391.0,0.625,0.625,0.63,0.68,0.77,0.9,11.98,0.458,59.085,Y +15,598398.0,0.636,0.638,0.64,0.7,0.78,0.91,4.49,0.433,66.296,Y +16,485023.0,0.648,0.644,0.64,0.7,0.757,0.917,4.09,0.118,68.496,- +17,450119.0,0.626,0.628,0.627,0.7,0.726,0.84,4.08,0.154,63.337,Y +18,411538.0,0.626,0.63,0.637,0.7,0.71,0.79,4.16,0.107,67.734,Y +19,435396.0,0.638,0.643,0.65,0.7,0.72,0.896,5.804,0.11,72.007,Y +20,212654.0,0.684,0.665,0.66,0.7,0.76,1.217,7.15,0.226,70.215,Y +21,31319.0,0.627,0.622,0.61,0.68,0.75,1.02,2.08,0.089,53.153,- +22,108879.0,0.829,0.834,0.797,0.84,0.98,2.05,10.11,0.056,95.434,Y +23,161079.0,1.032,0.913,0.8,0.89,1.307,2.77,13.174,0.092,95.41,- diff --git a/backtest/research/fetch_oanda_m15.py b/backtest/research/fetch_oanda_m15.py new file mode 100644 index 0000000..7a80f75 --- /dev/null +++ b/backtest/research/fetch_oanda_m15.py @@ -0,0 +1,158 @@ +"""Fetch XAU/USD M15 candles from the Oanda v20 REST API. + +Mechanical data-fetching infrastructure only — no analysis here. +Paginates GET /v3/instruments/XAU_USD/candles from a start cursor +through "now" and writes the result to a parquet file. + +Run from the repo root: + venv/bin/python backtest/research/fetch_oanda_m15.py +""" +import sys +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pandas as pd +import requests + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from app_config import oanda_api_token, oanda_base_url # noqa: E402 + +INSTRUMENT = "XAU_USD" +GRANULARITY = "M15" +START = datetime(2025, 7, 1, 0, 0, 0, tzinfo=timezone.utc) +OUT_PATH = Path(__file__).parent / "oanda_xau_m15_20250701_now.parquet" +COUNT = 5000 +REQUEST_SLEEP_S = 0.5 +FRESHNESS_WINDOW = timedelta(minutes=30) + + +def _rfc3339(dt: datetime) -> str: + return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def fetch_all_candles(): + token = oanda_api_token() + base_url = oanda_base_url() + if not token: + raise RuntimeError( + "No Oanda API token resolved (env OANDA_API_TOKEN / " + "config.json oanda.api_token). Aborting.") + + url = f"{base_url}/v3/instruments/{INSTRUMENT}/candles" + headers = { + "Authorization": f"Bearer {token}", + "Accept-Datetime-Format": "RFC3339", + } + + cursor = START + include_first = True + all_rows = [] + n_requests = 0 + + while True: + params = { + "granularity": GRANULARITY, + "price": "MBA", + "count": COUNT, + "dailyAlignment": 0, + "alignmentTimezone": "Etc/UTC", + "from": _rfc3339(cursor), + "includeFirst": "true" if include_first else "false", + } + + resp = requests.get(url, headers=headers, params=params, timeout=30) + n_requests += 1 + if resp.status_code != 200: + raise RuntimeError( + f"Oanda API request failed: status={resp.status_code}\n" + f"URL: {resp.url}\n" + f"Response body: {resp.text}") + + payload = resp.json() + candles = payload.get("candles", []) + + for c in candles: + if not c.get("complete", False): + continue + mid = c["mid"] + bid = c["bid"] + ask = c["ask"] + all_rows.append({ + "Datetime": pd.Timestamp(c["time"]).tz_convert("UTC") + if pd.Timestamp(c["time"]).tzinfo is not None + else pd.Timestamp(c["time"]).tz_localize("UTC"), + "Open": float(mid["o"]), + "High": float(mid["h"]), + "Low": float(mid["l"]), + "Close": float(mid["c"]), + "Volume": int(c["volume"]), + "bid_o": float(bid["o"]), + "bid_h": float(bid["h"]), + "bid_l": float(bid["l"]), + "bid_c": float(bid["c"]), + "ask_o": float(ask["o"]), + "ask_h": float(ask["h"]), + "ask_l": float(ask["l"]), + "ask_c": float(ask["c"]), + }) + + print(f" request {n_requests}: from={params['from']} " + f"got {len(candles)} candles (complete kept so far: " + f"{len(all_rows)})", file=sys.stderr) + + if not candles: + break + + last_time = pd.Timestamp(candles[-1]["time"]) + if last_time.tzinfo is None: + last_time = last_time.tz_localize("UTC") + else: + last_time = last_time.tz_convert("UTC") + + now = datetime.now(timezone.utc) + # Oanda's "count" includes the "from" boundary bar itself, so with + # includeFirst=false every full page comes back as count-1, not + # count (verified empirically: count=N, includeFirst=false always + # yields N-1 candles when a full page of data exists ahead of the + # cursor). Compare against the count actually expected for this + # request's includeFirst setting, not the raw requested count. + expected_full_page = COUNT if include_first else COUNT - 1 + got_fewer_than_requested = len(candles) < expected_full_page + near_now = (now - last_time.to_pydatetime()) < FRESHNESS_WINDOW + + if got_fewer_than_requested or near_now: + break + + cursor = last_time.to_pydatetime() + include_first = False + time.sleep(REQUEST_SLEEP_S) + + return all_rows, n_requests + + +def main(): + print(f"Fetching {INSTRUMENT} {GRANULARITY} candles from " + f"{_rfc3339(START)} through now...", file=sys.stderr) + + rows, n_requests = fetch_all_candles() + + if not rows: + raise RuntimeError("No complete candles were fetched.") + + df = pd.DataFrame(rows) + df = df.set_index("Datetime") + df = df.sort_index() + df = df[~df.index.duplicated(keep="last")] + + df.to_parquet(OUT_PATH) + + print(f"\nSaved {len(df)} rows to {OUT_PATH}", file=sys.stderr) + print(f"Requests made: {n_requests}", file=sys.stderr) + print(f"First timestamp: {df.index[0]}", file=sys.stderr) + print(f"Last timestamp: {df.index[-1]}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/backtest/research/oanda_td_divergence.py b/backtest/research/oanda_td_divergence.py new file mode 100644 index 0000000..e954247 --- /dev/null +++ b/backtest/research/oanda_td_divergence.py @@ -0,0 +1,199 @@ +""" +Oanda vs Twelve Data bar-divergence study — XAU/USD M15, 12 months. +==================================================================== +Compares the mid-price OHLC bars the signal engine would see from each +provider over the same window. Purpose: quantify whether migrating the +signal feed from Twelve Data to Oanda is a formality (bars ~identical) +or a recalibration (bars differ at a scale that moves indicators). + +Inputs: + --td Twelve Data parquet (raw bars from backtest/cache/) + --oanda Oanda parquet (from fetch_oanda_m5.py / fetch_oanda_m15.py) + --start / --end analysis window (UTC dates) + +Outputs: + printed report + backtest/research/oanda_td_divergence_by_hour.csv + backtest/research/oanda_td_bar_diffs.parquet (per-bar diffs, common index) +""" + +import argparse +from pathlib import Path + +import numpy as np +import pandas as pd + +RESEARCH_DIR = Path(__file__).parent +PIP = 0.10 # XAU pip in $ + + +def wilder_atr(df: pd.DataFrame, period: int = 14) -> pd.Series: + hl = df["High"] - df["Low"] + hc = (df["High"] - df["Close"].shift(1)).abs() + lc = (df["Low"] - df["Close"].shift(1)).abs() + tr = pd.concat([hl, hc, lc], axis=1).max(axis=1) + return tr.ewm(alpha=1.0 / period, adjust=False).mean() + + +def pct_stats(s: pd.Series) -> dict: + a = s.abs() + return { + "mean": a.mean(), "median": a.median(), + "p90": a.quantile(0.90), "p95": a.quantile(0.95), + "p99": a.quantile(0.99), "max": a.max(), + "bias(mean signed)": s.mean(), + } + + +def fmt_stats(name: str, s: pd.Series) -> str: + st = pct_stats(s) + return (f" {name:<6} mean ${st['mean']:.3f} med ${st['median']:.3f} " + f"p95 ${st['p95']:.3f} p99 ${st['p99']:.3f} max ${st['max']:.2f} " + f"bias {st['bias(mean signed)']:+.4f}") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--td", default=str(RESEARCH_DIR.parent / "cache" / + "XAU_USD_15min_20240522_20260723.parquet")) + ap.add_argument("--oanda", default=str(RESEARCH_DIR / + "oanda_xau_m15_20250701_now.parquet")) + ap.add_argument("--start", default="2025-07-24") + ap.add_argument("--end", default="2026-07-23") + args = ap.parse_args() + + td = pd.read_parquet(args.td) + oa = pd.read_parquet(args.oanda) + + start = pd.Timestamp(args.start, tz="UTC") + end = pd.Timestamp(args.end, tz="UTC") + pd.Timedelta(days=1) + start = max(start, td.index[0], oa.index[0]) + end = min(end, td.index[-1], oa.index[-1]) + td = td.loc[(td.index >= start) & (td.index <= end)] + oa = oa.loc[(oa.index >= start) & (oa.index <= end)] + + print("=" * 78) + print("OANDA vs TWELVE DATA — XAU/USD M15 bar divergence") + print(f"Window: {start} -> {end} ({(end - start).days} days)") + print("=" * 78) + + # ── Coverage ───────────────────────────────────────────────────── + common = td.index.intersection(oa.index) + td_only = td.index.difference(oa.index) + oa_only = oa.index.difference(td.index) + print(f"\nCOVERAGE") + print(f" Twelve Data bars : {len(td):>7,}") + print(f" Oanda bars : {len(oa):>7,}") + print(f" common : {len(common):>7,}") + print(f" TD-only : {len(td_only):>7,} " + f"({100 * len(td_only) / max(len(td), 1):.2f}% of TD)") + print(f" Oanda-only : {len(oa_only):>7,} " + f"({100 * len(oa_only) / max(len(oa), 1):.2f}% of Oanda)") + for label, idx in (("TD-only", td_only), ("Oanda-only", oa_only)): + if len(idx): + by_date = pd.Series(1, index=idx).groupby(idx.date).sum() + top = by_date.sort_values(ascending=False).head(8) + print(f" {label} bars cluster on: " + + ", ".join(f"{d} ({n})" for d, n in top.items())) + by_hour = pd.Series(1, index=idx).groupby(idx.hour).sum() + top_h = by_hour.sort_values(ascending=False).head(6) + print(f" {label} by UTC hour: " + + ", ".join(f"{h:02d}h ({n})" for h, n in top_h.items())) + + t = td.loc[common] + o = oa.loc[common] + + # ── Scale references ───────────────────────────────────────────── + atr = wilder_atr(t).dropna() + mean_price = t["Close"].mean() + print(f"\nSCALE REFERENCES (Twelve Data feed)") + print(f" mean close : ${mean_price:,.2f}") + print(f" mean bar range : ${(t['High'] - t['Low']).mean():.3f}") + print(f" mean ATR14 : ${atr.mean():.3f} median ${atr.median():.3f}") + + # ── Per-field diffs (TD − Oanda mid) ───────────────────────────── + print(f"\nPER-BAR DIFFS (TD − Oanda, $; pip = $0.10)") + diffs = {} + for col in ("Open", "High", "Low", "Close"): + d = t[col] - o[col] + diffs[col] = d + print(fmt_stats(col, d)) + + dclose = diffs["Close"].abs() + print(f"\n |ΔClose| as % of mean ATR14 : " + f"mean {100 * dclose.mean() / atr.mean():.1f}% " + f"p95 {100 * dclose.quantile(0.95) / atr.mean():.1f}%") + + print(f"\nTHRESHOLD EXCEEDANCE (|ΔClose|)") + for thr in (0.10, 0.30, 0.65, 1.00, 2.00, 5.00): + pct = 100 * (dclose > thr).mean() + print(f" > ${thr:<5.2f}: {pct:6.2f}% of bars ({(dclose > thr).sum():,})") + + # ── Direction / return agreement ───────────────────────────────── + t_ret = t["Close"].pct_change() + o_ret = o["Close"].pct_change() + # only consecutive common bars (gap = one M15 step) + step_ok = pd.Series(common, index=common).diff().eq(pd.Timedelta(minutes=15)) + tr, orr = t_ret[step_ok], o_ret[step_ok] + corr = tr.corr(orr) + t_dir = np.sign(t["Close"] - t["Open"]) + o_dir = np.sign(o["Close"] - o["Open"]) + both_nonflat = (t_dir != 0) & (o_dir != 0) + dir_agree = 100 * (t_dir[both_nonflat] == o_dir[both_nonflat]).mean() + print(f"\nAGREEMENT") + print(f" close-return correlation (consecutive bars): {corr:.5f} " + f"(n={step_ok.sum():,})") + print(f" bar direction (sign of close−open) agree : {dir_agree:.2f}%") + + # ── By-hour breakdown ──────────────────────────────────────────── + hours = common.hour + rows = [] + for h in range(24): + m = hours == h + if not m.any(): + continue + row = {"hour_utc": h, "bars": int(m.sum()), + "mean_abs_dclose": dclose[m].mean(), + "p95_abs_dclose": dclose[m].quantile(0.95), + "mean_abs_dhigh": diffs["High"][m].abs().mean(), + "mean_abs_dlow": diffs["Low"][m].abs().mean()} + if "bid_c" in o.columns: + row["oanda_median_spread"] = ( + (o["ask_c"] - o["bid_c"])[m].median()) + rows.append(row) + by_hour = pd.DataFrame(rows) + out_hour = RESEARCH_DIR / "oanda_td_divergence_by_hour.csv" + by_hour.to_csv(out_hour, index=False, float_format="%.4f") + print(f"\nBY UTC HOUR (written to {out_hour.name})") + print(by_hour.to_string(index=False, + float_format=lambda x: f"{x:.3f}")) + + # ── By-month drift ─────────────────────────────────────────────── + print(f"\nBY MONTH (mean |ΔClose|, $)") + by_month = dclose.groupby(common.to_period("M")).mean() + for p, v in by_month.items(): + print(f" {p}: {v:.3f}") + + # ── Oanda real spread (bonus, ties to Dukascopy study) ─────────── + if "bid_c" in o.columns: + spread = o["ask_c"] - o["bid_c"] + liquid = ~o.index.hour.isin([21, 22, 23, 0]) + print(f"\nOANDA REAL SPREAD (close bid/ask, $)") + print(f" all bars : median {spread.median():.3f} " + f"p25 {spread.quantile(0.25):.3f} p75 {spread.quantile(0.75):.3f}") + print(f" liquid hours : median {spread[liquid].median():.3f}") + print(f" 21-00 UTC : median {spread[~liquid].median():.3f}") + + # ── Persist per-bar diffs ──────────────────────────────────────── + out = pd.DataFrame({ + "td_close": t["Close"], "oa_close": o["Close"], + "d_open": diffs["Open"], "d_high": diffs["High"], + "d_low": diffs["Low"], "d_close": diffs["Close"], + }) + out_path = RESEARCH_DIR / "oanda_td_bar_diffs.parquet" + out.to_parquet(out_path) + print(f"\nPer-bar diffs written to {out_path}") + + +if __name__ == "__main__": + main() diff --git a/backtest/research/oanda_td_divergence_by_hour.csv b/backtest/research/oanda_td_divergence_by_hour.csv new file mode 100644 index 0000000..60e50e0 --- /dev/null +++ b/backtest/research/oanda_td_divergence_by_hour.csv @@ -0,0 +1,25 @@ +hour_utc,bars,mean_abs_dclose,p95_abs_dclose,mean_abs_dhigh,mean_abs_dlow,oanda_median_spread +0,1029,3.3483,13.9460,3.2906,3.4310,0.7900 +1,1028,3.3483,12.8292,3.3477,3.6398,0.7700 +2,1028,3.0288,10.8960,2.9469,3.1317,0.7700 +3,1028,2.8783,11.1300,2.8419,2.9992,0.7600 +4,1028,2.8232,10.4710,2.8165,2.8455,0.7400 +5,1028,2.9268,10.6027,2.9234,3.0038,0.7400 +6,1028,3.0466,10.9265,2.9487,3.0796,0.7200 +7,1028,3.0286,11.3512,3.0301,3.0797,0.6700 +8,1028,3.0223,11.1483,2.8989,3.0801,0.6600 +9,1028,3.0460,11.6345,2.9307,3.0609,0.6600 +10,1028,2.9280,10.9015,2.8647,3.0029,0.6600 +11,1028,3.0340,11.4712,2.9591,2.9472,0.6600 +12,1028,3.0986,11.3282,2.9412,3.1407,0.7000 +13,1028,3.4014,12.3343,3.1077,3.4228,0.7300 +14,1028,3.5074,11.3628,3.3960,3.6569,0.6900 +15,1028,3.4514,11.8982,3.4124,3.6733,0.6700 +16,1028,3.1275,12.5155,3.2038,3.3061,0.6700 +17,1020,2.9826,12.6435,3.0064,3.1008,0.6900 +18,1015,2.8721,12.5480,2.9101,2.9996,0.6900 +19,1004,2.8637,12.1335,2.8824,2.9454,0.6900 +20,996,2.8006,11.9325,2.8486,2.8577,0.7600 +21,335,1.1085,4.8600,1.1212,1.2034,0.9400 +22,676,4.0827,13.6787,4.8422,4.6761,0.8300 +23,1027,3.2722,12.5725,3.5617,3.7747,0.8700