""" Oanda vs Twelve Data bar-divergence study — XAU/USD M15, 12 months. ==================================================================== Compares the mid-price OHLC bars the signal engine would see from each provider over the same window. Purpose: quantify whether migrating the signal feed from Twelve Data to Oanda is a formality (bars ~identical) or a recalibration (bars differ at a scale that moves indicators). Inputs: --td Twelve Data parquet (raw bars from backtest/cache/) --oanda Oanda parquet (from fetch_oanda_m5.py / fetch_oanda_m15.py) --start / --end analysis window (UTC dates) Outputs: printed report backtest/research/oanda_td_divergence_by_hour.csv backtest/research/oanda_td_bar_diffs.parquet (per-bar diffs, common index) """ import argparse from pathlib import Path import numpy as np import pandas as pd RESEARCH_DIR = Path(__file__).parent PIP = 0.10 # XAU pip in $ def wilder_atr(df: pd.DataFrame, period: int = 14) -> pd.Series: hl = df["High"] - df["Low"] hc = (df["High"] - df["Close"].shift(1)).abs() lc = (df["Low"] - df["Close"].shift(1)).abs() tr = pd.concat([hl, hc, lc], axis=1).max(axis=1) return tr.ewm(alpha=1.0 / period, adjust=False).mean() def pct_stats(s: pd.Series) -> dict: a = s.abs() return { "mean": a.mean(), "median": a.median(), "p90": a.quantile(0.90), "p95": a.quantile(0.95), "p99": a.quantile(0.99), "max": a.max(), "bias(mean signed)": s.mean(), } def fmt_stats(name: str, s: pd.Series) -> str: st = pct_stats(s) return (f" {name:<6} mean ${st['mean']:.3f} med ${st['median']:.3f} " f"p95 ${st['p95']:.3f} p99 ${st['p99']:.3f} max ${st['max']:.2f} " f"bias {st['bias(mean signed)']:+.4f}") def main(): ap = argparse.ArgumentParser() ap.add_argument("--td", default=str(RESEARCH_DIR.parent / "cache" / "XAU_USD_15min_20240522_20260723.parquet")) ap.add_argument("--oanda", default=str(RESEARCH_DIR / "oanda_xau_m15_20250701_now.parquet")) ap.add_argument("--start", default="2025-07-24") ap.add_argument("--end", default="2026-07-23") args = ap.parse_args() td = pd.read_parquet(args.td) oa = pd.read_parquet(args.oanda) start = pd.Timestamp(args.start, tz="UTC") end = pd.Timestamp(args.end, tz="UTC") + pd.Timedelta(days=1) start = max(start, td.index[0], oa.index[0]) end = min(end, td.index[-1], oa.index[-1]) td = td.loc[(td.index >= start) & (td.index <= end)] oa = oa.loc[(oa.index >= start) & (oa.index <= end)] print("=" * 78) print("OANDA vs TWELVE DATA — XAU/USD M15 bar divergence") print(f"Window: {start} -> {end} ({(end - start).days} days)") print("=" * 78) # ── Coverage ───────────────────────────────────────────────────── common = td.index.intersection(oa.index) td_only = td.index.difference(oa.index) oa_only = oa.index.difference(td.index) print(f"\nCOVERAGE") print(f" Twelve Data bars : {len(td):>7,}") print(f" Oanda bars : {len(oa):>7,}") print(f" common : {len(common):>7,}") print(f" TD-only : {len(td_only):>7,} " f"({100 * len(td_only) / max(len(td), 1):.2f}% of TD)") print(f" Oanda-only : {len(oa_only):>7,} " f"({100 * len(oa_only) / max(len(oa), 1):.2f}% of Oanda)") for label, idx in (("TD-only", td_only), ("Oanda-only", oa_only)): if len(idx): by_date = pd.Series(1, index=idx).groupby(idx.date).sum() top = by_date.sort_values(ascending=False).head(8) print(f" {label} bars cluster on: " + ", ".join(f"{d} ({n})" for d, n in top.items())) by_hour = pd.Series(1, index=idx).groupby(idx.hour).sum() top_h = by_hour.sort_values(ascending=False).head(6) print(f" {label} by UTC hour: " + ", ".join(f"{h:02d}h ({n})" for h, n in top_h.items())) t = td.loc[common] o = oa.loc[common] # ── Scale references ───────────────────────────────────────────── atr = wilder_atr(t).dropna() mean_price = t["Close"].mean() print(f"\nSCALE REFERENCES (Twelve Data feed)") print(f" mean close : ${mean_price:,.2f}") print(f" mean bar range : ${(t['High'] - t['Low']).mean():.3f}") print(f" mean ATR14 : ${atr.mean():.3f} median ${atr.median():.3f}") # ── Per-field diffs (TD − Oanda mid) ───────────────────────────── print(f"\nPER-BAR DIFFS (TD − Oanda, $; pip = $0.10)") diffs = {} for col in ("Open", "High", "Low", "Close"): d = t[col] - o[col] diffs[col] = d print(fmt_stats(col, d)) dclose = diffs["Close"].abs() print(f"\n |ΔClose| as % of mean ATR14 : " f"mean {100 * dclose.mean() / atr.mean():.1f}% " f"p95 {100 * dclose.quantile(0.95) / atr.mean():.1f}%") print(f"\nTHRESHOLD EXCEEDANCE (|ΔClose|)") for thr in (0.10, 0.30, 0.65, 1.00, 2.00, 5.00): pct = 100 * (dclose > thr).mean() print(f" > ${thr:<5.2f}: {pct:6.2f}% of bars ({(dclose > thr).sum():,})") # ── Direction / return agreement ───────────────────────────────── t_ret = t["Close"].pct_change() o_ret = o["Close"].pct_change() # only consecutive common bars (gap = one M15 step) step_ok = pd.Series(common, index=common).diff().eq(pd.Timedelta(minutes=15)) tr, orr = t_ret[step_ok], o_ret[step_ok] corr = tr.corr(orr) t_dir = np.sign(t["Close"] - t["Open"]) o_dir = np.sign(o["Close"] - o["Open"]) both_nonflat = (t_dir != 0) & (o_dir != 0) dir_agree = 100 * (t_dir[both_nonflat] == o_dir[both_nonflat]).mean() print(f"\nAGREEMENT") print(f" close-return correlation (consecutive bars): {corr:.5f} " f"(n={step_ok.sum():,})") print(f" bar direction (sign of close−open) agree : {dir_agree:.2f}%") # ── By-hour breakdown ──────────────────────────────────────────── hours = common.hour rows = [] for h in range(24): m = hours == h if not m.any(): continue row = {"hour_utc": h, "bars": int(m.sum()), "mean_abs_dclose": dclose[m].mean(), "p95_abs_dclose": dclose[m].quantile(0.95), "mean_abs_dhigh": diffs["High"][m].abs().mean(), "mean_abs_dlow": diffs["Low"][m].abs().mean()} if "bid_c" in o.columns: row["oanda_median_spread"] = ( (o["ask_c"] - o["bid_c"])[m].median()) rows.append(row) by_hour = pd.DataFrame(rows) out_hour = RESEARCH_DIR / "oanda_td_divergence_by_hour.csv" by_hour.to_csv(out_hour, index=False, float_format="%.4f") print(f"\nBY UTC HOUR (written to {out_hour.name})") print(by_hour.to_string(index=False, float_format=lambda x: f"{x:.3f}")) # ── By-month drift ─────────────────────────────────────────────── print(f"\nBY MONTH (mean |ΔClose|, $)") by_month = dclose.groupby(common.to_period("M")).mean() for p, v in by_month.items(): print(f" {p}: {v:.3f}") # ── Oanda real spread (bonus, ties to Dukascopy study) ─────────── if "bid_c" in o.columns: spread = o["ask_c"] - o["bid_c"] liquid = ~o.index.hour.isin([21, 22, 23, 0]) print(f"\nOANDA REAL SPREAD (close bid/ask, $)") print(f" all bars : median {spread.median():.3f} " f"p25 {spread.quantile(0.25):.3f} p75 {spread.quantile(0.75):.3f}") print(f" liquid hours : median {spread[liquid].median():.3f}") print(f" 21-00 UTC : median {spread[~liquid].median():.3f}") # ── Persist per-bar diffs ──────────────────────────────────────── out = pd.DataFrame({ "td_close": t["Close"], "oa_close": o["Close"], "d_open": diffs["Open"], "d_high": diffs["High"], "d_low": diffs["Low"], "d_close": diffs["Close"], }) out_path = RESEARCH_DIR / "oanda_td_bar_diffs.parquet" out.to_parquet(out_path) print(f"\nPer-bar diffs written to {out_path}") if __name__ == "__main__": main()