Harwinder Sekhon
f760e9070a
Scripts + summary CSVs for the 2026-07-24 studies (real XAU spread ~$0.65 tradable hours; Oanda vs TD bars diverge ~11% of ATR with 10% direction flips). Large raw data files (parquet, per-boundary samples) gitignored. Claude-Session: https://claude.ai/code/session_01Mc7XKYyPySVXiUpcN82QEK
200 lines
8.7 KiB
Python
200 lines
8.7 KiB
Python
"""
|
||
Oanda vs Twelve Data bar-divergence study — XAU/USD M15, 12 months.
|
||
====================================================================
|
||
Compares the mid-price OHLC bars the signal engine would see from each
|
||
provider over the same window. Purpose: quantify whether migrating the
|
||
signal feed from Twelve Data to Oanda is a formality (bars ~identical)
|
||
or a recalibration (bars differ at a scale that moves indicators).
|
||
|
||
Inputs:
|
||
--td Twelve Data parquet (raw bars from backtest/cache/)
|
||
--oanda Oanda parquet (from fetch_oanda_m5.py / fetch_oanda_m15.py)
|
||
--start / --end analysis window (UTC dates)
|
||
|
||
Outputs:
|
||
printed report
|
||
backtest/research/oanda_td_divergence_by_hour.csv
|
||
backtest/research/oanda_td_bar_diffs.parquet (per-bar diffs, common index)
|
||
"""
|
||
|
||
import argparse
|
||
from pathlib import Path
|
||
|
||
import numpy as np
|
||
import pandas as pd
|
||
|
||
RESEARCH_DIR = Path(__file__).parent
|
||
PIP = 0.10 # XAU pip in $
|
||
|
||
|
||
def wilder_atr(df: pd.DataFrame, period: int = 14) -> pd.Series:
|
||
hl = df["High"] - df["Low"]
|
||
hc = (df["High"] - df["Close"].shift(1)).abs()
|
||
lc = (df["Low"] - df["Close"].shift(1)).abs()
|
||
tr = pd.concat([hl, hc, lc], axis=1).max(axis=1)
|
||
return tr.ewm(alpha=1.0 / period, adjust=False).mean()
|
||
|
||
|
||
def pct_stats(s: pd.Series) -> dict:
|
||
a = s.abs()
|
||
return {
|
||
"mean": a.mean(), "median": a.median(),
|
||
"p90": a.quantile(0.90), "p95": a.quantile(0.95),
|
||
"p99": a.quantile(0.99), "max": a.max(),
|
||
"bias(mean signed)": s.mean(),
|
||
}
|
||
|
||
|
||
def fmt_stats(name: str, s: pd.Series) -> str:
|
||
st = pct_stats(s)
|
||
return (f" {name:<6} mean ${st['mean']:.3f} med ${st['median']:.3f} "
|
||
f"p95 ${st['p95']:.3f} p99 ${st['p99']:.3f} max ${st['max']:.2f} "
|
||
f"bias {st['bias(mean signed)']:+.4f}")
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--td", default=str(RESEARCH_DIR.parent / "cache" /
|
||
"XAU_USD_15min_20240522_20260723.parquet"))
|
||
ap.add_argument("--oanda", default=str(RESEARCH_DIR /
|
||
"oanda_xau_m15_20250701_now.parquet"))
|
||
ap.add_argument("--start", default="2025-07-24")
|
||
ap.add_argument("--end", default="2026-07-23")
|
||
args = ap.parse_args()
|
||
|
||
td = pd.read_parquet(args.td)
|
||
oa = pd.read_parquet(args.oanda)
|
||
|
||
start = pd.Timestamp(args.start, tz="UTC")
|
||
end = pd.Timestamp(args.end, tz="UTC") + pd.Timedelta(days=1)
|
||
start = max(start, td.index[0], oa.index[0])
|
||
end = min(end, td.index[-1], oa.index[-1])
|
||
td = td.loc[(td.index >= start) & (td.index <= end)]
|
||
oa = oa.loc[(oa.index >= start) & (oa.index <= end)]
|
||
|
||
print("=" * 78)
|
||
print("OANDA vs TWELVE DATA — XAU/USD M15 bar divergence")
|
||
print(f"Window: {start} -> {end} ({(end - start).days} days)")
|
||
print("=" * 78)
|
||
|
||
# ── Coverage ─────────────────────────────────────────────────────
|
||
common = td.index.intersection(oa.index)
|
||
td_only = td.index.difference(oa.index)
|
||
oa_only = oa.index.difference(td.index)
|
||
print(f"\nCOVERAGE")
|
||
print(f" Twelve Data bars : {len(td):>7,}")
|
||
print(f" Oanda bars : {len(oa):>7,}")
|
||
print(f" common : {len(common):>7,}")
|
||
print(f" TD-only : {len(td_only):>7,} "
|
||
f"({100 * len(td_only) / max(len(td), 1):.2f}% of TD)")
|
||
print(f" Oanda-only : {len(oa_only):>7,} "
|
||
f"({100 * len(oa_only) / max(len(oa), 1):.2f}% of Oanda)")
|
||
for label, idx in (("TD-only", td_only), ("Oanda-only", oa_only)):
|
||
if len(idx):
|
||
by_date = pd.Series(1, index=idx).groupby(idx.date).sum()
|
||
top = by_date.sort_values(ascending=False).head(8)
|
||
print(f" {label} bars cluster on: "
|
||
+ ", ".join(f"{d} ({n})" for d, n in top.items()))
|
||
by_hour = pd.Series(1, index=idx).groupby(idx.hour).sum()
|
||
top_h = by_hour.sort_values(ascending=False).head(6)
|
||
print(f" {label} by UTC hour: "
|
||
+ ", ".join(f"{h:02d}h ({n})" for h, n in top_h.items()))
|
||
|
||
t = td.loc[common]
|
||
o = oa.loc[common]
|
||
|
||
# ── Scale references ─────────────────────────────────────────────
|
||
atr = wilder_atr(t).dropna()
|
||
mean_price = t["Close"].mean()
|
||
print(f"\nSCALE REFERENCES (Twelve Data feed)")
|
||
print(f" mean close : ${mean_price:,.2f}")
|
||
print(f" mean bar range : ${(t['High'] - t['Low']).mean():.3f}")
|
||
print(f" mean ATR14 : ${atr.mean():.3f} median ${atr.median():.3f}")
|
||
|
||
# ── Per-field diffs (TD − Oanda mid) ─────────────────────────────
|
||
print(f"\nPER-BAR DIFFS (TD − Oanda, $; pip = $0.10)")
|
||
diffs = {}
|
||
for col in ("Open", "High", "Low", "Close"):
|
||
d = t[col] - o[col]
|
||
diffs[col] = d
|
||
print(fmt_stats(col, d))
|
||
|
||
dclose = diffs["Close"].abs()
|
||
print(f"\n |ΔClose| as % of mean ATR14 : "
|
||
f"mean {100 * dclose.mean() / atr.mean():.1f}% "
|
||
f"p95 {100 * dclose.quantile(0.95) / atr.mean():.1f}%")
|
||
|
||
print(f"\nTHRESHOLD EXCEEDANCE (|ΔClose|)")
|
||
for thr in (0.10, 0.30, 0.65, 1.00, 2.00, 5.00):
|
||
pct = 100 * (dclose > thr).mean()
|
||
print(f" > ${thr:<5.2f}: {pct:6.2f}% of bars ({(dclose > thr).sum():,})")
|
||
|
||
# ── Direction / return agreement ─────────────────────────────────
|
||
t_ret = t["Close"].pct_change()
|
||
o_ret = o["Close"].pct_change()
|
||
# only consecutive common bars (gap = one M15 step)
|
||
step_ok = pd.Series(common, index=common).diff().eq(pd.Timedelta(minutes=15))
|
||
tr, orr = t_ret[step_ok], o_ret[step_ok]
|
||
corr = tr.corr(orr)
|
||
t_dir = np.sign(t["Close"] - t["Open"])
|
||
o_dir = np.sign(o["Close"] - o["Open"])
|
||
both_nonflat = (t_dir != 0) & (o_dir != 0)
|
||
dir_agree = 100 * (t_dir[both_nonflat] == o_dir[both_nonflat]).mean()
|
||
print(f"\nAGREEMENT")
|
||
print(f" close-return correlation (consecutive bars): {corr:.5f} "
|
||
f"(n={step_ok.sum():,})")
|
||
print(f" bar direction (sign of close−open) agree : {dir_agree:.2f}%")
|
||
|
||
# ── By-hour breakdown ────────────────────────────────────────────
|
||
hours = common.hour
|
||
rows = []
|
||
for h in range(24):
|
||
m = hours == h
|
||
if not m.any():
|
||
continue
|
||
row = {"hour_utc": h, "bars": int(m.sum()),
|
||
"mean_abs_dclose": dclose[m].mean(),
|
||
"p95_abs_dclose": dclose[m].quantile(0.95),
|
||
"mean_abs_dhigh": diffs["High"][m].abs().mean(),
|
||
"mean_abs_dlow": diffs["Low"][m].abs().mean()}
|
||
if "bid_c" in o.columns:
|
||
row["oanda_median_spread"] = (
|
||
(o["ask_c"] - o["bid_c"])[m].median())
|
||
rows.append(row)
|
||
by_hour = pd.DataFrame(rows)
|
||
out_hour = RESEARCH_DIR / "oanda_td_divergence_by_hour.csv"
|
||
by_hour.to_csv(out_hour, index=False, float_format="%.4f")
|
||
print(f"\nBY UTC HOUR (written to {out_hour.name})")
|
||
print(by_hour.to_string(index=False,
|
||
float_format=lambda x: f"{x:.3f}"))
|
||
|
||
# ── By-month drift ───────────────────────────────────────────────
|
||
print(f"\nBY MONTH (mean |ΔClose|, $)")
|
||
by_month = dclose.groupby(common.to_period("M")).mean()
|
||
for p, v in by_month.items():
|
||
print(f" {p}: {v:.3f}")
|
||
|
||
# ── Oanda real spread (bonus, ties to Dukascopy study) ───────────
|
||
if "bid_c" in o.columns:
|
||
spread = o["ask_c"] - o["bid_c"]
|
||
liquid = ~o.index.hour.isin([21, 22, 23, 0])
|
||
print(f"\nOANDA REAL SPREAD (close bid/ask, $)")
|
||
print(f" all bars : median {spread.median():.3f} "
|
||
f"p25 {spread.quantile(0.25):.3f} p75 {spread.quantile(0.75):.3f}")
|
||
print(f" liquid hours : median {spread[liquid].median():.3f}")
|
||
print(f" 21-00 UTC : median {spread[~liquid].median():.3f}")
|
||
|
||
# ── Persist per-bar diffs ────────────────────────────────────────
|
||
out = pd.DataFrame({
|
||
"td_close": t["Close"], "oa_close": o["Close"],
|
||
"d_open": diffs["Open"], "d_high": diffs["High"],
|
||
"d_low": diffs["Low"], "d_close": diffs["Close"],
|
||
})
|
||
out_path = RESEARCH_DIR / "oanda_td_bar_diffs.parquet"
|
||
out.to_parquet(out_path)
|
||
print(f"\nPer-bar diffs written to {out_path}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|