From 356a09e5abd63ed6eec1806f9a98d85664d2a545 Mon Sep 17 00:00:00 2001 From: Harwinder Sekhon Date: Mon, 3 Aug 2026 13:29:18 -0400 Subject: [PATCH] Add commit-ladder campaign driver; ignore its 26MB results folder Added: backtest/research/commit_ladder.py (recreates historical configs via --set on the fixed harness; 2026-08-03 campaign confirmed HEAD is the best config on all timeframes). Results stay local like all pinned trade books. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Nsd8HcrV67GbCZXxZEn1Ma --- .gitignore | 1 + backtest/research/commit_ladder.py | 178 +++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 backtest/research/commit_ladder.py diff --git a/.gitignore b/.gitignore index d4dd623..a092033 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ backtest/trailgrid5m/ backtest/__pycache__/ .env.* logs/ +backtest/research/commit_ladder_2026-08/ diff --git a/backtest/research/commit_ladder.py b/backtest/research/commit_ladder.py new file mode 100644 index 0000000..9dc03ef --- /dev/null +++ b/backtest/research/commit_ladder.py @@ -0,0 +1,178 @@ +"""Commit-ladder campaign driver (2026-08). + +Question: of all signal-code states shipped in the last two months, which +configuration is the most accurate? Historical in-era backtest numbers are +not comparable (the backtester's realism model changed on 07-08/07-16 and +the data feed switched to Oanda on 07-28), so every historical +configuration is recreated on TODAY's harness via --set overrides and +replayed on the same pinned Oanda bars. Bug-fix commits stay on; the +ladder compares shipped tuning decisions only. + +Usage: + python backtest/research/commit_ladder.py # full matrix + python backtest/research/commit_ladder.py --tf 5min # one timeframe + python backtest/research/commit_ladder.py --windows 12mo,oos + +Resumable: a run whose CSV already exists in the output dir is skipped. +Outputs: backtest/research/commit_ladder_2026-08/_.csv + + .log per run, summary.csv, and a printed ranking table. +""" +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + +import pandas as pd + +ROOT = Path(__file__).resolve().parents[2] +BT = ROOT / "backtest" / "gpu_backtest.py" +OUT = ROOT / "backtest" / "research" / "commit_ladder_2026-08" + +WINDOWS = { + "12mo": ("2025-07-30", "2026-07-30"), + "oos": ("2024-06-01", "2025-03-30"), + "5yr": ("2021-07-30", "2026-07-30"), +} +SPLIT_12MO = pd.Timestamp("2026-01-29", tz="UTC") # book_diff.py's split + +# (variant, "commit date what-it-represents", [--set overrides]) +VARIANTS: dict[str, list[tuple[str, str, list[str]]]] = { + "5min": [ + ("M5-A", "dad2432 07-09 gates+trailing+pullback", + ["xau_strategy.blocked_hours_utc=[0,1,2,10,11,13,16,21]", + "xau_strategy.cp_enabled=false", + "signals.trail_dist_r=0.75", + "xau_strategy.m5_min_abs_score=30", + "xau_strategy.m5_min_confirmation_score=10"]), + ("M5-B", "f3ba4d5 07-11 +hour-9 block", + ["xau_strategy.blocked_hours_utc=[0,1,2,10,11,13,16,21,9]", + "xau_strategy.cp_enabled=false", + "signals.trail_dist_r=0.75", + "xau_strategy.m5_min_abs_score=30", + "xau_strategy.m5_min_confirmation_score=10"]), + ("M5-C", "de6424e 07-16 +chart patterns", + ["xau_strategy.blocked_hours_utc=[0,1,2,10,11,13,16,21,9]", + "signals.trail_dist_r=0.75", + "xau_strategy.m5_min_abs_score=30", + "xau_strategy.m5_min_confirmation_score=10"]), + ("M5-D", "3ee79b5 07-21 +trail dist 0.85", + ["xau_strategy.blocked_hours_utc=[0,1,2,10,11,13,16,21,9]", + "xau_strategy.m5_min_abs_score=30", + "xau_strategy.m5_min_confirmation_score=10"]), + ("M5-E", "0396970 07-30 relaxation (HEAD)", []), + ], + "15min": [ + ("M15-A", "987ec5b 07-09 pre-campaigns", + ["signals.tp_floor_rr=0.5", + "xau_strategy.blocked_regimes=[]", + "xau_strategy.sell_blocked_hours_utc=[]", + "signals.trail_arm_r=0.60"]), + ("M15-B", "f3ba4d5/f88e39f 07-11 1R floor + regime block", + ["xau_strategy.sell_blocked_hours_utc=[]", + "signals.trail_arm_r=0.60"]), + ("M15-C", "7d3fd95 07-16 +SELL hour 15/17 block", + ["signals.trail_arm_r=0.60"]), + ("M15-D", "bf8e5fb 07-21 trail arm 0.70 (HEAD)", []), + ], + "1h": [ + ("H1-A", "pre-d95d31a 07-08 TP-tightening model", + ["signals.min_rr_ratio=0.4", + "signals.tp_tightening_factor=0.8"]), + ("H1-B", "HEAD", []), + ], +} + + +def run_one(tf: str, name: str, overrides: list[str], wkey: str, + spread: str | None = None) -> Path: + suffix = f"_s{spread}" if spread else "" + csv_path = OUT / f"{name}_{wkey}{suffix}.csv" + log_path = OUT / f"{name}_{wkey}{suffix}.log" + if csv_path.exists(): + print(f" [skip] {csv_path.name} exists") + return csv_path + start, end = WINDOWS[wkey] + cmd = [sys.executable, str(BT), "--pair", "XAUUSD", + "--primary-tf", tf, "--start", start, "--end", end, + "--export-csv", str(csv_path)] + if spread: + cmd += ["--spread", spread] + for ov in overrides: + cmd += ["--set", ov] + print(f" [run ] {csv_path.name} ...", flush=True) + with open(log_path, "w") as lf: + rc = subprocess.run(cmd, stdout=lf, stderr=subprocess.STDOUT, + cwd=ROOT).returncode + if rc != 0: + raise SystemExit(f"backtest failed (rc={rc}) — see {log_path}") + log = log_path.read_text() + n_ov = log.count("--set override") + if n_ov != len(overrides): + raise SystemExit( + f"{name}_{wkey}: expected {len(overrides)} --set overrides, " + f"log shows {n_ov} — see {log_path}") + if "Loading cached" not in log and wkey != "sanity": + print(f" [warn] {log_path.name}: no 'Loading cached' line — " + f"data was fetched live for at least one timeframe") + return csv_path + + +def metrics(csv_path: Path, wkey: str) -> dict: + df = pd.read_csv(csv_path, parse_dates=["entry_time"]) + if df.empty: + return {"trades": 0, "total_r": 0.0, "wr": 0.0, "pf": 0.0, + "max_dd": 0.0, "h1_r": 0.0, "h2_r": 0.0, "years_pos": ""} + if df["entry_time"].dt.tz is None: + df["entry_time"] = df["entry_time"].dt.tz_localize("UTC") + r = df["r_multiple"] + pos, neg = r[r > 0].sum(), r[r < 0].sum() + cum = r.cumsum() + dd = (pd.concat([pd.Series([0.0]), cum]).cummax()[1:] - cum).max() + out = { + "trades": len(df), + "total_r": round(r.sum(), 1), + "wr": round((r > 0).mean() * 100, 1), + "pf": round(pos / abs(neg), 2) if neg else float("inf"), + "max_dd": round(dd, 1), + "h1_r": "", "h2_r": "", "years_pos": "", + } + if wkey == "12mo": + h1 = r[df["entry_time"] < SPLIT_12MO].sum() + out["h1_r"] = round(h1, 1) + out["h2_r"] = round(r.sum() - h1, 1) + if wkey == "5yr": + by_year = r.groupby(df["entry_time"].dt.year).sum() + out["years_pos"] = f"{(by_year > 0).sum()}/{len(by_year)}" + return out + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--tf", choices=list(VARIANTS), default=None) + ap.add_argument("--windows", default="12mo,oos,5yr") + args = ap.parse_args() + OUT.mkdir(parents=True, exist_ok=True) + wkeys = [w.strip() for w in args.windows.split(",")] + tfs = [args.tf] if args.tf else list(VARIANTS) + + rows = [] + for tf in tfs: + for name, label, overrides in VARIANTS[tf]: + for wkey in wkeys: + csv_path = run_one(tf, name, overrides, wkey) + m = metrics(csv_path, wkey) + rows.append({"tf": tf, "variant": name, "represents": label, + "window": wkey, **m}) + print(f" {name} {wkey}: {m['trades']} trades, " + f"{m['total_r']}R, PF {m['pf']}, WR {m['wr']}%") + + summary = pd.DataFrame(rows) + summary.to_csv(OUT / "summary.csv", index=False) + print("\n=== SUMMARY (also in summary.csv) ===") + print(summary.to_string(index=False)) + + +if __name__ == "__main__": + main()