Add commit-ladder campaign driver; ignore its 26MB results folder

Added: backtest/research/commit_ladder.py (recreates historical configs via --set on the fixed harness; 2026-08-03 campaign confirmed HEAD is the best config on all timeframes). Results stay local like all pinned trade books. Claude-Session: https://claude.ai/code/session_01Nsd8HcrV67GbCZXxZEn1Ma
2026-08-03 13:29:18 -04:00
parent 6fff06ff90
commit 9e1c321f05
2 changed files with 179 additions and 0 deletions
@@ -0,0 +1,178 @@
+"""Commit-ladder campaign driver (2026-08).
+
+Question: of all signal-code states shipped in the last two months, which
+configuration is the most accurate? Historical in-era backtest numbers are
+not comparable (the backtester's realism model changed on 07-08/07-16 and
+the data feed switched to Oanda on 07-28), so every historical
+configuration is recreated on TODAY's harness via --set overrides and
+replayed on the same pinned Oanda bars. Bug-fix commits stay on; the
+ladder compares shipped tuning decisions only.
+
+Usage:
+    python backtest/research/commit_ladder.py            # full matrix
+    python backtest/research/commit_ladder.py --tf 5min  # one timeframe
+    python backtest/research/commit_ladder.py --windows 12mo,oos
+
+Resumable: a run whose CSV already exists in the output dir is skipped.
+Outputs: backtest/research/commit_ladder_2026-08/<variant>_<window>.csv
+         + .log per run, summary.csv, and a printed ranking table.
+"""
+from __future__ import annotations
+
+import argparse
+import subprocess
+import sys
+from pathlib import Path
+
+import pandas as pd
+
+ROOT = Path(__file__).resolve().parents[2]
+BT = ROOT / "backtest" / "gpu_backtest.py"
+OUT = ROOT / "backtest" / "research" / "commit_ladder_2026-08"
+
+WINDOWS = {
+    "12mo": ("2025-07-30", "2026-07-30"),
+    "oos": ("2024-06-01", "2025-03-30"),
+    "5yr": ("2021-07-30", "2026-07-30"),
+}
+SPLIT_12MO = pd.Timestamp("2026-01-29", tz="UTC")  # book_diff.py's split
+
+# (variant, "commit date what-it-represents", [--set overrides])
+VARIANTS: dict[str, list[tuple[str, str, list[str]]]] = {
+    "5min": [
+        ("M5-A", "dad2432 07-09 gates+trailing+pullback",
+         ["xau_strategy.blocked_hours_utc=[0,1,2,10,11,13,16,21]",
+          "xau_strategy.cp_enabled=false",
+          "signals.trail_dist_r=0.75",
+          "xau_strategy.m5_min_abs_score=30",
+          "xau_strategy.m5_min_confirmation_score=10"]),
+        ("M5-B", "f3ba4d5 07-11 +hour-9 block",
+         ["xau_strategy.blocked_hours_utc=[0,1,2,10,11,13,16,21,9]",
+          "xau_strategy.cp_enabled=false",
+          "signals.trail_dist_r=0.75",
+          "xau_strategy.m5_min_abs_score=30",
+          "xau_strategy.m5_min_confirmation_score=10"]),
+        ("M5-C", "de6424e 07-16 +chart patterns",
+         ["xau_strategy.blocked_hours_utc=[0,1,2,10,11,13,16,21,9]",
+          "signals.trail_dist_r=0.75",
+          "xau_strategy.m5_min_abs_score=30",
+          "xau_strategy.m5_min_confirmation_score=10"]),
+        ("M5-D", "3ee79b5 07-21 +trail dist 0.85",
+         ["xau_strategy.blocked_hours_utc=[0,1,2,10,11,13,16,21,9]",
+          "xau_strategy.m5_min_abs_score=30",
+          "xau_strategy.m5_min_confirmation_score=10"]),
+        ("M5-E", "0396970 07-30 relaxation (HEAD)", []),
+    ],
+    "15min": [
+        ("M15-A", "987ec5b 07-09 pre-campaigns",
+         ["signals.tp_floor_rr=0.5",
+          "xau_strategy.blocked_regimes=[]",
+          "xau_strategy.sell_blocked_hours_utc=[]",
+          "signals.trail_arm_r=0.60"]),
+        ("M15-B", "f3ba4d5/f88e39f 07-11 1R floor + regime block",
+         ["xau_strategy.sell_blocked_hours_utc=[]",
+          "signals.trail_arm_r=0.60"]),
+        ("M15-C", "7d3fd95 07-16 +SELL hour 15/17 block",
+         ["signals.trail_arm_r=0.60"]),
+        ("M15-D", "bf8e5fb 07-21 trail arm 0.70 (HEAD)", []),
+    ],
+    "1h": [
+        ("H1-A", "pre-d95d31a 07-08 TP-tightening model",
+         ["signals.min_rr_ratio=0.4",
+          "signals.tp_tightening_factor=0.8"]),
+        ("H1-B", "HEAD", []),
+    ],
+}
+
+
+def run_one(tf: str, name: str, overrides: list[str], wkey: str,
+            spread: str | None = None) -> Path:
+    suffix = f"_s{spread}" if spread else ""
+    csv_path = OUT / f"{name}_{wkey}{suffix}.csv"
+    log_path = OUT / f"{name}_{wkey}{suffix}.log"
+    if csv_path.exists():
+        print(f"  [skip] {csv_path.name} exists")
+        return csv_path
+    start, end = WINDOWS[wkey]
+    cmd = [sys.executable, str(BT), "--pair", "XAUUSD",
+           "--primary-tf", tf, "--start", start, "--end", end,
+           "--export-csv", str(csv_path)]
+    if spread:
+        cmd += ["--spread", spread]
+    for ov in overrides:
+        cmd += ["--set", ov]
+    print(f"  [run ] {csv_path.name} ...", flush=True)
+    with open(log_path, "w") as lf:
+        rc = subprocess.run(cmd, stdout=lf, stderr=subprocess.STDOUT,
+                            cwd=ROOT).returncode
+    if rc != 0:
+        raise SystemExit(f"backtest failed (rc={rc}) — see {log_path}")
+    log = log_path.read_text()
+    n_ov = log.count("--set override")
+    if n_ov != len(overrides):
+        raise SystemExit(
+            f"{name}_{wkey}: expected {len(overrides)} --set overrides, "
+            f"log shows {n_ov} — see {log_path}")
+    if "Loading cached" not in log and wkey != "sanity":
+        print(f"  [warn] {log_path.name}: no 'Loading cached' line — "
+              f"data was fetched live for at least one timeframe")
+    return csv_path
+
+
+def metrics(csv_path: Path, wkey: str) -> dict:
+    df = pd.read_csv(csv_path, parse_dates=["entry_time"])
+    if df.empty:
+        return {"trades": 0, "total_r": 0.0, "wr": 0.0, "pf": 0.0,
+                "max_dd": 0.0, "h1_r": 0.0, "h2_r": 0.0, "years_pos": ""}
+    if df["entry_time"].dt.tz is None:
+        df["entry_time"] = df["entry_time"].dt.tz_localize("UTC")
+    r = df["r_multiple"]
+    pos, neg = r[r > 0].sum(), r[r < 0].sum()
+    cum = r.cumsum()
+    dd = (pd.concat([pd.Series([0.0]), cum]).cummax()[1:] - cum).max()
+    out = {
+        "trades": len(df),
+        "total_r": round(r.sum(), 1),
+        "wr": round((r > 0).mean() * 100, 1),
+        "pf": round(pos / abs(neg), 2) if neg else float("inf"),
+        "max_dd": round(dd, 1),
+        "h1_r": "", "h2_r": "", "years_pos": "",
+    }
+    if wkey == "12mo":
+        h1 = r[df["entry_time"] < SPLIT_12MO].sum()
+        out["h1_r"] = round(h1, 1)
+        out["h2_r"] = round(r.sum() - h1, 1)
+    if wkey == "5yr":
+        by_year = r.groupby(df["entry_time"].dt.year).sum()
+        out["years_pos"] = f"{(by_year > 0).sum()}/{len(by_year)}"
+    return out
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--tf", choices=list(VARIANTS), default=None)
+    ap.add_argument("--windows", default="12mo,oos,5yr")
+    args = ap.parse_args()
+    OUT.mkdir(parents=True, exist_ok=True)
+    wkeys = [w.strip() for w in args.windows.split(",")]
+    tfs = [args.tf] if args.tf else list(VARIANTS)
+
+    rows = []
+    for tf in tfs:
+        for name, label, overrides in VARIANTS[tf]:
+            for wkey in wkeys:
+                csv_path = run_one(tf, name, overrides, wkey)
+                m = metrics(csv_path, wkey)
+                rows.append({"tf": tf, "variant": name, "represents": label,
+                             "window": wkey, **m})
+                print(f"    {name} {wkey}: {m['trades']} trades, "
+                      f"{m['total_r']}R, PF {m['pf']}, WR {m['wr']}%")
+
+    summary = pd.DataFrame(rows)
+    summary.to_csv(OUT / "summary.csv", index=False)
+    print("\n=== SUMMARY (also in summary.csv) ===")
+    print(summary.to_string(index=False))
+
+
+if __name__ == "__main__":
+    main()