ForexBot2/backtest/research/commit_ladder.py

"""Commit-ladder campaign driver (2026-08).

Question: of all signal-code states shipped in the last two months, which
configuration is the most accurate? Historical in-era backtest numbers are
not comparable (the backtester's realism model changed on 07-08/07-16 and
the data feed switched to Oanda on 07-28), so every historical
configuration is recreated on TODAY's harness via --set overrides and
replayed on the same pinned Oanda bars. Bug-fix commits stay on; the
ladder compares shipped tuning decisions only.

Usage:
    python backtest/research/commit_ladder.py            # full matrix
    python backtest/research/commit_ladder.py --tf 5min  # one timeframe
    python backtest/research/commit_ladder.py --windows 12mo,oos

Resumable: a run whose CSV already exists in the output dir is skipped.
Outputs: backtest/research/commit_ladder_2026-08/<variant>_<window>.csv
         + .log per run, summary.csv, and a printed ranking table.
"""
from __future__ import annotations

import argparse
import subprocess
import sys
from pathlib import Path

import pandas as pd

ROOT = Path(__file__).resolve().parents[2]
BT = ROOT / "backtest" / "gpu_backtest.py"
OUT = ROOT / "backtest" / "research" / "commit_ladder_2026-08"

WINDOWS = {
    "12mo": ("2025-07-30", "2026-07-30"),
    "oos": ("2024-06-01", "2025-03-30"),
    "5yr": ("2021-07-30", "2026-07-30"),
}
SPLIT_12MO = pd.Timestamp("2026-01-29", tz="UTC")  # book_diff.py's split

# (variant, "commit date what-it-represents", [--set overrides])
VARIANTS: dict[str, list[tuple[str, str, list[str]]]] = {
    "5min": [
        ("M5-A", "dad2432 07-09 gates+trailing+pullback",
         ["xau_strategy.blocked_hours_utc=[0,1,2,10,11,13,16,21]",
          "xau_strategy.cp_enabled=false",
          "signals.trail_dist_r=0.75",
          "xau_strategy.m5_min_abs_score=30",
          "xau_strategy.m5_min_confirmation_score=10"]),
        ("M5-B", "f3ba4d5 07-11 +hour-9 block",
         ["xau_strategy.blocked_hours_utc=[0,1,2,10,11,13,16,21,9]",
          "xau_strategy.cp_enabled=false",
          "signals.trail_dist_r=0.75",
          "xau_strategy.m5_min_abs_score=30",
          "xau_strategy.m5_min_confirmation_score=10"]),
        ("M5-C", "de6424e 07-16 +chart patterns",
         ["xau_strategy.blocked_hours_utc=[0,1,2,10,11,13,16,21,9]",
          "signals.trail_dist_r=0.75",
          "xau_strategy.m5_min_abs_score=30",
          "xau_strategy.m5_min_confirmation_score=10"]),
        ("M5-D", "3ee79b5 07-21 +trail dist 0.85",
         ["xau_strategy.blocked_hours_utc=[0,1,2,10,11,13,16,21,9]",
          "xau_strategy.m5_min_abs_score=30",
          "xau_strategy.m5_min_confirmation_score=10"]),
        ("M5-E", "0396970 07-30 relaxation (HEAD)", []),
    ],
    "15min": [
        ("M15-A", "987ec5b 07-09 pre-campaigns",
         ["signals.tp_floor_rr=0.5",
          "xau_strategy.blocked_regimes=[]",
          "xau_strategy.sell_blocked_hours_utc=[]",
          "signals.trail_arm_r=0.60"]),
        ("M15-B", "f3ba4d5/f88e39f 07-11 1R floor + regime block",
         ["xau_strategy.sell_blocked_hours_utc=[]",
          "signals.trail_arm_r=0.60"]),
        ("M15-C", "7d3fd95 07-16 +SELL hour 15/17 block",
         ["signals.trail_arm_r=0.60"]),
        ("M15-D", "bf8e5fb 07-21 trail arm 0.70 (HEAD)", []),
    ],
    "1h": [
        ("H1-A", "pre-d95d31a 07-08 TP-tightening model",
         ["signals.min_rr_ratio=0.4",
          "signals.tp_tightening_factor=0.8"]),
        ("H1-B", "HEAD", []),
    ],
}


def run_one(tf: str, name: str, overrides: list[str], wkey: str,
            spread: str | None = None) -> Path:
    suffix = f"_s{spread}" if spread else ""
    csv_path = OUT / f"{name}_{wkey}{suffix}.csv"
    log_path = OUT / f"{name}_{wkey}{suffix}.log"
    if csv_path.exists():
        print(f"  [skip] {csv_path.name} exists")
        return csv_path
    start, end = WINDOWS[wkey]
    cmd = [sys.executable, str(BT), "--pair", "XAUUSD",
           "--primary-tf", tf, "--start", start, "--end", end,
           "--export-csv", str(csv_path)]
    if spread:
        cmd += ["--spread", spread]
    for ov in overrides:
        cmd += ["--set", ov]
    print(f"  [run ] {csv_path.name} ...", flush=True)
    with open(log_path, "w") as lf:
        rc = subprocess.run(cmd, stdout=lf, stderr=subprocess.STDOUT,
                            cwd=ROOT).returncode
    if rc != 0:
        raise SystemExit(f"backtest failed (rc={rc}) — see {log_path}")
    log = log_path.read_text()
    n_ov = log.count("--set override")
    if n_ov != len(overrides):
        raise SystemExit(
            f"{name}_{wkey}: expected {len(overrides)} --set overrides, "
            f"log shows {n_ov} — see {log_path}")
    if "Loading cached" not in log and wkey != "sanity":
        print(f"  [warn] {log_path.name}: no 'Loading cached' line — "
              f"data was fetched live for at least one timeframe")
    return csv_path


def metrics(csv_path: Path, wkey: str) -> dict:
    df = pd.read_csv(csv_path, parse_dates=["entry_time"])
    if df.empty:
        return {"trades": 0, "total_r": 0.0, "wr": 0.0, "pf": 0.0,
                "max_dd": 0.0, "h1_r": 0.0, "h2_r": 0.0, "years_pos": ""}
    if df["entry_time"].dt.tz is None:
        df["entry_time"] = df["entry_time"].dt.tz_localize("UTC")
    r = df["r_multiple"]
    pos, neg = r[r > 0].sum(), r[r < 0].sum()
    cum = r.cumsum()
    dd = (pd.concat([pd.Series([0.0]), cum]).cummax()[1:] - cum).max()
    out = {
        "trades": len(df),
        "total_r": round(r.sum(), 1),
        "wr": round((r > 0).mean() * 100, 1),
        "pf": round(pos / abs(neg), 2) if neg else float("inf"),
        "max_dd": round(dd, 1),
        "h1_r": "", "h2_r": "", "years_pos": "",
    }
    if wkey == "12mo":
        h1 = r[df["entry_time"] < SPLIT_12MO].sum()
        out["h1_r"] = round(h1, 1)
        out["h2_r"] = round(r.sum() - h1, 1)
    if wkey == "5yr":
        by_year = r.groupby(df["entry_time"].dt.year).sum()
        out["years_pos"] = f"{(by_year > 0).sum()}/{len(by_year)}"
    return out


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--tf", choices=list(VARIANTS), default=None)
    ap.add_argument("--windows", default="12mo,oos,5yr")
    args = ap.parse_args()
    OUT.mkdir(parents=True, exist_ok=True)
    wkeys = [w.strip() for w in args.windows.split(",")]
    tfs = [args.tf] if args.tf else list(VARIANTS)

    rows = []
    for tf in tfs:
        for name, label, overrides in VARIANTS[tf]:
            for wkey in wkeys:
                csv_path = run_one(tf, name, overrides, wkey)
                m = metrics(csv_path, wkey)
                rows.append({"tf": tf, "variant": name, "represents": label,
                             "window": wkey, **m})
                print(f"    {name} {wkey}: {m['trades']} trades, "
                      f"{m['total_r']}R, PF {m['pf']}, WR {m['wr']}%")

    summary = pd.DataFrame(rows)
    summary.to_csv(OUT / "summary.csv", index=False)
    print("\n=== SUMMARY (also in summary.csv) ===")
    print(summary.to_string(index=False))


if __name__ == "__main__":
    main()