ForexBot2/scripts/capture_baseline.py

#!/usr/bin/env python
"""Capture the byte-for-byte backtest baseline at the current commit.

Runs the 8-run matrix (M5/M15/H1/H4 × spread 0.30/0.65, window
2025-08-04 → 2026-08-04) through backtest/gpu_backtest.py and pins every
output under baseline/<commit_short>/:

    baseline/<commit>/
        manifest.json          commands, input hashes, output hashes, env
        README.md              layout + how to verify
        <run_key>/
            trades.csv         full trade book (--export-csv)
            summary.txt        backtester stdout (byte-comparable)
            equity.csv         derived cumulative-R curve
            metrics.json       derived headline + grouped stats
            run.log            backtester stderr (provenance only)

Aborts before running anything if the tracked tree is dirty or any needed
cache parquet is missing (a cache miss would fetch live — wrong data, and
the local .env Oanda token is stale). Verify afterwards with
scripts/compare_baseline.py — it must print all-PASS at the same commit.

Usage:
    venv/bin/python scripts/capture_baseline.py [--runs m5_s030,...] [--force]
"""
from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))
import baseline_common as bc


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    ap.add_argument("--runs", default="",
                    help="comma-separated run keys to capture (default: all)")
    ap.add_argument("--force", action="store_true",
                    help="overwrite existing run dirs / manifest")
    args = ap.parse_args()

    git = bc.git_info()
    if git["dirty"]:
        print("ABORT: tracked working tree is dirty — a baseline must be "
              "attributable to exactly one commit. Commit or stash first.")
        return 2

    specs = bc.run_specs()
    if args.runs:
        wanted = [k.strip() for k in args.runs.split(",") if k.strip()]
        unknown = [k for k in wanted if k not in specs]
        if unknown:
            print(f"ABORT: unknown run key(s) {unknown}; "
                  f"valid: {', '.join(specs)}")
            return 2
        specs = {k: specs[k] for k in wanted}

    base_dir = bc.BASELINE_ROOT / git["commit_short"]
    if not args.force:
        clashes = [k for k in specs if (base_dir / k).exists()]
        if clashes:
            print(f"ABORT: {base_dir} already has run(s) {clashes} "
                  "(use --force to overwrite)")
            return 2

    # Pre-flight: every parquet the runs will read must already exist and
    # postdate the raw-cache format change. Never trigger a network fetch.
    problems = []
    cache_hashes = {}
    for name in bc.all_expected_cache_files(specs):
        path = bc.CACHE_DIR / name
        if not path.exists():
            problems.append(f"missing cache parquet: {name}")
        elif path.stat().st_mtime < bc.RAW_CACHE_EPOCH:
            problems.append(f"pre-raw-format cache parquet (cleaned bars): "
                            f"{name}")
        else:
            cache_hashes[name] = bc.sha256_file(path)
    if problems:
        print("ABORT: cache pre-flight failed — refusing to run "
              "(a miss would fetch live data):")
        for p in problems:
            print(f"  - {p}")
        return 2
    print(f"Pre-flight OK: {len(cache_hashes)} cache parquets present "
          f"and hashed. Capturing {len(specs)} run(s) at {git['commit_short']}.")

    manifest_runs = {}
    backend = None
    for i, (key, spec) in enumerate(specs.items(), 1):
        run_dir = base_dir / key
        print(f"[{i}/{len(specs)}] {key}: {spec['primary_tf']} "
              f"spread {spec['spread']} ...", flush=True)
        bc.run_backtest(spec["argv"], run_dir)
        log_problems, run_backend = bc.check_run_log(run_dir,
                                                     spec["primary_tf"])
        if log_problems:
            print(f"ABORT: {key} failed stderr sanity checks:")
            for p in log_problems:
                print(f"  - {p}")
            print(f"  log: {run_dir / 'run.log'}")
            return 2
        if backend is None:
            backend = run_backend
        elif run_backend != backend:
            print(f"ABORT: backend flipped mid-capture "
                  f"({backend!r} → {run_backend!r})")
            return 2
        bc.derive_equity(run_dir / "trades.csv", run_dir / "equity.csv")
        metrics = bc.derive_metrics(run_dir / "trades.csv",
                                    run_dir / "metrics.json")
        manifest_runs[key] = {
            "argv": spec["argv"],
            "primary_tf": spec["primary_tf"],
            "cache_files": bc.expected_cache_files(spec["primary_tf"]),
            "files": {f: bc.sha256_file(run_dir / f)
                      for f in bc.COMPARED_FILES},
        }
        o = metrics["overall"]
        print(f"    {o['trades']} trades, {o['total_r']:+.2f}R, "
              f"PF {o['profit_factor']}, maxDD {o['max_dd_r']:.2f}R")

    manifest = {
        "created_utc": bc.utc_now_iso(),
        "commit": git["commit"],
        "commit_short": git["commit_short"],
        "dirty": git["dirty"],
        "window": {"start": bc.WINDOW_START, "end": bc.WINDOW_END},
        "backend": backend,
        "env_overrides": {"CUDA_PATH": "removed", "PYTHONHASHSEED": "0"},
        "versions": bc.tool_versions(),
        "config_inputs": bc.config_inputs(),
        "cache_files": cache_hashes,
        "compared_files": list(bc.COMPARED_FILES),
        "runs": manifest_runs,
    }
    manifest_path = base_dir / "manifest.json"
    if manifest_path.exists():
        # Partial re-capture with --runs: merge into the existing manifest
        # (only the just-captured keys are replaced; same commit is
        # guaranteed by the dir name).
        old = json.loads(manifest_path.read_text())
        old["runs"].update(manifest_runs)
        old["cache_files"].update(cache_hashes)
        old["created_utc"] = manifest["created_utc"]
        manifest = old
    manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True)
                             + "\n")
    _write_readme(base_dir)
    print(f"\nBaseline captured: {base_dir}")
    print("Verify now: venv/bin/python scripts/compare_baseline.py "
          "(must be all-PASS at this commit)")
    return 0


def _write_readme(base_dir: Path) -> None:
    (base_dir / "README.md").write_text(f"""\
# Backtest baseline — commit {base_dir.name}

Reference outputs of `backtest/gpu_backtest.py` for XAU/USD, window
{bc.WINDOW_START} → {bc.WINDOW_END}, 4 primary TFs × spreads 0.30/0.65.
Captured by `scripts/capture_baseline.py`; verify with
`venv/bin/python scripts/compare_baseline.py` (re-runs every command and
byte-diffs the outputs, PASS/FAIL per file).

Per run dir: `trades.csv` (full book), `summary.txt` (backtester stdout),
`equity.csv` + `metrics.json` (derived from the book). Those four are
byte-compared. `run.log` is the backtester's stderr — timestamped, kept
for provenance only, never compared.

`manifest.json` records the exact argv per run, sha256 of every input
parquet and output file, backend, tool versions, and the non-secret
config inputs that alter results. Input data must come from
`backtest/cache/` — if those parquets change, comparisons are no longer
attributable to code changes and the compare script aborts.
""")


if __name__ == "__main__":
    sys.exit(main())