ForexBot2/scripts/compare_baseline.py

#!/usr/bin/env python
"""Re-run the baseline backtests and byte-diff every output against baseline/.

For each captured run, replays the exact argv recorded in the baseline's
manifest.json (same env pinning: no CUDA_PATH, PYTHONHASHSEED=0), rebuilds
the derived files, and compares trades.csv / summary.txt / equity.csv /
metrics.json byte-for-byte (sha256), printing PASS/FAIL per file.
run.log (stderr) is provenance-only and never compared.

All-PASS at the capture commit proves determinism; a FAIL after a code
change means the change altered backtest behavior — which is exactly what
this script exists to detect. Exit code 0 only if every file PASSes.

Pre-flight aborts (before any run) if an input cache parquet is missing or
its hash changed: with different input data, diffs are not attributable to
code. Live fetches are treated as failures for the same reason.

Usage:
    venv/bin/python scripts/compare_baseline.py
        [--baseline baseline/<commit>] [--runs m5_s030,...] [--keep]
"""
from __future__ import annotations

import argparse
import json
import shutil
import sys
import tempfile
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))
import baseline_common as bc


def newest_baseline() -> Path | None:
    if not bc.BASELINE_ROOT.is_dir():
        return None
    dirs = [d for d in bc.BASELINE_ROOT.iterdir()
            if (d / "manifest.json").exists()]
    return max(dirs, key=lambda d: (d / "manifest.json").stat().st_mtime,
               default=None)


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    ap.add_argument("--baseline", default=None,
                    help="baseline dir (default: newest under baseline/)")
    ap.add_argument("--runs", default="",
                    help="comma-separated run keys to compare (default: all)")
    ap.add_argument("--keep", action="store_true",
                    help="keep re-run outputs even on all-PASS")
    args = ap.parse_args()

    base_dir = Path(args.baseline) if args.baseline else newest_baseline()
    if not base_dir or not (base_dir / "manifest.json").exists():
        print("ABORT: no baseline found — run scripts/capture_baseline.py "
              "first (or pass --baseline).")
        return 2
    manifest = json.loads((base_dir / "manifest.json").read_text())

    runs = manifest["runs"]
    if args.runs:
        wanted = [k.strip() for k in args.runs.split(",") if k.strip()]
        unknown = [k for k in wanted if k not in runs]
        if unknown:
            print(f"ABORT: unknown run key(s) {unknown}; "
                  f"baseline has: {', '.join(runs)}")
            return 2
        runs = {k: runs[k] for k in wanted}

    git = bc.git_info()
    print(f"Baseline: {base_dir}  (captured at {manifest['commit_short']}"
          f"{', DIRTY' if manifest.get('dirty') else ''})")
    print(f"Current HEAD: {git['commit_short']}"
          f"{' (dirty)' if git['dirty'] else ''}"
          + ("  — same commit, expecting all-PASS"
             if git["commit"] == manifest["commit"]
             else "  — differs from baseline; FAILs would mean changed "
                  "backtest behavior"))

    # ── Pre-flight: identical input data, or nothing is attributable ──
    needed = sorted({f for r in runs.values() for f in r["cache_files"]})
    data_problems = []
    for name in needed:
        path = bc.CACHE_DIR / name
        if not path.exists():
            data_problems.append(f"missing cache parquet: {name}")
        elif bc.sha256_file(path) != manifest["cache_files"].get(name):
            data_problems.append(f"cache parquet CHANGED since capture: "
                                 f"{name}")
    if data_problems:
        print("ABORT: input data drift — diffs would not be attributable "
              "to code:")
        for p in data_problems:
            print(f"  - {p}")
        return 2

    cfg_now = bc.config_inputs()
    if cfg_now != manifest.get("config_inputs"):
        print("WARNING: non-secret config.json inputs differ from capture "
              "time (data_providers.active / pairs overlay) — these alter "
              "results:")
        print(f"  baseline: {manifest.get('config_inputs')}")
        print(f"  now:      {cfg_now}")

    work_dir = Path(tempfile.mkdtemp(prefix="baseline_compare_"))
    print(f"Re-running {len(runs)} run(s) into {work_dir}\n")

    n_pass = n_fail = 0
    failed_runs = []
    for i, (key, spec) in enumerate(runs.items(), 1):
        run_dir = work_dir / key
        print(f"[{i}/{len(runs)}] {key}")
        try:
            bc.run_backtest(spec["argv"], run_dir)
            problems, backend = bc.check_run_log(run_dir, spec["primary_tf"])
            if backend and backend != manifest.get("backend"):
                problems.append(f"backend {backend!r} != baseline "
                                f"{manifest.get('backend')!r}")
            if problems:
                raise RuntimeError("; ".join(problems))
            bc.derive_equity(run_dir / "trades.csv", run_dir / "equity.csv")
            bc.derive_metrics(run_dir / "trades.csv",
                              run_dir / "metrics.json")
        except Exception as exc:
            print(f"  RUN FAILED: {exc}")
            for fname in bc.COMPARED_FILES:
                print(f"  {fname:<13} FAIL (run failed)")
                n_fail += 1
            failed_runs.append(key)
            continue

        run_ok = True
        for fname in bc.COMPARED_FILES:
            new = run_dir / fname
            ref = base_dir / key / fname
            same = (new.exists() and ref.exists()
                    and bc.sha256_file(new) == spec["files"][fname])
            print(f"  {fname:<13} {'PASS' if same else 'FAIL'}")
            if same:
                n_pass += 1
            else:
                n_fail += 1
                run_ok = False
                if new.exists() and ref.exists():
                    print(f"    baseline: {ref} ({ref.stat().st_size} B)")
                    print(f"    re-run:   {new} ({new.stat().st_size} B)")
        if not run_ok:
            failed_runs.append(key)

    total = n_pass + n_fail
    print(f"\nRESULT: {n_pass}/{total} files PASS across {len(runs)} run(s) "
          f"vs baseline {manifest['commit_short']}")
    if n_fail:
        print(f"FAILED runs: {', '.join(dict.fromkeys(failed_runs))}")
        print(f"Re-run outputs kept for inspection: {work_dir}")
        return 1
    if args.keep:
        print(f"Re-run outputs kept (--keep): {work_dir}")
    else:
        shutil.rmtree(work_dir, ignore_errors=True)
    return 0


if __name__ == "__main__":
    sys.exit(main())