baseline: byte-for-byte backtest reference at da61cff + compare tooling
Pin reference outputs of backtest/gpu_backtest.py (XAU/USD, window 2025-08-04..2026-08-04, 4 primary TFs x spread 0.30/0.65 = 8 runs) under baseline/da61cff/: trade books, stdout summaries, derived equity curves and metrics, plus a manifest with exact argv, input-parquet and output hashes, backend, and tool versions. scripts/capture_baseline.py captures (cache-warm only, aborts rather than fetch live); scripts/compare_baseline.py re-runs the identical commands and byte-diffs every output, PASS/FAIL per file. Verified 32/32 files PASS on full re-run at this commit; m5_s065 reproduces the pinned HEAD reference exactly (555 trades, +98.64R, PF 1.483). No signal, execution, or scanner logic touched; 359 tests green. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01J3mZCQRHnJKt7qXQFG3RhB
This commit is contained in:
Harwinder Sekhon
@@ -0,0 +1,183 @@
|
||||
#!/usr/bin/env python
|
||||
"""Capture the byte-for-byte backtest baseline at the current commit.
|
||||
|
||||
Runs the 8-run matrix (M5/M15/H1/H4 × spread 0.30/0.65, window
|
||||
2025-08-04 → 2026-08-04) through backtest/gpu_backtest.py and pins every
|
||||
output under baseline/<commit_short>/:
|
||||
|
||||
baseline/<commit>/
|
||||
manifest.json commands, input hashes, output hashes, env
|
||||
README.md layout + how to verify
|
||||
<run_key>/
|
||||
trades.csv full trade book (--export-csv)
|
||||
summary.txt backtester stdout (byte-comparable)
|
||||
equity.csv derived cumulative-R curve
|
||||
metrics.json derived headline + grouped stats
|
||||
run.log backtester stderr (provenance only)
|
||||
|
||||
Aborts before running anything if the tracked tree is dirty or any needed
|
||||
cache parquet is missing (a cache miss would fetch live — wrong data, and
|
||||
the local .env Oanda token is stale). Verify afterwards with
|
||||
scripts/compare_baseline.py — it must print all-PASS at the same commit.
|
||||
|
||||
Usage:
|
||||
venv/bin/python scripts/capture_baseline.py [--runs m5_s030,...] [--force]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import baseline_common as bc
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument("--runs", default="",
|
||||
help="comma-separated run keys to capture (default: all)")
|
||||
ap.add_argument("--force", action="store_true",
|
||||
help="overwrite existing run dirs / manifest")
|
||||
args = ap.parse_args()
|
||||
|
||||
git = bc.git_info()
|
||||
if git["dirty"]:
|
||||
print("ABORT: tracked working tree is dirty — a baseline must be "
|
||||
"attributable to exactly one commit. Commit or stash first.")
|
||||
return 2
|
||||
|
||||
specs = bc.run_specs()
|
||||
if args.runs:
|
||||
wanted = [k.strip() for k in args.runs.split(",") if k.strip()]
|
||||
unknown = [k for k in wanted if k not in specs]
|
||||
if unknown:
|
||||
print(f"ABORT: unknown run key(s) {unknown}; "
|
||||
f"valid: {', '.join(specs)}")
|
||||
return 2
|
||||
specs = {k: specs[k] for k in wanted}
|
||||
|
||||
base_dir = bc.BASELINE_ROOT / git["commit_short"]
|
||||
if not args.force:
|
||||
clashes = [k for k in specs if (base_dir / k).exists()]
|
||||
if clashes:
|
||||
print(f"ABORT: {base_dir} already has run(s) {clashes} "
|
||||
"(use --force to overwrite)")
|
||||
return 2
|
||||
|
||||
# Pre-flight: every parquet the runs will read must already exist and
|
||||
# postdate the raw-cache format change. Never trigger a network fetch.
|
||||
problems = []
|
||||
cache_hashes = {}
|
||||
for name in bc.all_expected_cache_files(specs):
|
||||
path = bc.CACHE_DIR / name
|
||||
if not path.exists():
|
||||
problems.append(f"missing cache parquet: {name}")
|
||||
elif path.stat().st_mtime < bc.RAW_CACHE_EPOCH:
|
||||
problems.append(f"pre-raw-format cache parquet (cleaned bars): "
|
||||
f"{name}")
|
||||
else:
|
||||
cache_hashes[name] = bc.sha256_file(path)
|
||||
if problems:
|
||||
print("ABORT: cache pre-flight failed — refusing to run "
|
||||
"(a miss would fetch live data):")
|
||||
for p in problems:
|
||||
print(f" - {p}")
|
||||
return 2
|
||||
print(f"Pre-flight OK: {len(cache_hashes)} cache parquets present "
|
||||
f"and hashed. Capturing {len(specs)} run(s) at {git['commit_short']}.")
|
||||
|
||||
manifest_runs = {}
|
||||
backend = None
|
||||
for i, (key, spec) in enumerate(specs.items(), 1):
|
||||
run_dir = base_dir / key
|
||||
print(f"[{i}/{len(specs)}] {key}: {spec['primary_tf']} "
|
||||
f"spread {spec['spread']} ...", flush=True)
|
||||
bc.run_backtest(spec["argv"], run_dir)
|
||||
log_problems, run_backend = bc.check_run_log(run_dir,
|
||||
spec["primary_tf"])
|
||||
if log_problems:
|
||||
print(f"ABORT: {key} failed stderr sanity checks:")
|
||||
for p in log_problems:
|
||||
print(f" - {p}")
|
||||
print(f" log: {run_dir / 'run.log'}")
|
||||
return 2
|
||||
if backend is None:
|
||||
backend = run_backend
|
||||
elif run_backend != backend:
|
||||
print(f"ABORT: backend flipped mid-capture "
|
||||
f"({backend!r} → {run_backend!r})")
|
||||
return 2
|
||||
bc.derive_equity(run_dir / "trades.csv", run_dir / "equity.csv")
|
||||
metrics = bc.derive_metrics(run_dir / "trades.csv",
|
||||
run_dir / "metrics.json")
|
||||
manifest_runs[key] = {
|
||||
"argv": spec["argv"],
|
||||
"primary_tf": spec["primary_tf"],
|
||||
"cache_files": bc.expected_cache_files(spec["primary_tf"]),
|
||||
"files": {f: bc.sha256_file(run_dir / f)
|
||||
for f in bc.COMPARED_FILES},
|
||||
}
|
||||
o = metrics["overall"]
|
||||
print(f" {o['trades']} trades, {o['total_r']:+.2f}R, "
|
||||
f"PF {o['profit_factor']}, maxDD {o['max_dd_r']:.2f}R")
|
||||
|
||||
manifest = {
|
||||
"created_utc": bc.utc_now_iso(),
|
||||
"commit": git["commit"],
|
||||
"commit_short": git["commit_short"],
|
||||
"dirty": git["dirty"],
|
||||
"window": {"start": bc.WINDOW_START, "end": bc.WINDOW_END},
|
||||
"backend": backend,
|
||||
"env_overrides": {"CUDA_PATH": "removed", "PYTHONHASHSEED": "0"},
|
||||
"versions": bc.tool_versions(),
|
||||
"config_inputs": bc.config_inputs(),
|
||||
"cache_files": cache_hashes,
|
||||
"compared_files": list(bc.COMPARED_FILES),
|
||||
"runs": manifest_runs,
|
||||
}
|
||||
manifest_path = base_dir / "manifest.json"
|
||||
if manifest_path.exists():
|
||||
# Partial re-capture with --runs: merge into the existing manifest
|
||||
# (only the just-captured keys are replaced; same commit is
|
||||
# guaranteed by the dir name).
|
||||
old = json.loads(manifest_path.read_text())
|
||||
old["runs"].update(manifest_runs)
|
||||
old["cache_files"].update(cache_hashes)
|
||||
old["created_utc"] = manifest["created_utc"]
|
||||
manifest = old
|
||||
manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True)
|
||||
+ "\n")
|
||||
_write_readme(base_dir)
|
||||
print(f"\nBaseline captured: {base_dir}")
|
||||
print("Verify now: venv/bin/python scripts/compare_baseline.py "
|
||||
"(must be all-PASS at this commit)")
|
||||
return 0
|
||||
|
||||
|
||||
def _write_readme(base_dir: Path) -> None:
|
||||
(base_dir / "README.md").write_text(f"""\
|
||||
# Backtest baseline — commit {base_dir.name}
|
||||
|
||||
Reference outputs of `backtest/gpu_backtest.py` for XAU/USD, window
|
||||
{bc.WINDOW_START} → {bc.WINDOW_END}, 4 primary TFs × spreads 0.30/0.65.
|
||||
Captured by `scripts/capture_baseline.py`; verify with
|
||||
`venv/bin/python scripts/compare_baseline.py` (re-runs every command and
|
||||
byte-diffs the outputs, PASS/FAIL per file).
|
||||
|
||||
Per run dir: `trades.csv` (full book), `summary.txt` (backtester stdout),
|
||||
`equity.csv` + `metrics.json` (derived from the book). Those four are
|
||||
byte-compared. `run.log` is the backtester's stderr — timestamped, kept
|
||||
for provenance only, never compared.
|
||||
|
||||
`manifest.json` records the exact argv per run, sha256 of every input
|
||||
parquet and output file, backend, tool versions, and the non-secret
|
||||
config inputs that alter results. Input data must come from
|
||||
`backtest/cache/` — if those parquets change, comparisons are no longer
|
||||
attributable to code changes and the compare script aborts.
|
||||
""")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user