baseline: byte-for-byte backtest reference at da61cff + compare tooling
Pin reference outputs of backtest/gpu_backtest.py (XAU/USD, window 2025-08-04..2026-08-04, 4 primary TFs x spread 0.30/0.65 = 8 runs) under baseline/da61cff/: trade books, stdout summaries, derived equity curves and metrics, plus a manifest with exact argv, input-parquet and output hashes, backend, and tool versions. scripts/capture_baseline.py captures (cache-warm only, aborts rather than fetch live); scripts/compare_baseline.py re-runs the identical commands and byte-diffs every output, PASS/FAIL per file. Verified 32/32 files PASS on full re-run at this commit; m5_s065 reproduces the pinned HEAD reference exactly (555 trades, +98.64R, PF 1.483). No signal, execution, or scanner logic touched; 359 tests green. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01J3mZCQRHnJKt7qXQFG3RhB
This commit is contained in:
Harwinder Sekhon
@@ -0,0 +1,307 @@
|
||||
"""Shared machinery for the backtest baseline capture/compare pair.
|
||||
|
||||
`capture_baseline.py` and `compare_baseline.py` both import everything
|
||||
here so the two sides produce their outputs from literally the same code:
|
||||
the run-spec table, the subprocess runner, the equity/metrics derivations,
|
||||
and the cache/stderr sanity checks.
|
||||
|
||||
Reproducibility invariants encoded here (see baseline/<commit>/README.md):
|
||||
* --start/--end always pinned — `--days` bakes wall-clock time-of-day
|
||||
into the first replay bar (gpu_backtest.py resolves `end` with
|
||||
datetime.now when --end is absent).
|
||||
* --warmup-bars pinned at 250 — it feeds pad_days and therefore the
|
||||
parquet cache FILENAME.
|
||||
* Subprocess env drops CUDA_PATH (pins the NumPy CPU backend; setting
|
||||
it would silently flip _simulate_trailing onto the CuPy path) and
|
||||
sets PYTHONHASHSEED=0 (removes str-hash set-ordering as a variable).
|
||||
* Data must come from backtest/cache/ — a cache miss would hit the
|
||||
Oanda API (and the stale .env token 401s), and freshly fetched bars
|
||||
are not the baseline's bars. Any "Fetching" line in stderr fails the
|
||||
run.
|
||||
* stdout (summary.txt) is byte-comparable; stderr (run.log) carries
|
||||
%(asctime)s timestamps and elapsed times and is provenance-only.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
import pandas as pd
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
VENV_PYTHON = ROOT / "venv" / "bin" / "python"
|
||||
BACKTESTER_REL = "backtest/gpu_backtest.py"
|
||||
CACHE_DIR = ROOT / "backtest" / "cache"
|
||||
BASELINE_ROOT = ROOT / "baseline"
|
||||
|
||||
WINDOW_START = "2025-08-04"
|
||||
WINDOW_END = "2026-08-04"
|
||||
|
||||
#: Files that must match byte-for-byte on compare. run.log is deliberately
|
||||
#: absent (timestamped stderr, never comparable).
|
||||
COMPARED_FILES = ("trades.csv", "summary.txt", "equity.csv", "metrics.json")
|
||||
|
||||
#: Canonical factory confirm map (pair_signals/xauusd.py) — pinned on the
|
||||
#: command line so each invocation is self-contained.
|
||||
CONFIRMS = {
|
||||
"5min": "15min,1h",
|
||||
"15min": "5min,1h",
|
||||
"1h": "15min,4h",
|
||||
"4h": "1h,1d",
|
||||
}
|
||||
|
||||
#: Mirrors BaseDataProvider.BARS_PER_DAY for the TFs the baseline touches.
|
||||
#: Kept local so the scripts never import repo modules (importing app_config
|
||||
#: pulls the .env into os.environ). Drift is self-detecting: if the pad
|
||||
#: math ever changes upstream, the expected cache file goes missing and the
|
||||
#: pre-flight aborts before any network fetch.
|
||||
_BARS_PER_DAY = {"5min": 288, "15min": 96, "1h": 24, "4h": 6, "1d": 1}
|
||||
|
||||
#: fetch_history logs an ERROR for parquets older than the raw-cache format
|
||||
#: change (2026-07-08) — those contain CLEANED bars and are invalid input.
|
||||
RAW_CACHE_EPOCH = 1783468800.0
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# Run specs
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
def run_specs() -> Dict[str, dict]:
|
||||
"""The 8-run baseline matrix: 4 primary TFs × spreads {0.30, 0.65}."""
|
||||
specs: Dict[str, dict] = {}
|
||||
for key, tf in (("m5", "5min"), ("m15", "15min"), ("h1", "1h"), ("h4", "4h")):
|
||||
for spread in ("0.30", "0.65"):
|
||||
run_key = f"{key}_s{spread.replace('.', '')}" # 0.30 → s030
|
||||
specs[run_key] = {
|
||||
"primary_tf": tf,
|
||||
"confirm_tfs": CONFIRMS[tf],
|
||||
"spread": spread,
|
||||
"argv": build_argv(tf, spread),
|
||||
}
|
||||
return specs
|
||||
|
||||
|
||||
def build_argv(primary_tf: str, spread: str) -> List[str]:
|
||||
"""Backtester argv with every reproducibility-relevant knob pinned.
|
||||
|
||||
--export-csv is appended at run time (path depends on the output dir).
|
||||
"""
|
||||
return [
|
||||
BACKTESTER_REL,
|
||||
"--pair", "XAUUSD",
|
||||
"--primary-tf", primary_tf,
|
||||
"--confirm-tfs", CONFIRMS[primary_tf],
|
||||
"--start", WINDOW_START,
|
||||
"--end", WINDOW_END,
|
||||
"--spread", spread,
|
||||
"--quality", "A+,A,B,C",
|
||||
"--warmup-bars", "250",
|
||||
"--cooldown-bars", "5",
|
||||
"--max-hold-bars", "300",
|
||||
"--workers", "4",
|
||||
"--reasoning-samples", "3",
|
||||
"--detail",
|
||||
]
|
||||
|
||||
|
||||
def expected_cache_files(primary_tf: str) -> List[str]:
|
||||
"""Parquet filenames a run will read (mirrors gpu_backtest pad math).
|
||||
|
||||
pad_days = max(60, warmup_bars // 4) = 62 at the pinned 250;
|
||||
per-TF floor min_pad_days = ceil(210 / bars_per_day * 7/5) + 5.
|
||||
"""
|
||||
start = datetime.strptime(WINDOW_START, "%Y-%m-%d")
|
||||
end = datetime.strptime(WINDOW_END, "%Y-%m-%d")
|
||||
pad_days = max(60, 250 // 4)
|
||||
names = []
|
||||
for tf in [primary_tf] + CONFIRMS[primary_tf].split(","):
|
||||
min_pad = math.ceil(210 / _BARS_PER_DAY[tf] * 7 / 5) + 5
|
||||
tf_start = start - timedelta(days=max(pad_days, min_pad))
|
||||
names.append(
|
||||
f"oanda_XAU_USD_{tf}_{tf_start:%Y%m%d}_{end:%Y%m%d}.parquet")
|
||||
return names
|
||||
|
||||
|
||||
def all_expected_cache_files(specs: Dict[str, dict]) -> List[str]:
|
||||
seen: List[str] = []
|
||||
for spec in specs.values():
|
||||
for name in expected_cache_files(spec["primary_tf"]):
|
||||
if name not in seen:
|
||||
seen.append(name)
|
||||
return seen
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# Subprocess runner + stderr sanity
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
def run_backtest(argv: List[str], run_dir: Path) -> None:
|
||||
"""Run one backtest: stdout → summary.txt, stderr → run.log.
|
||||
|
||||
Raises RuntimeError (with the log path) on a nonzero exit.
|
||||
"""
|
||||
run_dir.mkdir(parents=True, exist_ok=True)
|
||||
env = {k: v for k, v in os.environ.items() if k != "CUDA_PATH"}
|
||||
env["PYTHONHASHSEED"] = "0"
|
||||
cmd = ([str(VENV_PYTHON)] + argv
|
||||
+ ["--export-csv", str(run_dir / "trades.csv")])
|
||||
with open(run_dir / "summary.txt", "wb") as out, \
|
||||
open(run_dir / "run.log", "wb") as err:
|
||||
rc = subprocess.run(cmd, cwd=ROOT, env=env,
|
||||
stdout=out, stderr=err).returncode
|
||||
if rc != 0:
|
||||
raise RuntimeError(
|
||||
f"backtest exited {rc} — see {run_dir / 'run.log'}")
|
||||
|
||||
|
||||
def check_run_log(run_dir: Path, primary_tf: str) -> Tuple[List[str], str]:
|
||||
"""Sanity-check a run's stderr. Returns (problems, backend_string).
|
||||
|
||||
Hard requirements: every TF loaded from the parquet cache (zero
|
||||
"Fetching" lines — a live fetch means the input data is not the
|
||||
baseline's data) and a parseable Backend line.
|
||||
"""
|
||||
text = (run_dir / "run.log").read_text(errors="replace")
|
||||
problems: List[str] = []
|
||||
if "Fetching " in text:
|
||||
problems.append("live fetch attempted (cache miss) — data is not "
|
||||
"the baseline's data")
|
||||
for tf in [primary_tf] + CONFIRMS[primary_tf].split(","):
|
||||
if f"Loading cached XAU/USD {tf}:" not in text:
|
||||
problems.append(f"no cache-hit line for TF {tf}")
|
||||
if "predates the raw-cache format" in text:
|
||||
problems.append("pre-2026-07-08 cache file in use (cleaned bars)")
|
||||
backend = ""
|
||||
for line in text.splitlines():
|
||||
if "Backend: " in line:
|
||||
backend = line.split("Backend: ", 1)[1].split(" | ", 1)[0].strip()
|
||||
break
|
||||
if not backend:
|
||||
problems.append("no Backend line found in run.log")
|
||||
return problems, backend
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# Derived outputs (the backtester exports no equity curve — derive it
|
||||
# from the trade book instead of modifying gpu_backtest.py)
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _resolved_trades(trades_csv: Path) -> pd.DataFrame:
|
||||
"""Resolved trades in book order (matches compute_metrics' filter:
|
||||
unresolved '' / NO_DATA rows are exported to CSV but never scored)."""
|
||||
df = pd.read_csv(trades_csv, dtype=str, keep_default_na=False)
|
||||
df["r_multiple"] = pd.to_numeric(df["r_multiple"], errors="coerce")
|
||||
mask = (df["exit_reason"] != "") & (df["exit_reason"] != "NO_DATA") \
|
||||
& df["r_multiple"].notna()
|
||||
return df[mask].reset_index(drop=True)
|
||||
|
||||
|
||||
def derive_equity(trades_csv: Path, out_path: Path) -> None:
|
||||
"""Cumulative-R equity curve over resolved trades in book order."""
|
||||
df = _resolved_trades(trades_csv)
|
||||
eq = pd.DataFrame({
|
||||
"trade_n": range(1, len(df) + 1),
|
||||
"entry_time": df["entry_time"],
|
||||
"exit_time": df["exit_time"],
|
||||
"quality": df["quality"],
|
||||
"strategy": df["strategy"],
|
||||
"r_multiple": df["r_multiple"],
|
||||
})
|
||||
eq["cum_r"] = eq["r_multiple"].cumsum()
|
||||
eq["high_water"] = eq["cum_r"].cummax().clip(lower=0.0)
|
||||
eq["drawdown_r"] = eq["cum_r"] - eq["high_water"]
|
||||
eq.to_csv(out_path, index=False, float_format="%.6f")
|
||||
|
||||
|
||||
def derive_metrics(trades_csv: Path, out_path: Path) -> dict:
|
||||
"""Headline + grouped stats as sorted, fixed-precision JSON."""
|
||||
df = _resolved_trades(trades_csv)
|
||||
metrics = {"overall": _stats(df["r_multiple"]),
|
||||
"by_quality": {}, "by_strategy": {}}
|
||||
for col, bucket in (("quality", "by_quality"), ("strategy", "by_strategy")):
|
||||
for value in sorted(df[col].unique()):
|
||||
metrics[bucket][value] = _stats(df.loc[df[col] == value,
|
||||
"r_multiple"])
|
||||
out_path.write_text(json.dumps(metrics, indent=2, sort_keys=True) + "\n")
|
||||
return metrics
|
||||
|
||||
|
||||
def _stats(r: pd.Series) -> dict:
|
||||
n = len(r)
|
||||
wins = int((r > 0).sum())
|
||||
gross_win = float(r[r > 0].sum())
|
||||
gross_loss = float(-r[r < 0].sum())
|
||||
cum = r.cumsum()
|
||||
dd = cum - cum.cummax().clip(lower=0.0)
|
||||
return {
|
||||
"trades": n,
|
||||
"total_r": round(float(r.sum()), 6),
|
||||
"win_rate_pct": round(wins / n * 100, 6) if n else 0.0,
|
||||
"profit_factor": (round(gross_win / gross_loss, 6)
|
||||
if gross_loss > 0 else None),
|
||||
"max_dd_r": round(float(dd.min()), 6) if n else 0.0,
|
||||
}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# Hashing / manifest helpers
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 20), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def config_inputs() -> dict:
|
||||
"""Non-secret config.json fields that alter backtest results (the pair
|
||||
factory overlays `pairs`, and `data_providers.active` keys the cache)."""
|
||||
cfg_path = ROOT / "config.json"
|
||||
if not cfg_path.exists():
|
||||
return {"data_providers.active": None, "pairs": None}
|
||||
cfg = json.loads(cfg_path.read_text())
|
||||
return {
|
||||
"data_providers.active": cfg.get("data_providers", {}).get("active"),
|
||||
"pairs": cfg.get("pairs"),
|
||||
}
|
||||
|
||||
|
||||
def git_info() -> dict:
|
||||
def _git(*args: str) -> str:
|
||||
return subprocess.run(["git", *args], cwd=ROOT, capture_output=True,
|
||||
text=True).stdout.strip()
|
||||
return {
|
||||
"commit": _git("rev-parse", "HEAD"),
|
||||
"commit_short": _git("rev-parse", "--short", "HEAD"),
|
||||
# -uno: untracked files (e.g. these scripts pre-commit) can't
|
||||
# change the backtester; only tracked modifications count.
|
||||
"dirty": bool(_git("status", "--porcelain", "-uno")),
|
||||
}
|
||||
|
||||
|
||||
def tool_versions() -> dict:
|
||||
import numpy
|
||||
versions = {
|
||||
"python": ".".join(map(str, sys.version_info[:3])),
|
||||
"numpy": numpy.__version__,
|
||||
"pandas": pd.__version__,
|
||||
}
|
||||
try:
|
||||
from importlib.metadata import version
|
||||
versions["cupy"] = version("cupy")
|
||||
except Exception:
|
||||
versions["cupy"] = None
|
||||
return versions
|
||||
|
||||
|
||||
def utc_now_iso() -> str:
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
@@ -0,0 +1,183 @@
|
||||
#!/usr/bin/env python
|
||||
"""Capture the byte-for-byte backtest baseline at the current commit.
|
||||
|
||||
Runs the 8-run matrix (M5/M15/H1/H4 × spread 0.30/0.65, window
|
||||
2025-08-04 → 2026-08-04) through backtest/gpu_backtest.py and pins every
|
||||
output under baseline/<commit_short>/:
|
||||
|
||||
baseline/<commit>/
|
||||
manifest.json commands, input hashes, output hashes, env
|
||||
README.md layout + how to verify
|
||||
<run_key>/
|
||||
trades.csv full trade book (--export-csv)
|
||||
summary.txt backtester stdout (byte-comparable)
|
||||
equity.csv derived cumulative-R curve
|
||||
metrics.json derived headline + grouped stats
|
||||
run.log backtester stderr (provenance only)
|
||||
|
||||
Aborts before running anything if the tracked tree is dirty or any needed
|
||||
cache parquet is missing (a cache miss would fetch live — wrong data, and
|
||||
the local .env Oanda token is stale). Verify afterwards with
|
||||
scripts/compare_baseline.py — it must print all-PASS at the same commit.
|
||||
|
||||
Usage:
|
||||
venv/bin/python scripts/capture_baseline.py [--runs m5_s030,...] [--force]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import baseline_common as bc
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument("--runs", default="",
|
||||
help="comma-separated run keys to capture (default: all)")
|
||||
ap.add_argument("--force", action="store_true",
|
||||
help="overwrite existing run dirs / manifest")
|
||||
args = ap.parse_args()
|
||||
|
||||
git = bc.git_info()
|
||||
if git["dirty"]:
|
||||
print("ABORT: tracked working tree is dirty — a baseline must be "
|
||||
"attributable to exactly one commit. Commit or stash first.")
|
||||
return 2
|
||||
|
||||
specs = bc.run_specs()
|
||||
if args.runs:
|
||||
wanted = [k.strip() for k in args.runs.split(",") if k.strip()]
|
||||
unknown = [k for k in wanted if k not in specs]
|
||||
if unknown:
|
||||
print(f"ABORT: unknown run key(s) {unknown}; "
|
||||
f"valid: {', '.join(specs)}")
|
||||
return 2
|
||||
specs = {k: specs[k] for k in wanted}
|
||||
|
||||
base_dir = bc.BASELINE_ROOT / git["commit_short"]
|
||||
if not args.force:
|
||||
clashes = [k for k in specs if (base_dir / k).exists()]
|
||||
if clashes:
|
||||
print(f"ABORT: {base_dir} already has run(s) {clashes} "
|
||||
"(use --force to overwrite)")
|
||||
return 2
|
||||
|
||||
# Pre-flight: every parquet the runs will read must already exist and
|
||||
# postdate the raw-cache format change. Never trigger a network fetch.
|
||||
problems = []
|
||||
cache_hashes = {}
|
||||
for name in bc.all_expected_cache_files(specs):
|
||||
path = bc.CACHE_DIR / name
|
||||
if not path.exists():
|
||||
problems.append(f"missing cache parquet: {name}")
|
||||
elif path.stat().st_mtime < bc.RAW_CACHE_EPOCH:
|
||||
problems.append(f"pre-raw-format cache parquet (cleaned bars): "
|
||||
f"{name}")
|
||||
else:
|
||||
cache_hashes[name] = bc.sha256_file(path)
|
||||
if problems:
|
||||
print("ABORT: cache pre-flight failed — refusing to run "
|
||||
"(a miss would fetch live data):")
|
||||
for p in problems:
|
||||
print(f" - {p}")
|
||||
return 2
|
||||
print(f"Pre-flight OK: {len(cache_hashes)} cache parquets present "
|
||||
f"and hashed. Capturing {len(specs)} run(s) at {git['commit_short']}.")
|
||||
|
||||
manifest_runs = {}
|
||||
backend = None
|
||||
for i, (key, spec) in enumerate(specs.items(), 1):
|
||||
run_dir = base_dir / key
|
||||
print(f"[{i}/{len(specs)}] {key}: {spec['primary_tf']} "
|
||||
f"spread {spec['spread']} ...", flush=True)
|
||||
bc.run_backtest(spec["argv"], run_dir)
|
||||
log_problems, run_backend = bc.check_run_log(run_dir,
|
||||
spec["primary_tf"])
|
||||
if log_problems:
|
||||
print(f"ABORT: {key} failed stderr sanity checks:")
|
||||
for p in log_problems:
|
||||
print(f" - {p}")
|
||||
print(f" log: {run_dir / 'run.log'}")
|
||||
return 2
|
||||
if backend is None:
|
||||
backend = run_backend
|
||||
elif run_backend != backend:
|
||||
print(f"ABORT: backend flipped mid-capture "
|
||||
f"({backend!r} → {run_backend!r})")
|
||||
return 2
|
||||
bc.derive_equity(run_dir / "trades.csv", run_dir / "equity.csv")
|
||||
metrics = bc.derive_metrics(run_dir / "trades.csv",
|
||||
run_dir / "metrics.json")
|
||||
manifest_runs[key] = {
|
||||
"argv": spec["argv"],
|
||||
"primary_tf": spec["primary_tf"],
|
||||
"cache_files": bc.expected_cache_files(spec["primary_tf"]),
|
||||
"files": {f: bc.sha256_file(run_dir / f)
|
||||
for f in bc.COMPARED_FILES},
|
||||
}
|
||||
o = metrics["overall"]
|
||||
print(f" {o['trades']} trades, {o['total_r']:+.2f}R, "
|
||||
f"PF {o['profit_factor']}, maxDD {o['max_dd_r']:.2f}R")
|
||||
|
||||
manifest = {
|
||||
"created_utc": bc.utc_now_iso(),
|
||||
"commit": git["commit"],
|
||||
"commit_short": git["commit_short"],
|
||||
"dirty": git["dirty"],
|
||||
"window": {"start": bc.WINDOW_START, "end": bc.WINDOW_END},
|
||||
"backend": backend,
|
||||
"env_overrides": {"CUDA_PATH": "removed", "PYTHONHASHSEED": "0"},
|
||||
"versions": bc.tool_versions(),
|
||||
"config_inputs": bc.config_inputs(),
|
||||
"cache_files": cache_hashes,
|
||||
"compared_files": list(bc.COMPARED_FILES),
|
||||
"runs": manifest_runs,
|
||||
}
|
||||
manifest_path = base_dir / "manifest.json"
|
||||
if manifest_path.exists():
|
||||
# Partial re-capture with --runs: merge into the existing manifest
|
||||
# (only the just-captured keys are replaced; same commit is
|
||||
# guaranteed by the dir name).
|
||||
old = json.loads(manifest_path.read_text())
|
||||
old["runs"].update(manifest_runs)
|
||||
old["cache_files"].update(cache_hashes)
|
||||
old["created_utc"] = manifest["created_utc"]
|
||||
manifest = old
|
||||
manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True)
|
||||
+ "\n")
|
||||
_write_readme(base_dir)
|
||||
print(f"\nBaseline captured: {base_dir}")
|
||||
print("Verify now: venv/bin/python scripts/compare_baseline.py "
|
||||
"(must be all-PASS at this commit)")
|
||||
return 0
|
||||
|
||||
|
||||
def _write_readme(base_dir: Path) -> None:
|
||||
(base_dir / "README.md").write_text(f"""\
|
||||
# Backtest baseline — commit {base_dir.name}
|
||||
|
||||
Reference outputs of `backtest/gpu_backtest.py` for XAU/USD, window
|
||||
{bc.WINDOW_START} → {bc.WINDOW_END}, 4 primary TFs × spreads 0.30/0.65.
|
||||
Captured by `scripts/capture_baseline.py`; verify with
|
||||
`venv/bin/python scripts/compare_baseline.py` (re-runs every command and
|
||||
byte-diffs the outputs, PASS/FAIL per file).
|
||||
|
||||
Per run dir: `trades.csv` (full book), `summary.txt` (backtester stdout),
|
||||
`equity.csv` + `metrics.json` (derived from the book). Those four are
|
||||
byte-compared. `run.log` is the backtester's stderr — timestamped, kept
|
||||
for provenance only, never compared.
|
||||
|
||||
`manifest.json` records the exact argv per run, sha256 of every input
|
||||
parquet and output file, backend, tool versions, and the non-secret
|
||||
config inputs that alter results. Input data must come from
|
||||
`backtest/cache/` — if those parquets change, comparisons are no longer
|
||||
attributable to code changes and the compare script aborts.
|
||||
""")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env python
|
||||
"""Re-run the baseline backtests and byte-diff every output against baseline/.
|
||||
|
||||
For each captured run, replays the exact argv recorded in the baseline's
|
||||
manifest.json (same env pinning: no CUDA_PATH, PYTHONHASHSEED=0), rebuilds
|
||||
the derived files, and compares trades.csv / summary.txt / equity.csv /
|
||||
metrics.json byte-for-byte (sha256), printing PASS/FAIL per file.
|
||||
run.log (stderr) is provenance-only and never compared.
|
||||
|
||||
All-PASS at the capture commit proves determinism; a FAIL after a code
|
||||
change means the change altered backtest behavior — which is exactly what
|
||||
this script exists to detect. Exit code 0 only if every file PASSes.
|
||||
|
||||
Pre-flight aborts (before any run) if an input cache parquet is missing or
|
||||
its hash changed: with different input data, diffs are not attributable to
|
||||
code. Live fetches are treated as failures for the same reason.
|
||||
|
||||
Usage:
|
||||
venv/bin/python scripts/compare_baseline.py
|
||||
[--baseline baseline/<commit>] [--runs m5_s030,...] [--keep]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import baseline_common as bc
|
||||
|
||||
|
||||
def newest_baseline() -> Path | None:
|
||||
if not bc.BASELINE_ROOT.is_dir():
|
||||
return None
|
||||
dirs = [d for d in bc.BASELINE_ROOT.iterdir()
|
||||
if (d / "manifest.json").exists()]
|
||||
return max(dirs, key=lambda d: (d / "manifest.json").stat().st_mtime,
|
||||
default=None)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument("--baseline", default=None,
|
||||
help="baseline dir (default: newest under baseline/)")
|
||||
ap.add_argument("--runs", default="",
|
||||
help="comma-separated run keys to compare (default: all)")
|
||||
ap.add_argument("--keep", action="store_true",
|
||||
help="keep re-run outputs even on all-PASS")
|
||||
args = ap.parse_args()
|
||||
|
||||
base_dir = Path(args.baseline) if args.baseline else newest_baseline()
|
||||
if not base_dir or not (base_dir / "manifest.json").exists():
|
||||
print("ABORT: no baseline found — run scripts/capture_baseline.py "
|
||||
"first (or pass --baseline).")
|
||||
return 2
|
||||
manifest = json.loads((base_dir / "manifest.json").read_text())
|
||||
|
||||
runs = manifest["runs"]
|
||||
if args.runs:
|
||||
wanted = [k.strip() for k in args.runs.split(",") if k.strip()]
|
||||
unknown = [k for k in wanted if k not in runs]
|
||||
if unknown:
|
||||
print(f"ABORT: unknown run key(s) {unknown}; "
|
||||
f"baseline has: {', '.join(runs)}")
|
||||
return 2
|
||||
runs = {k: runs[k] for k in wanted}
|
||||
|
||||
git = bc.git_info()
|
||||
print(f"Baseline: {base_dir} (captured at {manifest['commit_short']}"
|
||||
f"{', DIRTY' if manifest.get('dirty') else ''})")
|
||||
print(f"Current HEAD: {git['commit_short']}"
|
||||
f"{' (dirty)' if git['dirty'] else ''}"
|
||||
+ (" — same commit, expecting all-PASS"
|
||||
if git["commit"] == manifest["commit"]
|
||||
else " — differs from baseline; FAILs would mean changed "
|
||||
"backtest behavior"))
|
||||
|
||||
# ── Pre-flight: identical input data, or nothing is attributable ──
|
||||
needed = sorted({f for r in runs.values() for f in r["cache_files"]})
|
||||
data_problems = []
|
||||
for name in needed:
|
||||
path = bc.CACHE_DIR / name
|
||||
if not path.exists():
|
||||
data_problems.append(f"missing cache parquet: {name}")
|
||||
elif bc.sha256_file(path) != manifest["cache_files"].get(name):
|
||||
data_problems.append(f"cache parquet CHANGED since capture: "
|
||||
f"{name}")
|
||||
if data_problems:
|
||||
print("ABORT: input data drift — diffs would not be attributable "
|
||||
"to code:")
|
||||
for p in data_problems:
|
||||
print(f" - {p}")
|
||||
return 2
|
||||
|
||||
cfg_now = bc.config_inputs()
|
||||
if cfg_now != manifest.get("config_inputs"):
|
||||
print("WARNING: non-secret config.json inputs differ from capture "
|
||||
"time (data_providers.active / pairs overlay) — these alter "
|
||||
"results:")
|
||||
print(f" baseline: {manifest.get('config_inputs')}")
|
||||
print(f" now: {cfg_now}")
|
||||
|
||||
work_dir = Path(tempfile.mkdtemp(prefix="baseline_compare_"))
|
||||
print(f"Re-running {len(runs)} run(s) into {work_dir}\n")
|
||||
|
||||
n_pass = n_fail = 0
|
||||
failed_runs = []
|
||||
for i, (key, spec) in enumerate(runs.items(), 1):
|
||||
run_dir = work_dir / key
|
||||
print(f"[{i}/{len(runs)}] {key}")
|
||||
try:
|
||||
bc.run_backtest(spec["argv"], run_dir)
|
||||
problems, backend = bc.check_run_log(run_dir, spec["primary_tf"])
|
||||
if backend and backend != manifest.get("backend"):
|
||||
problems.append(f"backend {backend!r} != baseline "
|
||||
f"{manifest.get('backend')!r}")
|
||||
if problems:
|
||||
raise RuntimeError("; ".join(problems))
|
||||
bc.derive_equity(run_dir / "trades.csv", run_dir / "equity.csv")
|
||||
bc.derive_metrics(run_dir / "trades.csv",
|
||||
run_dir / "metrics.json")
|
||||
except Exception as exc:
|
||||
print(f" RUN FAILED: {exc}")
|
||||
for fname in bc.COMPARED_FILES:
|
||||
print(f" {fname:<13} FAIL (run failed)")
|
||||
n_fail += 1
|
||||
failed_runs.append(key)
|
||||
continue
|
||||
|
||||
run_ok = True
|
||||
for fname in bc.COMPARED_FILES:
|
||||
new = run_dir / fname
|
||||
ref = base_dir / key / fname
|
||||
same = (new.exists() and ref.exists()
|
||||
and bc.sha256_file(new) == spec["files"][fname])
|
||||
print(f" {fname:<13} {'PASS' if same else 'FAIL'}")
|
||||
if same:
|
||||
n_pass += 1
|
||||
else:
|
||||
n_fail += 1
|
||||
run_ok = False
|
||||
if new.exists() and ref.exists():
|
||||
print(f" baseline: {ref} ({ref.stat().st_size} B)")
|
||||
print(f" re-run: {new} ({new.stat().st_size} B)")
|
||||
if not run_ok:
|
||||
failed_runs.append(key)
|
||||
|
||||
total = n_pass + n_fail
|
||||
print(f"\nRESULT: {n_pass}/{total} files PASS across {len(runs)} run(s) "
|
||||
f"vs baseline {manifest['commit_short']}")
|
||||
if n_fail:
|
||||
print(f"FAILED runs: {', '.join(dict.fromkeys(failed_runs))}")
|
||||
print(f"Re-run outputs kept for inspection: {work_dir}")
|
||||
return 1
|
||||
if args.keep:
|
||||
print(f"Re-run outputs kept (--keep): {work_dir}")
|
||||
else:
|
||||
shutil.rmtree(work_dir, ignore_errors=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user