backtest: walk-forward mode + extended-range data safety
--walk-forward slices ONE continuous replay into rolling train/test folds by entry time (default 12mo/3mo/3mo; no per-fold re-optimization) and reports per-fold PF / expectancy / max DD plus aggregated OOS stats, flagging test folds with PF < 1. Off by default — default runs are byte-identical (scripts/compare_baseline.py 32/32 PASS with this diff). Data safety for deep-history fetches: fetch_history refuses to recreate any baseline-pinned parquet filename and records the sha256 of every freshly written parquet in backtest/cache/data_manifest.json. Tests: Module 23 (fold builder, slicing, flagging, OOS dedup, CLI defaults, pin guard, manifest recording) — 367 total. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017FBjfomAipxvPWwRJEDDyJ
This commit is contained in:
Harwinder Sekhon
+241
@@ -6943,6 +6943,247 @@ def _():
|
||||
assert "must be > 0" in str(e)
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# MODULE 23: walk-forward mode + extended-range data safety
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
print("\n═══ Module 23: walk-forward mode + extended-range data safety ═══")
|
||||
|
||||
|
||||
def _wf_trade(ts, r):
|
||||
"""Resolved trade at ts (UTC) with realized R — for fold slicing tests."""
|
||||
t = _m21_sim_trade(ts=ts)
|
||||
t.exit_reason = "TP1" if r > 0 else "SL"
|
||||
t.exit_time = t.entry_time
|
||||
t.r_multiple = r
|
||||
t.bars_held = 3
|
||||
return t
|
||||
|
||||
|
||||
@test("walk-forward: fold builder — 12/3/3 over 24mo = 4 contiguous folds")
|
||||
def _():
|
||||
import gpu_backtest as _gb
|
||||
start = datetime(2021, 1, 1, tzinfo=timezone.utc)
|
||||
end = datetime(2023, 1, 1, tzinfo=timezone.utc)
|
||||
folds = _gb.build_walkforward_folds(start, end, 12, 3, 3)
|
||||
assert len(folds) == 4, f"expected 4 folds, got {len(folds)}"
|
||||
for i, (tr_s, tr_e, te_s, te_e) in enumerate(folds):
|
||||
assert te_s == tr_e, "test must start exactly at train end"
|
||||
assert tr_s == datetime(2021, 1 + 3 * i, 1, tzinfo=timezone.utc)
|
||||
assert folds[0][3] == datetime(2022, 4, 1, tzinfo=timezone.utc)
|
||||
assert folds[-1][3] == end, "last fold must end exactly at window end"
|
||||
|
||||
|
||||
@test("walk-forward: fold builder — month-end clamp, short window, bad months")
|
||||
def _():
|
||||
import gpu_backtest as _gb
|
||||
# Jan 31 + 1 month clamps to Feb 28 (calendar-month arithmetic).
|
||||
d = _gb._add_months(datetime(2021, 1, 31, tzinfo=timezone.utc), 1)
|
||||
assert (d.year, d.month, d.day) == (2021, 2, 28)
|
||||
# 14mo window can't fit train 12 + test 3 → no folds, no exception.
|
||||
start = datetime(2021, 1, 1, tzinfo=timezone.utc)
|
||||
assert _gb.build_walkforward_folds(
|
||||
start, datetime(2022, 3, 1, tzinfo=timezone.utc), 12, 3, 3) == []
|
||||
# 18mo window fits exactly two stepped folds.
|
||||
assert len(_gb.build_walkforward_folds(
|
||||
start, datetime(2022, 7, 1, tzinfo=timezone.utc), 12, 3, 3)) == 2
|
||||
for bad in ((0, 3, 3), (12, 0, 3), (12, 3, -1)):
|
||||
try:
|
||||
_gb.build_walkforward_folds(start, start, *bad)
|
||||
assert False, f"months {bad} must raise ValueError"
|
||||
except ValueError as e:
|
||||
assert "must all be > 0" in str(e)
|
||||
|
||||
|
||||
@test("walk-forward: fold slicing — half-open boundaries, per-fold metrics")
|
||||
def _():
|
||||
import gpu_backtest as _gb
|
||||
fold = (datetime(2024, 1, 1, tzinfo=timezone.utc),
|
||||
datetime(2025, 1, 1, tzinfo=timezone.utc),
|
||||
datetime(2025, 1, 1, tzinfo=timezone.utc),
|
||||
datetime(2025, 4, 1, tzinfo=timezone.utc))
|
||||
trades = [
|
||||
_wf_trade("2024-02-01 10:00", +1.0), # train
|
||||
_wf_trade("2024-03-01 10:00", -1.0), # train
|
||||
_wf_trade("2024-06-01 10:00", +2.0), # train
|
||||
_wf_trade("2025-01-01 00:00", -1.0), # exactly test_start → test
|
||||
_wf_trade("2025-02-15 10:00", +0.5), # test
|
||||
_wf_trade("2025-04-01 00:00", +9.0), # exactly test_end → excluded
|
||||
]
|
||||
unresolved = _wf_trade("2025-02-20 10:00", +1.0)
|
||||
unresolved.exit_reason = "" # never scored
|
||||
trades.append(unresolved)
|
||||
stats = _gb.walkforward_stats(trades, [fold])
|
||||
f = stats["folds"][0]
|
||||
tr, te = f["train"], f["test"]
|
||||
assert tr.n_trades == 3 and te.n_trades == 2
|
||||
assert abs(tr.profit_factor - 3.0) < 1e-9
|
||||
assert abs(tr.avg_r - 2.0 / 3.0) < 1e-9
|
||||
assert abs(tr.max_drawdown_r - 1.0) < 1e-9
|
||||
assert abs(te.profit_factor - 0.5) < 1e-9
|
||||
assert abs(te.avg_r - (-0.25)) < 1e-9
|
||||
assert abs(te.max_drawdown_r - 1.0) < 1e-9
|
||||
assert f["flagged"], "test PF 0.5 < 1.0 must be flagged"
|
||||
|
||||
|
||||
@test("walk-forward: flagging — PF≥1 and zero-trade folds are not flagged")
|
||||
def _():
|
||||
import gpu_backtest as _gb
|
||||
good = (datetime(2024, 1, 1, tzinfo=timezone.utc),
|
||||
datetime(2025, 1, 1, tzinfo=timezone.utc),
|
||||
datetime(2025, 1, 1, tzinfo=timezone.utc),
|
||||
datetime(2025, 4, 1, tzinfo=timezone.utc))
|
||||
empty = (datetime(2024, 4, 1, tzinfo=timezone.utc),
|
||||
datetime(2025, 4, 1, tzinfo=timezone.utc),
|
||||
datetime(2025, 4, 1, tzinfo=timezone.utc),
|
||||
datetime(2025, 7, 1, tzinfo=timezone.utc))
|
||||
trades = [_wf_trade("2025-02-01 10:00", +1.0),
|
||||
_wf_trade("2025-03-01 10:00", -0.5)]
|
||||
stats = _gb.walkforward_stats(trades, [good, empty])
|
||||
assert not stats["folds"][0]["flagged"], "PF 2.0 must not be flagged"
|
||||
assert stats["folds"][1]["test"].n_trades == 0
|
||||
assert not stats["folds"][1]["flagged"], "empty fold must not be flagged"
|
||||
assert stats["n_flagged"] == 0
|
||||
assert _gb._fmt_pf(stats["folds"][1]["test"]) == "—"
|
||||
|
||||
|
||||
@test("walk-forward: aggregated OOS dedupes overlapping test folds")
|
||||
def _():
|
||||
import gpu_backtest as _gb
|
||||
f1 = (datetime(2024, 1, 1, tzinfo=timezone.utc),
|
||||
datetime(2025, 1, 1, tzinfo=timezone.utc),
|
||||
datetime(2025, 1, 1, tzinfo=timezone.utc),
|
||||
datetime(2025, 4, 1, tzinfo=timezone.utc))
|
||||
f2 = (datetime(2024, 2, 1, tzinfo=timezone.utc),
|
||||
datetime(2025, 2, 1, tzinfo=timezone.utc),
|
||||
datetime(2025, 2, 1, tzinfo=timezone.utc),
|
||||
datetime(2025, 5, 1, tzinfo=timezone.utc))
|
||||
trades = [
|
||||
_wf_trade("2025-01-05 10:00", -1.0), # f1 test only
|
||||
_wf_trade("2025-02-15 10:00", +0.5), # BOTH test windows
|
||||
_wf_trade("2025-04-15 10:00", +2.0), # f2 test only
|
||||
]
|
||||
stats = _gb.walkforward_stats(trades, [f1, f2])
|
||||
o = stats["oos"]
|
||||
assert o.n_trades == 3, "overlapping-window trade must count exactly once"
|
||||
assert abs(o.total_r - 1.5) < 1e-9
|
||||
assert abs(o.max_drawdown_r - 1.0) < 1e-9 # chronological OOS equity
|
||||
assert stats["n_flagged"] == 1 # f1 PF 0.5; f2 has no losses
|
||||
|
||||
|
||||
@test("walk-forward: CLI — off by default, 12/3/3 defaults, short window refused")
|
||||
def _():
|
||||
import gpu_backtest as _gb
|
||||
a = _gb._parse_args(["--pair", "XAUUSD"])
|
||||
assert a.walk_forward is False
|
||||
assert (a.wf_train_months, a.wf_test_months, a.wf_step_months) == (12, 3, 3)
|
||||
a = _gb._parse_args(["--pair", "XAUUSD", "--walk-forward",
|
||||
"--wf-train-months", "6", "--wf-test-months", "2",
|
||||
"--wf-step-months", "1"])
|
||||
assert a.walk_forward is True
|
||||
assert (a.wf_train_months, a.wf_test_months, a.wf_step_months) == (6, 2, 1)
|
||||
# Default --days 180 can't fit train 12 + test 3 → refuse before any
|
||||
# data loading.
|
||||
try:
|
||||
_gb.main(["--pair", "XAUUSD", "--walk-forward"])
|
||||
assert False, "short window must raise SystemExit"
|
||||
except SystemExit as e:
|
||||
assert "shorter than one fold" in str(e)
|
||||
try:
|
||||
_gb.main(["--pair", "XAUUSD", "--walk-forward",
|
||||
"--wf-train-months", "0"])
|
||||
assert False, "zero train months must raise SystemExit"
|
||||
except SystemExit as e:
|
||||
assert "must all be > 0" in str(e)
|
||||
|
||||
|
||||
class _WFMockProvider:
|
||||
"""Offline provider stand-in for fetch_history cache-write tests."""
|
||||
name = "mockprov"
|
||||
|
||||
def __init__(self, df):
|
||||
self.df = df
|
||||
self.fetches = 0
|
||||
|
||||
def fetch_range(self, symbol, tf, start, end):
|
||||
self.fetches += 1
|
||||
return self.df
|
||||
|
||||
def validate_and_clean(self, df, symbol, tf):
|
||||
return df
|
||||
|
||||
|
||||
@test("fetch_history: refuses to recreate a baseline-pinned parquet")
|
||||
def _():
|
||||
import tempfile
|
||||
from pathlib import Path as _Path
|
||||
import gpu_backtest as _gb
|
||||
old_cache, old_base = _gb.CACHE_DIR, _gb.BASELINE_ROOT
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
_gb.CACHE_DIR = _Path(td) / "cache"
|
||||
_gb.BASELINE_ROOT = _Path(td) / "baseline"
|
||||
try:
|
||||
start = datetime(2020, 1, 1, tzinfo=timezone.utc)
|
||||
end = datetime(2020, 6, 1, tzinfo=timezone.utc)
|
||||
pinned = _gb._cache_path("XAU/USD", Timeframe.H1, start, end,
|
||||
"mockprov").name
|
||||
mdir = _gb.BASELINE_ROOT / "abc1234"
|
||||
mdir.mkdir(parents=True)
|
||||
(mdir / "manifest.json").write_text(
|
||||
json.dumps({"cache_files": {pinned: "0" * 64}}))
|
||||
prov = _WFMockProvider(make_ohlcv(30, seed=7))
|
||||
try:
|
||||
_gb.fetch_history(prov, "XAU/USD", Timeframe.H1, start, end)
|
||||
assert False, "must refuse to write a baseline-pinned name"
|
||||
except RuntimeError as e:
|
||||
assert "baseline" in str(e)
|
||||
assert not (_gb.CACHE_DIR / pinned).exists(), \
|
||||
"refused write must leave nothing behind"
|
||||
# A different window is a different filename → allowed.
|
||||
end2 = datetime(2020, 7, 1, tzinfo=timezone.utc)
|
||||
out = _gb.fetch_history(prov, "XAU/USD", Timeframe.H1, start, end2)
|
||||
assert len(out) == 30
|
||||
other = _gb._cache_path("XAU/USD", Timeframe.H1, start, end2,
|
||||
"mockprov").name
|
||||
assert (_gb.CACHE_DIR / other).exists()
|
||||
finally:
|
||||
_gb.CACHE_DIR, _gb.BASELINE_ROOT = old_cache, old_base
|
||||
|
||||
|
||||
@test("fetch_history: new parquet sha256 lands in data_manifest.json "
|
||||
"(baseline cache_files format); cache hits never refetch or rehash")
|
||||
def _():
|
||||
import hashlib
|
||||
import tempfile
|
||||
from pathlib import Path as _Path
|
||||
import gpu_backtest as _gb
|
||||
old_cache, old_base = _gb.CACHE_DIR, _gb.BASELINE_ROOT
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
_gb.CACHE_DIR = _Path(td) / "cache"
|
||||
_gb.BASELINE_ROOT = _Path(td) / "baseline" # no manifests → no pins
|
||||
try:
|
||||
start = datetime(2020, 1, 1, tzinfo=timezone.utc)
|
||||
end = datetime(2020, 6, 1, tzinfo=timezone.utc)
|
||||
prov = _WFMockProvider(make_ohlcv(30, seed=8))
|
||||
_gb.fetch_history(prov, "XAU/USD", Timeframe.H1, start, end)
|
||||
name = _gb._cache_path("XAU/USD", Timeframe.H1, start, end,
|
||||
"mockprov").name
|
||||
mpath = _gb.CACHE_DIR / "data_manifest.json"
|
||||
manifest = json.loads(mpath.read_text())
|
||||
assert "cache_files" in manifest and "updated_utc" in manifest
|
||||
want = hashlib.sha256(
|
||||
(_gb.CACHE_DIR / name).read_bytes()).hexdigest()
|
||||
assert manifest["cache_files"] == {name: want}, \
|
||||
"manifest must map filename → sha256 like the baseline format"
|
||||
# Second call: cache hit — no refetch, manifest untouched.
|
||||
before = mpath.read_text()
|
||||
_gb.fetch_history(prov, "XAU/USD", Timeframe.H1, start, end)
|
||||
assert prov.fetches == 1, "cache hit must not refetch"
|
||||
assert mpath.read_text() == before, \
|
||||
"cache hit must not rewrite the data manifest"
|
||||
finally:
|
||||
_gb.CACHE_DIR, _gb.BASELINE_ROOT = old_cache, old_base
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# SUMMARY
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
Reference in New Issue
Block a user