From c742c65ef696bf48755d510d1fcb269d4cae9688 Mon Sep 17 00:00:00 2001 From: Harwinder Sekhon Date: Thu, 10 Sep 2026 08:29:04 -0400 Subject: [PATCH] backtest: seed slow-TF history to the live 1000-bar floor (_history_pad_days) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-TF warm-up pad before --start targeted 210 bars, so a 3-year H4 book started with 372 H4 bars and ~210 D1 bars of seeding while the live scanner now fetches at least MIN_LIVE_BARS (1000) on every timeframe. _history_pad_days keeps the 62-day intraday pad (already >= 1488 bars, so M5/M15/H1 cache windows are untouched) and pads H4 by 234 and D1 by 1400 calendar days; the raw-path fetch for trade resolution reuses the primary timeframe's own start (one cache file per TF, as before). Verified on the 3-year books at $0.65: M5 and M15 byte-identical (c2722f98…, b2cfa9e2…); H1 and H4 trade-for-trade identical (253 / +70.02R, 114 / +21.68R) — the only differences are one H4 confirmation reading 15 days into the H1 window and 1e-6…1e-9 numerics in D1 Wilder recursions during Oct–Dec 2023, i.e. the window-start seeding zone. Evidence: backtest/research/cycle_2026-09-10.md. Tests: Module 21 "history pad seeds slow TFs like live". Suite 501 passed. Claude-Session: https://claude.ai/code/session_01NQDjW8rXYrJR4JDWMWKUij --- backtest/gpu_backtest.py | 42 +++++++++++++++++++++++++++------------- test_all.py | 15 ++++++++++++++ 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/backtest/gpu_backtest.py b/backtest/gpu_backtest.py index 3922895..9671606 100644 --- a/backtest/gpu_backtest.py +++ b/backtest/gpu_backtest.py @@ -973,6 +973,28 @@ class GPUTechnicalEngine(TechnicalEngine): CACHE_DIR = Path(__file__).resolve().parent / "cache" +def _history_pad_days(tf: Timeframe, warmup_bars: int) -> int: + """Calendar days of history fetched BEFORE --start for one timeframe. + + Base pad: max(60, warmup_bars // 4) days (62 at the default 250), the + EMA200/ADX warm-up every book since 2026-07 was built on. Slow TFs + need more: 62 days of H4 is 372 bars and of D1 only ~44, while the + live scanner now fetches at least BaseDataProvider.MIN_LIVE_BARS + (1000) bars on every timeframe (2026-09-10 seeding-parity fix — the + engine's EMAs use ewm(adjust=False), so the first bar of the frame + keeps weight (1-2/201)^n in the trend EMA). Target that same depth + here, converting trading to calendar days with 7/5, so the window + START is seeded like the rest of the window and like live. The + intraday pads are unchanged (M5/M15/H1 already hold >= 1488 bars in + 62 days), which keeps their cache windows and books byte-identical; + only H4 (234 days) and D1 (1400 days) fetch deeper. + """ + base = max(60, warmup_bars // 4) + bars_per_day = BaseDataProvider.BARS_PER_DAY[tf] + seed_days = int(np.ceil(BaseDataProvider.MIN_LIVE_BARS / bars_per_day * 7 / 5)) + return max(base, seed_days) + + def _cache_path( symbol: str, tf: Timeframe, start: datetime, end: datetime, provider_name: str = "twelve_data", @@ -1817,23 +1839,17 @@ def run_backtest( flt = platform_cfg.backtest if flt.regime_filter and flt.regime_filter_timeframe not in timeframes: timeframes.append(flt.regime_filter_timeframe) - # Pad lookback so earliest TF has enough warmup for EMA200, ADX, etc. - pad_days = max(60, bt_cfg.warmup_bars // 4) - fetch_start = bt_cfg.start - timedelta(days=pad_days) - dfs_full: Dict[Timeframe, pd.DataFrame] = {} # Which parquet each TF came from (recorded in --record's manifest so # a study can prove two runs read the same bars). cache_names: Dict[str, str] = {} + # Per-TF history start; the raw-path fetch below reuses the primary's + # so the cleaned and raw frames come from the SAME cache file. + tf_starts: Dict[Timeframe, datetime] = {} for tf in timeframes: - # Slow TFs need a longer pad: 60 days of D1 is only ~44 bars — no - # EMA200, truncated swing/divergence windows — while live always has - # ≥210 bars (outputsize floor). Target data_provider's min_required - # (210 bars); ×7/5 converts trading days to calendar days. Intraday - # TFs stay on the base pad, so their books are unchanged. - bars_per_day = BaseDataProvider.BARS_PER_DAY[tf] - min_pad_days = int(np.ceil(210 / bars_per_day * 7 / 5)) + 5 - tf_start = bt_cfg.start - timedelta(days=max(pad_days, min_pad_days)) + tf_start = bt_cfg.start - timedelta( + days=_history_pad_days(tf, bt_cfg.warmup_bars)) + tf_starts[tf] = tf_start cache_names[tf.value] = _cache_path( bt_cfg.pair, tf, tf_start, bt_cfg.end, provider.name).name df = fetch_history( @@ -1939,7 +1955,7 @@ def run_backtest( # (news candles) that hit stops in reality. Cache hit → no API call. sim_df = fetch_history( provider, bt_cfg.pair, bt_cfg.primary_tf, - start=fetch_start, end=bt_cfg.end, + start=tf_starts[bt_cfg.primary_tf], end=bt_cfg.end, use_cache=bt_cfg.use_cache, clean=False, ) _reanchor_entries_to_raw(trades, signal_ts_list, sim_df) diff --git a/test_all.py b/test_all.py index 15aa4a9..7dc6717 100644 --- a/test_all.py +++ b/test_all.py @@ -5849,6 +5849,21 @@ def _(): "no 0.0-defaulted prices may appear" +@test("backtester: history pad seeds slow TFs like live (H4/D1 deeper, intraday unchanged)") +def _(): + import backtest.gpu_backtest as _gb + pad = {tf: _gb._history_pad_days(tf, 250) for tf in + (Timeframe.M5, Timeframe.M15, Timeframe.H1, Timeframe.H4, Timeframe.D1)} + # intraday pads keep the 62-day base -> cache windows and books unchanged + assert pad[Timeframe.M5] == 62 and pad[Timeframe.M15] == 62 and pad[Timeframe.H1] == 62, pad + # slow TFs fetch enough history for MIN_LIVE_BARS (1000) bars: H4 6/day, D1 1/day + assert pad[Timeframe.H4] == 234 and pad[Timeframe.D1] == 1400, pad + # base pad still follows --warmup-bars when that is the larger requirement + assert _gb._history_pad_days(Timeframe.M5, 1000) == 250 + # the floor is the provider's constant, not a second copy of the number + assert _gb.BaseDataProvider.MIN_LIVE_BARS == 1000 + + @test("backtester: NaN bar inside a trailing window cannot poison R/metrics") def _(): import gpu_backtest as _gb