From 9942736762297114ea191abf999db413f0af500c Mon Sep 17 00:00:00 2001 From: Harwinder Sekhon Date: Thu, 10 Sep 2026 07:55:20 -0400 Subject: [PATCH] providers: floor every live fetch at 1000 bars (H4/D1 indicator-seeding parity) The engine's EMAs use ewm(adjust=False), so the first bar of a fetched frame keeps weight (1-2/201)^n in the 200-bar trend EMA. With the 60-day lookback the live H4 frame had 380 bars (seed weight 2.3%) and D1 300 (5%), while the backtester seeds from its whole cached history. Measured on the deepest Oanda caches (267 H4 / 313 D1 bars sampled over two years, raw and cleaned frames): ema_trend flipped on 2/267 H4 and 1/313 D1 bars at those depths (composite off by up to 9.5 points) and on none from 600 bars up. MIN_LIVE_BARS = 1000 leaves a seed weight of e^-10; M5/M15 (5000) and H1 (1460) already exceed it and are unchanged. One request either way. The backtester never calls fetch_ohlcv, so no book changes. Tests: new Module 16 depth test; the Module 21 cache test's 300-bar expectation becomes 1000. Suite 500 passed. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01NQDjW8rXYrJR4JDWMWKUij --- providers/base.py | 26 +++++++++++++++++++++----- test_all.py | 32 +++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/providers/base.py b/providers/base.py index fb105bb..440798a 100644 --- a/providers/base.py +++ b/providers/base.py @@ -96,6 +96,20 @@ class BaseDataProvider(ABC): #: Hard per-request bar cap shared by Twelve Data and Oanda. MAX_BARS_PER_REQUEST = 5000 + #: Minimum bars per LIVE fetch on every timeframe (2026-09-10, + #: indicator-seeding parity with the backtester). The engine's EMAs + #: use ewm(adjust=False), so the first bar of the fetched frame keeps + #: weight (1-2/201)^n in the 200-bar trend EMA. With the 60-day + #: lookback, H4 fetched 380 bars (seed weight 2.3%) and D1 300 (5%) + #: while the backtester seeds from its whole cached history. Measured + #: on the deepest Oanda caches (267 H4 / 313 D1 bars sampled over two + #: years, raw and cleaned frames alike): ema_trend flipped on 2/267 H4 + #: and 1/313 D1 bars at those depths (composite off by up to 9.5 + #: points) and on none from 600 bars up; 1000 leaves a seed weight of + #: e^-10. M5/M15 (5000) and H1 (1460) already exceed it and are + #: unchanged. One request either way (cap 5000). + MIN_LIVE_BARS = 1000 + # Grace wait before the one retry when a fresh response at a candle # boundary is missing the just-closed bar (feed latency). FRESH_FETCH_RETRY_SECONDS = 3.0 @@ -140,6 +154,12 @@ class BaseDataProvider(ABC): # ── Public API ───────────────────────────────────────────────── + def _live_outputsize(self, timeframe: Timeframe, lookback_days: int) -> int: + """Bars requested for one live fetch: the lookback in bars, floored + at MIN_LIVE_BARS (indicator seeding) and capped at the request cap.""" + wanted = int(lookback_days * self.BARS_PER_DAY[timeframe]) + 20 + return max(self.MIN_LIVE_BARS, min(wanted, self.MAX_BARS_PER_REQUEST)) + def fetch_ohlcv( self, symbol: str = XAUUSD, @@ -151,11 +171,7 @@ class BaseDataProvider(ABC): lookback_days = self.config.data_lookback_days cache_key = f"{symbol}_{timeframe.value}" - outputsize = max( - 300, - min(int(lookback_days * self.BARS_PER_DAY[timeframe]) + 20, - self.MAX_BARS_PER_REQUEST), - ) + outputsize = self._live_outputsize(timeframe, lookback_days) while True: now = time.time() diff --git a/test_all.py b/test_all.py index 4fa7fbf..15aa4a9 100644 --- a/test_all.py +++ b/test_all.py @@ -4023,6 +4023,35 @@ def _(): assert TiingoDataProvider.CANDLE_SECONDS is BaseDataProvider.CANDLE_SECONDS +@test("providers: live fetch depth floors every TF at MIN_LIVE_BARS (H4/D1 seeding parity)") +def _(): + import providers.base as pbase + + class _Depth(pbase.BaseDataProvider): + PROVIDER = "depth16" + + def _fetch_raw(self, symbol, timeframe, outputsize, + start=None, end=None): + raise RuntimeError("must not fetch") + + def get_pair_info(self, symbol): + return PAIR_INFO[XAUUSD] + + p = _Depth(PlatformConfig()) # data_lookback_days = 60 + assert pbase.BaseDataProvider.MIN_LIVE_BARS == 1000 + depth = {tf: p._live_outputsize(tf, 60) for tf in + (Timeframe.M5, Timeframe.M15, Timeframe.H1, Timeframe.H4, Timeframe.D1)} + # unchanged where the lookback already exceeds the floor ... + assert depth[Timeframe.M5] == 5000 and depth[Timeframe.M15] == 5000, depth + assert depth[Timeframe.H1] == 1460, depth + # ... raised where it did not (were 380 and 300 bars) + assert depth[Timeframe.H4] == 1000 and depth[Timeframe.D1] == 1000, depth + assert all(v <= pbase.BaseDataProvider.MAX_BARS_PER_REQUEST for v in depth.values()) + assert p._live_outputsize(Timeframe.M5, 1) == 1000, "floor applies to short lookbacks" + # the seed bar's weight in the 200-bar trend EMA at the floor + assert (1 - 2 / 201) ** pbase.BaseDataProvider.MIN_LIVE_BARS < 1e-4 + + @test("providers: fetch_range paginates, dedupes, and trims to the window") def _(): class FakeProvider(BaseDataProvider): @@ -5700,7 +5729,8 @@ def _(): with patch("providers.base._is_forex_weekend", lambda now: False): p = _m21_stub_provider() p.fetch_ohlcv(XAUUSD, Timeframe.H1, lookback_days=1) - assert len(p.calls) == 1 and p.calls[0] == 300 + # 1 day of H1 = 44 bars, floored at MIN_LIVE_BARS (2026-09-10). + assert len(p.calls) == 1 and p.calls[0] == 1000 p.fetch_ohlcv(XAUUSD, Timeframe.H1, lookback_days=60) assert len(p.calls) == 2, \ "a longer-lookback call within TTL must refetch, not serve the " \