providers: floor every live fetch at 1000 bars (H4/D1 indicator-seeding parity)
The engine's EMAs use ewm(adjust=False), so the first bar of a fetched frame keeps weight (1-2/201)^n in the 200-bar trend EMA. With the 60-day lookback the live H4 frame had 380 bars (seed weight 2.3%) and D1 300 (5%), while the backtester seeds from its whole cached history. Measured on the deepest Oanda caches (267 H4 / 313 D1 bars sampled over two years, raw and cleaned frames): ema_trend flipped on 2/267 H4 and 1/313 D1 bars at those depths (composite off by up to 9.5 points) and on none from 600 bars up. MIN_LIVE_BARS = 1000 leaves a seed weight of e^-10; M5/M15 (5000) and H1 (1460) already exceed it and are unchanged. One request either way. The backtester never calls fetch_ohlcv, so no book changes. Tests: new Module 16 depth test; the Module 21 cache test's 300-bar expectation becomes 1000. Suite 500 passed. Claude-Session: https://claude.ai/code/session_01NQDjW8rXYrJR4JDWMWKUij
This commit is contained in:
+21
-5
@@ -96,6 +96,20 @@ class BaseDataProvider(ABC):
|
|||||||
#: Hard per-request bar cap shared by Twelve Data and Oanda.
|
#: Hard per-request bar cap shared by Twelve Data and Oanda.
|
||||||
MAX_BARS_PER_REQUEST = 5000
|
MAX_BARS_PER_REQUEST = 5000
|
||||||
|
|
||||||
|
#: Minimum bars per LIVE fetch on every timeframe (2026-09-10,
|
||||||
|
#: indicator-seeding parity with the backtester). The engine's EMAs
|
||||||
|
#: use ewm(adjust=False), so the first bar of the fetched frame keeps
|
||||||
|
#: weight (1-2/201)^n in the 200-bar trend EMA. With the 60-day
|
||||||
|
#: lookback, H4 fetched 380 bars (seed weight 2.3%) and D1 300 (5%)
|
||||||
|
#: while the backtester seeds from its whole cached history. Measured
|
||||||
|
#: on the deepest Oanda caches (267 H4 / 313 D1 bars sampled over two
|
||||||
|
#: years, raw and cleaned frames alike): ema_trend flipped on 2/267 H4
|
||||||
|
#: and 1/313 D1 bars at those depths (composite off by up to 9.5
|
||||||
|
#: points) and on none from 600 bars up; 1000 leaves a seed weight of
|
||||||
|
#: e^-10. M5/M15 (5000) and H1 (1460) already exceed it and are
|
||||||
|
#: unchanged. One request either way (cap 5000).
|
||||||
|
MIN_LIVE_BARS = 1000
|
||||||
|
|
||||||
# Grace wait before the one retry when a fresh response at a candle
|
# Grace wait before the one retry when a fresh response at a candle
|
||||||
# boundary is missing the just-closed bar (feed latency).
|
# boundary is missing the just-closed bar (feed latency).
|
||||||
FRESH_FETCH_RETRY_SECONDS = 3.0
|
FRESH_FETCH_RETRY_SECONDS = 3.0
|
||||||
@@ -140,6 +154,12 @@ class BaseDataProvider(ABC):
|
|||||||
|
|
||||||
# ── Public API ─────────────────────────────────────────────────
|
# ── Public API ─────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def _live_outputsize(self, timeframe: Timeframe, lookback_days: int) -> int:
|
||||||
|
"""Bars requested for one live fetch: the lookback in bars, floored
|
||||||
|
at MIN_LIVE_BARS (indicator seeding) and capped at the request cap."""
|
||||||
|
wanted = int(lookback_days * self.BARS_PER_DAY[timeframe]) + 20
|
||||||
|
return max(self.MIN_LIVE_BARS, min(wanted, self.MAX_BARS_PER_REQUEST))
|
||||||
|
|
||||||
def fetch_ohlcv(
|
def fetch_ohlcv(
|
||||||
self,
|
self,
|
||||||
symbol: str = XAUUSD,
|
symbol: str = XAUUSD,
|
||||||
@@ -151,11 +171,7 @@ class BaseDataProvider(ABC):
|
|||||||
lookback_days = self.config.data_lookback_days
|
lookback_days = self.config.data_lookback_days
|
||||||
|
|
||||||
cache_key = f"{symbol}_{timeframe.value}"
|
cache_key = f"{symbol}_{timeframe.value}"
|
||||||
outputsize = max(
|
outputsize = self._live_outputsize(timeframe, lookback_days)
|
||||||
300,
|
|
||||||
min(int(lookback_days * self.BARS_PER_DAY[timeframe]) + 20,
|
|
||||||
self.MAX_BARS_PER_REQUEST),
|
|
||||||
)
|
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
now = time.time()
|
now = time.time()
|
||||||
|
|||||||
+31
-1
@@ -4023,6 +4023,35 @@ def _():
|
|||||||
assert TiingoDataProvider.CANDLE_SECONDS is BaseDataProvider.CANDLE_SECONDS
|
assert TiingoDataProvider.CANDLE_SECONDS is BaseDataProvider.CANDLE_SECONDS
|
||||||
|
|
||||||
|
|
||||||
|
@test("providers: live fetch depth floors every TF at MIN_LIVE_BARS (H4/D1 seeding parity)")
|
||||||
|
def _():
|
||||||
|
import providers.base as pbase
|
||||||
|
|
||||||
|
class _Depth(pbase.BaseDataProvider):
|
||||||
|
PROVIDER = "depth16"
|
||||||
|
|
||||||
|
def _fetch_raw(self, symbol, timeframe, outputsize,
|
||||||
|
start=None, end=None):
|
||||||
|
raise RuntimeError("must not fetch")
|
||||||
|
|
||||||
|
def get_pair_info(self, symbol):
|
||||||
|
return PAIR_INFO[XAUUSD]
|
||||||
|
|
||||||
|
p = _Depth(PlatformConfig()) # data_lookback_days = 60
|
||||||
|
assert pbase.BaseDataProvider.MIN_LIVE_BARS == 1000
|
||||||
|
depth = {tf: p._live_outputsize(tf, 60) for tf in
|
||||||
|
(Timeframe.M5, Timeframe.M15, Timeframe.H1, Timeframe.H4, Timeframe.D1)}
|
||||||
|
# unchanged where the lookback already exceeds the floor ...
|
||||||
|
assert depth[Timeframe.M5] == 5000 and depth[Timeframe.M15] == 5000, depth
|
||||||
|
assert depth[Timeframe.H1] == 1460, depth
|
||||||
|
# ... raised where it did not (were 380 and 300 bars)
|
||||||
|
assert depth[Timeframe.H4] == 1000 and depth[Timeframe.D1] == 1000, depth
|
||||||
|
assert all(v <= pbase.BaseDataProvider.MAX_BARS_PER_REQUEST for v in depth.values())
|
||||||
|
assert p._live_outputsize(Timeframe.M5, 1) == 1000, "floor applies to short lookbacks"
|
||||||
|
# the seed bar's weight in the 200-bar trend EMA at the floor
|
||||||
|
assert (1 - 2 / 201) ** pbase.BaseDataProvider.MIN_LIVE_BARS < 1e-4
|
||||||
|
|
||||||
|
|
||||||
@test("providers: fetch_range paginates, dedupes, and trims to the window")
|
@test("providers: fetch_range paginates, dedupes, and trims to the window")
|
||||||
def _():
|
def _():
|
||||||
class FakeProvider(BaseDataProvider):
|
class FakeProvider(BaseDataProvider):
|
||||||
@@ -5700,7 +5729,8 @@ def _():
|
|||||||
with patch("providers.base._is_forex_weekend", lambda now: False):
|
with patch("providers.base._is_forex_weekend", lambda now: False):
|
||||||
p = _m21_stub_provider()
|
p = _m21_stub_provider()
|
||||||
p.fetch_ohlcv(XAUUSD, Timeframe.H1, lookback_days=1)
|
p.fetch_ohlcv(XAUUSD, Timeframe.H1, lookback_days=1)
|
||||||
assert len(p.calls) == 1 and p.calls[0] == 300
|
# 1 day of H1 = 44 bars, floored at MIN_LIVE_BARS (2026-09-10).
|
||||||
|
assert len(p.calls) == 1 and p.calls[0] == 1000
|
||||||
p.fetch_ohlcv(XAUUSD, Timeframe.H1, lookback_days=60)
|
p.fetch_ohlcv(XAUUSD, Timeframe.H1, lookback_days=60)
|
||||||
assert len(p.calls) == 2, \
|
assert len(p.calls) == 2, \
|
||||||
"a longer-lookback call within TTL must refetch, not serve the " \
|
"a longer-lookback call within TTL must refetch, not serve the " \
|
||||||
|
|||||||
Reference in New Issue
Block a user