Provider abstraction: providers/ package with BaseDataProvider template

One class per provider (TwelveData, Oanda, Tiingo), all subclassing BaseDataProvider which owns cache, validation/cleaning, freshness + boundary guarantees, fetch_range pagination, and streaming stubs — subclasses implement only _fetch_raw. config.json data_providers.active selects the live feed; data_provider.py / oanda_provider.py stay as back-compat shims. Module 16 provider-abstraction tests (253 total green). Claude-Session: https://claude.ai/code/session_01Mc7XKYyPySVXiUpcN82QEK
2026-07-28 20:09:00 -04:00
parent f760e9070a
commit 083a4e8d63
17 changed files with 1573 additions and 935 deletions
@@ -1561,7 +1561,6 @@ from telegram_bot import (
    _seconds_until_boundary, _check_level_breach,
    scan_keyboard, SCAN_BUTTONS, SCAN_LOOKUP,
    _update_level_cache, _last_prices, _level_cache, _level_cache_ts,
-    _WS_TO_PAIR,
 )


@@ -1836,12 +1835,16 @@ def _():
    _level_cache_ts.clear()


-@test("_WS_TO_PAIR mapping covers every scanned pair")
+@test("streaming symbol mapping covers every scanned pair")
 def _():
+    # iter_ticks maps the provider's WS symbols (display names) back to
+    # canonical pair symbols — the mapping must be invertible for every
+    # scanned pair or ticks would be silently dropped.
    from telegram_bot import SCAN_PAIRS
+    ws_to_pair = {PAIR_INFO[p]["display"]: p for p in SCAN_PAIRS}
    for p in SCAN_PAIRS:
-        assert _WS_TO_PAIR[PAIR_INFO[p]["display"]] == p
-    assert _WS_TO_PAIR["XAU/USD"] == XAUUSD
+        assert ws_to_pair[PAIR_INFO[p]["display"]] == p
+    assert ws_to_pair["XAU/USD"] == XAUUSD


 # ══════════════════════════════════════════════════════════════════════
@@ -2290,7 +2293,7 @@ def _():
        stale = bars_ending(last_closed - pd.Timedelta(minutes=15))
        fresh = bars_ending(last_closed)
        key = "XAU/USD_15min"
-        with patch("data_provider._is_forex_weekend", return_value=False):
+        with patch("providers.base._is_forex_weekend", return_value=False):
            # (a) snapshot cached mid-candle 5 min ago, missing the
            # just-closed bar: TTL says fresh, boundary check must refetch.
            provider._cache[key] = stale
@@ -2324,7 +2327,7 @@ def _():
        last_closed = started - pd.Timedelta(minutes=15)
        lagging = bars_ending(last_closed - pd.Timedelta(minutes=15))
        complete = bars_ending(last_closed)
-        with patch("data_provider._is_forex_weekend", return_value=False), \
+        with patch("providers.base._is_forex_weekend", return_value=False), \
             patch.object(ForexDataProvider, "FRESH_FETCH_RETRY_SECONDS", 0.0):
            # (a) lagging first response → one retry returns the closed bar.
            with patch.object(provider, "_fetch_twelve_data",
@@ -3963,6 +3966,331 @@ def _():
    daemon._heartbeat_once()   # must not raise


+# ══════════════════════════════════════════════════════════════════════
+#  MODULE 16: Provider abstraction (providers/ package)
+# ══════════════════════════════════════════════════════════════════════
+
+print("\n═══ Module 16: Provider abstraction ═══")
+
+from providers import (
+    BaseDataProvider, PROVIDER_REGISTRY, TwelveDataProvider,
+    make_data_provider,
+)
+from providers.oanda import OandaDataProvider as _POanda
+from providers.tiingo import TiingoDataError, TiingoDataProvider
+
+
+@test("providers: registry holds all providers, active default builds TD")
+def _():
+    assert PROVIDER_REGISTRY["twelve_data"] is TwelveDataProvider
+    assert PROVIDER_REGISTRY["oanda"] is _POanda
+    assert PROVIDER_REGISTRY["tiingo"] is TiingoDataProvider
+    dp = make_data_provider(PlatformConfig())
+    assert isinstance(dp, TwelveDataProvider)
+    assert isinstance(dp, BaseDataProvider)
+
+
+@test("providers: single knob — data_providers.active selects the class")
+def _():
+    from app_config import AppConfig
+    for name, cls in PROVIDER_REGISTRY.items():
+        fake = AppConfig()
+        fake.active_provider = name
+        fake.provider_settings = {name: {"api_key": ""}}
+        with patch("providers.get_app_config", return_value=fake):
+            dp = make_data_provider(PlatformConfig())
+        assert type(dp) is cls, f"active={name} built {type(dp).__name__}"
+    # An unknown name must fail loudly, not fall back silently.
+    from app_config import AppConfigError
+    fake = AppConfig()
+    fake.active_provider = "nonsense"
+    with patch("providers.get_app_config", return_value=fake):
+        try:
+            make_data_provider(PlatformConfig())
+            assert False, "unknown provider must raise"
+        except AppConfigError as e:
+            assert "nonsense" in str(e)
+
+
+@test("providers: shared behavior is defined ONCE on the base class")
+def _():
+    # The pre-refactor bug this guards against: Oanda's provider was a
+    # copy-paste of the TD one, so shared-logic fixes had to be made
+    # twice. Every shared method must now BE the base implementation.
+    for meth in ("fetch_ohlcv", "fetch_range", "_validate_and_clean",
+                 "_is_data_fresh", "_has_latest_closed_bar",
+                 "_drop_forming_candle", "clear_cache", "get_pair_info"):
+        for cls in PROVIDER_REGISTRY.values():
+            assert getattr(cls, meth) is getattr(BaseDataProvider, meth), \
+                f"{cls.__name__} overrides shared method {meth}"
+    assert TwelveDataProvider.CANDLE_SECONDS is BaseDataProvider.CANDLE_SECONDS
+    assert _POanda.BARS_PER_DAY is BaseDataProvider.BARS_PER_DAY
+    assert TiingoDataProvider.CANDLE_SECONDS is BaseDataProvider.CANDLE_SECONDS
+
+
+@test("providers: back-compat shims re-export the real classes")
+def _():
+    import data_provider as dp_shim
+    import oanda_provider as oa_shim
+    assert dp_shim.ForexDataProvider is TwelveDataProvider
+    assert dp_shim.make_data_provider is make_data_provider
+    assert oa_shim.OandaDataProvider is _POanda
+    from providers.base import DataIntegrityError as base_die
+    assert dp_shim.DataIntegrityError is base_die
+    # Provider-specific errors are catchable as the shared type.
+    assert issubclass(oa_shim.OandaDataError, base_die)
+
+
+@test("providers: fetch_range paginates, dedupes, and trims to the window")
+def _():
+    class FakeProvider(BaseDataProvider):
+        name = "fake"
+
+        def __init__(self, config):
+            super().__init__(config)
+            idx = pd.date_range("2026-06-01", "2026-06-30 23:00",
+                                freq="1h", tz="UTC")
+            v = np.linspace(3000.0, 3100.0, len(idx))
+            self.series = pd.DataFrame(
+                {"Open": v, "High": v + 1, "Low": v - 1,
+                 "Close": v, "Volume": 100.0}, index=idx)
+            self.series.index.name = "Datetime"
+            self.calls = 0
+
+        def _fetch_raw(self, symbol, timeframe, outputsize,
+                       start=None, end=None):
+            self.calls += 1
+            df = self.series
+            if start is not None:
+                df = df[df.index >= pd.Timestamp(start)]
+            if end is not None:
+                df = df[df.index <= pd.Timestamp(end)]
+            # Overlap by a bar to prove concat dedupes.
+            return df.tail(min(len(df), 200))
+
+        # Keep the test date-stable: nothing here is "forming".
+        def _drop_forming_candle(self, df, timeframe):
+            return df
+
+    fp = FakeProvider(PlatformConfig())
+    start = datetime(2026, 6, 5, tzinfo=timezone.utc)
+    end = datetime(2026, 6, 25, tzinfo=timezone.utc)
+    df = fp.fetch_range("XAU/USD", Timeframe.H1, start, end)
+
+    assert fp.calls > 1, "a 20-day H1 window must take multiple chunks"
+    assert df.index.is_monotonic_increasing
+    assert not df.index.duplicated().any()
+    assert df.index[0] >= pd.Timestamp(start)
+    assert df.index[-1] <= pd.Timestamp(end)
+    expected = fp.series[(fp.series.index >= pd.Timestamp(start))
+                         & (fp.series.index <= pd.Timestamp(end))]
+    assert len(df) == len(expected), \
+        f"window must be gapless: {len(df)} vs {len(expected)}"
+
+
+@test("providers: streaming capability flags and base refusal")
+def _():
+    assert TwelveDataProvider.supports_streaming is True
+    assert _POanda.supports_streaming is False
+    assert TiingoDataProvider.supports_streaming is False
+
+    class Minimal(BaseDataProvider):
+        name = "minimal"
+
+        def _fetch_raw(self, symbol, timeframe, outputsize,
+                       start=None, end=None):
+            raise NotImplementedError
+
+    m = Minimal(PlatformConfig())
+    assert m.supports_streaming is False
+    try:
+        m.iter_ticks(["XAU/USD"])
+        assert False, "base iter_ticks must refuse"
+    except NotImplementedError:
+        pass
+
+
+@test("providers: TD WS price-message parsing")
+def _():
+    parse = TwelveDataProvider._parse_ws_price
+    assert parse('{"event":"price","symbol":"XAU/USD","price":"3345.5"}') \
+        == ("XAU/USD", 3345.5)
+    assert parse('{"event":"subscribe-status","fails":[]}') is None
+    assert parse('{"event":"heartbeat"}') is None
+    assert parse('not json{{') is None
+    assert parse('{"event":"price","symbol":"XAU/USD"}') is None  # no price
+    assert parse('{"event":"price","symbol":"XAU/USD","price":"abc"}') is None
+
+
+@test("providers: Oanda date-window fetch omits count and sends from/to")
+def _():
+    captured = {}
+
+    def mock_get(self, url, params=None, timeout=None):
+        captured["params"] = params
+        return type("R", (), {
+            "status_code": 200,
+            "json": lambda self: {"candles": [{
+                "complete": True,
+                "time": "2026-06-02T10:00:00.000000000Z",
+                "mid": {"o": "3000", "h": "3001", "l": "2999", "c": "3000.5"},
+                "bid": {"c": "3000.2"}, "ask": {"c": "3000.8"},
+                "volume": 10,
+            }]},
+        })()
+
+    provider = _POanda.__new__(_POanda)
+    provider.config = PlatformConfig()
+    provider._base_url = "https://test"
+    provider._spread_cache = {}
+    provider._session = type("S", (), {"get": mock_get})()
+
+    start = datetime(2026, 6, 1, tzinfo=timezone.utc)
+    end = datetime(2026, 6, 3, tzinfo=timezone.utc)
+    df = provider._fetch_raw("XAU/USD", Timeframe.H1, 5000, start, end)
+    assert "count" not in captured["params"], \
+        "v20 rejects count combined with from/to"
+    assert captured["params"]["from"].startswith("2026-06-01")
+    assert captured["params"]["to"].startswith("2026-06-03")
+    assert len(df) == 1
+
+    # Live path (no window) keeps sending count only.
+    provider._fetch_raw("XAU/USD", Timeframe.H1, 300)
+    assert captured["params"]["count"] == 300
+    assert "from" not in captured["params"]
+
+
+def _tiingo_provider(response):
+    """TiingoDataProvider with a canned session response (no network)."""
+    provider = TiingoDataProvider.__new__(TiingoDataProvider)
+    provider.config = PlatformConfig()
+    provider._api_key = "test-key"
+    provider._spread_cache = {}
+    captured = {}
+
+    def mock_get(self, url, params=None, timeout=None):
+        captured["url"] = url
+        captured["params"] = params
+        return response
+
+    provider._session = type("S", (), {"get": mock_get})()
+    return provider, captured
+
+
+def _tiingo_resp(status_code, payload):
+    return type("R", (), {
+        "status_code": status_code,
+        "json": lambda self: payload,
+        "text": json.dumps(payload) if not isinstance(payload, str) else payload,
+    })()
+
+
+@test("providers: Tiingo parses FX JSON into the normalized frame")
+def _():
+    payload = [
+        {"date": "2026-07-20T11:00:00.000Z", "ticker": "xauusd",
+         "open": 2403.0, "high": 2408.0, "low": 2401.0, "close": 2406.0},
+        # Deliberately out of order — provider must sort oldest-first.
+        {"date": "2026-07-20T10:00:00.000Z", "ticker": "xauusd",
+         "open": 2400.0, "high": 2405.0, "low": 2398.0, "close": 2403.0},
+    ]
+    provider, captured = _tiingo_provider(_tiingo_resp(200, payload))
+    df = provider._fetch_raw("XAU/USD", Timeframe.H1, 300)
+
+    assert list(df.columns) == ["Open", "High", "Low", "Close", "Volume"]
+    assert df.index.name == "Datetime"
+    assert df.index.tz is not None, "Index must be tz-aware UTC"
+    assert df.index.is_monotonic_increasing, "must be oldest-first"
+    assert df.iloc[0]["Open"] == 2400.0 and df.iloc[1]["Close"] == 2406.0
+    assert (df["Volume"] == 0.0).all(), "FX endpoint has no volume"
+    assert captured["url"].endswith("/tiingo/fx/xauusd/prices")
+    assert captured["params"]["resampleFreq"] == "1hour"
+    assert "startDate" in captured["params"], \
+        "live fetch must derive a start window from the bar count"
+    assert "token" not in captured["params"], \
+        "key must travel in the auth header, never the URL"
+
+
+@test("providers: Tiingo date-window fetch sends startDate AND endDate")
+def _():
+    payload = [{"date": "2026-06-02T10:00:00.000Z",
+                "open": 1, "high": 2, "low": 0, "close": 1}]
+    provider, captured = _tiingo_provider(_tiingo_resp(200, payload))
+    start = datetime(2026, 6, 1, tzinfo=timezone.utc)
+    end = datetime(2026, 6, 3, tzinfo=timezone.utc)
+    provider._fetch_raw("XAU/USD", Timeframe.H1, 5000, start, end)
+    assert captured["params"]["startDate"].startswith("2026-06-01")
+    assert captured["params"]["endDate"].startswith("2026-06-03")
+
+
+@test("providers: Tiingo fails LOUDLY on every bad-response shape")
+def _():
+    cases = [
+        (_tiingo_resp(429, "quota"), "rate limit"),
+        (_tiingo_resp(401, "denied"), "auth failed"),
+        (_tiingo_resp(500, "boom"), "HTTP 500"),
+        (_tiingo_resp(200, []), "Empty candle response"),
+        (_tiingo_resp(200, [{"open": 1}]), "missing 'date'"),
+        (_tiingo_resp(200, [{"date": "2026-07-20T10:00:00Z",
+                             "close": "n/a"}]), "no numeric closes"),
+    ]
+    for resp, needle in cases:
+        provider, _cap = _tiingo_provider(resp)
+        try:
+            provider._fetch_raw("XAU/USD", Timeframe.H1, 300)
+            assert False, f"must raise for: {needle}"
+        except TiingoDataError as e:
+            assert needle.lower() in str(e).lower(), \
+                f"error must name the cause ({needle}): {e}"
+
+    # Missing key and unsupported timeframe are named errors too.
+    provider, _cap = _tiingo_provider(_tiingo_resp(200, []))
+    provider._api_key = None
+    try:
+        provider._fetch_raw("XAU/USD", Timeframe.H1, 300)
+        assert False, "missing key must raise"
+    except TiingoDataError as e:
+        assert "TIINGO_API_KEY" in str(e)
+    provider._api_key = "k"
+    try:
+        provider._fetch_raw("XAU/USD", Timeframe.W1, 300)
+        assert False, "unsupported TF must raise"
+    except TiingoDataError as e:
+        assert "Unsupported timeframe" in str(e)
+    # And everything above is catchable as the shared base error.
+    assert issubclass(TiingoDataError, DataIntegrityError)
+
+
+@test("providers: per-provider env keys (TIINGO_API_KEY vs TWELVE_DATA_API_KEY)")
+def _():
+    from app_config import AppConfig, provider_api_key
+    app = AppConfig()
+    app.active_provider = "tiingo"
+    app.provider_settings = {"tiingo": {"api_key": "json-tiingo"},
+                             "twelve_data": {"api_key": "json-td"}}
+    with patch.dict(os.environ, {"TIINGO_API_KEY": "env-tiingo",
+                                 "TWELVE_DATA_API_KEY": "env-td"}):
+        assert provider_api_key(app) == "env-tiingo"
+        assert provider_api_key(app, provider="twelve_data") == "env-td"
+    with patch.dict(os.environ, {}, clear=False):
+        os.environ.pop("TIINGO_API_KEY", None)
+        assert provider_api_key(app) == "json-tiingo"
+        # The TD env var must never leak into another provider's key.
+        with patch.dict(os.environ, {"TWELVE_DATA_API_KEY": "env-td"}):
+            assert provider_api_key(app, provider="tiingo") == "json-tiingo"
+
+
+@test("providers: aggregator bar-close clock uses the base-class table")
+def _():
+    # signal_aggregator computes bar-CLOSE hours (session filter, SELL
+    # hour gate) from BaseDataProvider.CANDLE_SECONDS — a provider swap
+    # must never change the session clock.
+    import signal_aggregator as sa
+    assert sa.BaseDataProvider.CANDLE_SECONDS[Timeframe.M15] == 900
+    assert not hasattr(sa, "ForexDataProvider"), \
+        "aggregator must not depend on a concrete provider class"
+
+
 # ══════════════════════════════════════════════════════════════════════
 #  SUMMARY
 # ══════════════════════════════════════════════════════════════════════