Provider abstraction: providers/ package with BaseDataProvider template
One class per provider (TwelveData, Oanda, Tiingo), all subclassing BaseDataProvider which owns cache, validation/cleaning, freshness + boundary guarantees, fetch_range pagination, and streaming stubs — subclasses implement only _fetch_raw. config.json data_providers.active selects the live feed; data_provider.py / oanda_provider.py stay as back-compat shims. Module 16 provider-abstraction tests (253 total green). Claude-Session: https://claude.ai/code/session_01Mc7XKYyPySVXiUpcN82QEK
This commit is contained in:
Harwinder Sekhon
@@ -0,0 +1,200 @@
|
||||
"""
|
||||
ForexBot2 — Tiingo data provider
|
||||
=================================
|
||||
OHLCV candles from Tiingo's FX endpoint
|
||||
(`/tiingo/fx/{ticker}/prices`, resampled mid-price bars, no volume).
|
||||
All shared behavior (cache, validation, boundary guarantees, range
|
||||
pagination) lives in `BaseDataProvider`; this module only maps
|
||||
symbols/resample frequencies, sizes the request window, and parses
|
||||
Tiingo's JSON into the normalized frame.
|
||||
|
||||
Every failure mode raises `TiingoDataError` (a `DataIntegrityError`)
|
||||
with a specific message — the scan loops log it and carry on; nothing
|
||||
fails silently. NOTE: this is the CANDLE feed only. Tiingo's free
|
||||
`/tiingo/fx/top` quote endpoint publishes indicative two-tier spreads
|
||||
and must not be used for spread display (removed 2026-07-23) —
|
||||
`get_spread` deliberately stays on the base-class default.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
from app_config import provider_api_key
|
||||
from config import PlatformConfig, Timeframe
|
||||
|
||||
from providers.base import BaseDataProvider, DataIntegrityError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TiingoDataError(DataIntegrityError):
|
||||
"""Raised when a Tiingo data fetch fails or returns invalid data."""
|
||||
|
||||
|
||||
class TiingoDataProvider(BaseDataProvider):
|
||||
"""Tiingo FX REST candle provider."""
|
||||
|
||||
name = "tiingo"
|
||||
DATA_ERROR = TiingoDataError
|
||||
|
||||
BASE_URL = "https://api.tiingo.com"
|
||||
|
||||
TIINGO_RESAMPLE = {
|
||||
Timeframe.M1: "1min",
|
||||
Timeframe.M5: "5min",
|
||||
Timeframe.M15: "15min",
|
||||
Timeframe.M30: "30min",
|
||||
Timeframe.H1: "1hour",
|
||||
Timeframe.H4: "4hour",
|
||||
Timeframe.D1: "1day",
|
||||
# W1 intentionally unmapped: the FX endpoint has no weekly
|
||||
# resample — requesting it raises loudly instead of guessing.
|
||||
}
|
||||
|
||||
TIINGO_TICKER = {
|
||||
"XAU/USD": "xauusd",
|
||||
}
|
||||
|
||||
# Window padding over the theoretical bar span: FX is closed ~2/7 of
|
||||
# the week (Fri 22:00 → Sun 22:00 UTC), so a calendar window must be
|
||||
# wider than `bars × candle_seconds` to yield the requested count.
|
||||
WINDOW_PAD_FACTOR = 1.6
|
||||
|
||||
def __init__(self, config: PlatformConfig, api_key: Optional[str] = None):
|
||||
super().__init__(config)
|
||||
# Credentials default to env TIINGO_API_KEY → config.json
|
||||
# data_providers.tiingo.api_key, so the provider registry can
|
||||
# build this class with config only.
|
||||
if api_key is None:
|
||||
api_key = provider_api_key(provider="tiingo")
|
||||
self._api_key = api_key or None
|
||||
if self._api_key is None:
|
||||
logger.warning(
|
||||
"Tiingo provider built WITHOUT an API key — every fetch "
|
||||
"will fail with a named error. Set the TIINGO_API_KEY env "
|
||||
"var or data_providers.tiingo.api_key in config.json.")
|
||||
self._session = requests.Session()
|
||||
self._session.headers.update({
|
||||
"Content-Type": "application/json",
|
||||
# Header auth keeps the key out of URLs (and out of logs).
|
||||
"Authorization": f"Token {api_key}" if api_key else "",
|
||||
})
|
||||
|
||||
def _to_tiingo_ticker(self, symbol: str) -> str:
|
||||
if symbol in self.TIINGO_TICKER:
|
||||
return self.TIINGO_TICKER[symbol]
|
||||
return symbol.replace("/", "").replace("=X", "").lower()
|
||||
|
||||
# ── Raw fetch ──────────────────────────────────────────────────
|
||||
|
||||
def _fetch_raw(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: Timeframe,
|
||||
outputsize: int,
|
||||
start: Optional[datetime] = None,
|
||||
end: Optional[datetime] = None,
|
||||
) -> pd.DataFrame:
|
||||
if self._api_key is None:
|
||||
raise TiingoDataError(
|
||||
"No Tiingo API key. Set the TIINGO_API_KEY env var or "
|
||||
"data_providers.tiingo.api_key in config.json.")
|
||||
|
||||
resample = self.TIINGO_RESAMPLE.get(timeframe)
|
||||
if resample is None:
|
||||
raise TiingoDataError(
|
||||
f"Unsupported timeframe {timeframe} for Tiingo "
|
||||
f"(no FX resample frequency)")
|
||||
|
||||
ticker = self._to_tiingo_ticker(symbol)
|
||||
|
||||
# Tiingo takes date windows, not bar counts: derive the start
|
||||
# from the requested count for live fetches.
|
||||
if start is None:
|
||||
span = self.CANDLE_SECONDS[timeframe] * (outputsize + 10)
|
||||
start_ts = (pd.Timestamp.now(tz="UTC")
|
||||
- pd.Timedelta(seconds=span * self.WINDOW_PAD_FACTOR))
|
||||
else:
|
||||
start_ts = pd.Timestamp(start)
|
||||
|
||||
params = {
|
||||
"resampleFreq": resample,
|
||||
"startDate": start_ts.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
}
|
||||
if end is not None:
|
||||
params["endDate"] = pd.Timestamp(end).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
|
||||
url = f"{self.BASE_URL}/tiingo/fx/{ticker}/prices"
|
||||
try:
|
||||
resp = self._session.get(url, params=params, timeout=30)
|
||||
except requests.RequestException as e:
|
||||
raise TiingoDataError(
|
||||
f"Tiingo candle fetch failed for {ticker}: {e}") from e
|
||||
|
||||
if resp.status_code == 429:
|
||||
raise TiingoDataError(
|
||||
f"Tiingo rate limit hit for {ticker} (HTTP 429) — "
|
||||
f"plan quota exhausted; scan will retry next boundary")
|
||||
if resp.status_code in (401, 403):
|
||||
raise TiingoDataError(
|
||||
f"Tiingo auth failed for {ticker} (HTTP {resp.status_code}) "
|
||||
f"— check TIINGO_API_KEY / config.json api_key")
|
||||
if resp.status_code != 200:
|
||||
raise TiingoDataError(
|
||||
f"Tiingo candle fetch failed for {ticker}: "
|
||||
f"HTTP {resp.status_code}: {resp.text[:300]}")
|
||||
|
||||
try:
|
||||
data = resp.json()
|
||||
except ValueError as e:
|
||||
raise TiingoDataError(
|
||||
f"Tiingo candle response for {ticker} is not valid "
|
||||
f"JSON: {e}") from e
|
||||
|
||||
if not isinstance(data, list) or not data:
|
||||
raise TiingoDataError(
|
||||
f"Empty candle response from Tiingo for {ticker} "
|
||||
f"(window {params['startDate']} → "
|
||||
f"{params.get('endDate', 'now')}; check ticker and plan "
|
||||
f"history limits)")
|
||||
|
||||
rows = []
|
||||
for item in data:
|
||||
if "date" not in item:
|
||||
raise TiingoDataError(
|
||||
f"Malformed Tiingo candle for {ticker}: missing "
|
||||
f"'date' in {str(item)[:120]}")
|
||||
rows.append({
|
||||
"Datetime": pd.Timestamp(item["date"]),
|
||||
"Open": item.get("open"),
|
||||
"High": item.get("high"),
|
||||
"Low": item.get("low"),
|
||||
"Close": item.get("close"),
|
||||
# FX endpoint has no volume; 0.0 keeps the engine contract.
|
||||
"Volume": 0.0,
|
||||
})
|
||||
|
||||
df = pd.DataFrame(rows)
|
||||
df = df.set_index("Datetime")
|
||||
df = df.sort_index()
|
||||
if df.index.tz is None:
|
||||
df.index = df.index.tz_localize("UTC")
|
||||
else:
|
||||
df.index = df.index.tz_convert("UTC")
|
||||
df.index.name = "Datetime"
|
||||
|
||||
for col in ("Open", "High", "Low", "Close"):
|
||||
df[col] = pd.to_numeric(df[col], errors="coerce")
|
||||
if df["Close"].isna().all():
|
||||
raise TiingoDataError(
|
||||
f"Tiingo response for {ticker} has no numeric closes — "
|
||||
f"feed format changed?")
|
||||
|
||||
logger.debug(
|
||||
"Fetched %d bars for %s @ %s from Tiingo",
|
||||
len(df), ticker, resample)
|
||||
return df
|
||||
Reference in New Issue
Block a user