Swap raw requests for Twelve Data Python SDK
data_provider now uses TDClient.time_series().as_pandas() instead of hand-rolled REST calls. Throttle, validation, and parquet caching are unchanged. Adds twelvedata>=1.2.0 to requirements.txt. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
+37
-46
@@ -1,10 +1,10 @@
|
||||
"""
|
||||
ForexBot2 — Data Provider (Twelve Data, USD/JPY only)
|
||||
=======================================================
|
||||
Fetches and validates OHLCV from Twelve Data. No synthetic fallback —
|
||||
if the API fails the caller learns about it. Adds a parquet on-disk
|
||||
cache so the backtest worker pool reuses one download instead of
|
||||
hammering Twelve Data 27 times during a parameter sweep.
|
||||
ForexBot2 — Data Provider (Twelve Data SDK, USD/JPY only)
|
||||
==========================================================
|
||||
Fetches and validates OHLCV via the official `twelvedata` Python SDK.
|
||||
No synthetic fallback — if the API fails the caller learns about it.
|
||||
Adds a parquet on-disk cache so the backtest worker pool reuses one
|
||||
download instead of hammering Twelve Data during a parameter sweep.
|
||||
"""
|
||||
|
||||
import logging
|
||||
@@ -14,7 +14,7 @@ from collections import deque
|
||||
from typing import Dict, Optional
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
from twelvedata import TDClient
|
||||
|
||||
from config import PlatformConfig, Timeframe, USDJPY, DISPLAY, PIP_VALUE
|
||||
|
||||
@@ -27,9 +27,7 @@ class DataIntegrityError(Exception):
|
||||
|
||||
|
||||
class ForexDataProvider:
|
||||
"""Twelve Data REST client + on-disk parquet cache."""
|
||||
|
||||
TD_BASE_URL = "https://api.twelvedata.com"
|
||||
"""Twelve Data SDK client + on-disk parquet cache."""
|
||||
|
||||
TD_INTERVALS = {
|
||||
Timeframe.M1: "1min",
|
||||
@@ -66,10 +64,9 @@ class ForexDataProvider:
|
||||
self._cache: Dict[str, pd.DataFrame] = {}
|
||||
self._cache_timestamps: Dict[str, float] = {}
|
||||
self._request_times: deque = deque()
|
||||
self._session = requests.Session()
|
||||
self._session.headers.update({
|
||||
"User-Agent": "Mozilla/5.0 (compatible; ForexIntel/1.0)"
|
||||
})
|
||||
self._td: Optional[TDClient] = (
|
||||
TDClient(apikey=self._api_key) if self._api_key else None
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _to_td_symbol(symbol: str) -> str:
|
||||
@@ -230,7 +227,7 @@ class ForexDataProvider:
|
||||
start_date: Optional[str] = None,
|
||||
end_date: Optional[str] = None,
|
||||
) -> pd.DataFrame:
|
||||
if not self._api_key:
|
||||
if self._td is None:
|
||||
raise DataIntegrityError(
|
||||
"No Twelve Data API key. Set TWELVE_DATA_API_KEY env var "
|
||||
"or config.twelve_data_api_key."
|
||||
@@ -241,50 +238,44 @@ class ForexDataProvider:
|
||||
|
||||
self._throttle(credits=1)
|
||||
|
||||
params = {
|
||||
kwargs = {
|
||||
"symbol": td_symbol,
|
||||
"interval": interval,
|
||||
"outputsize": outputsize,
|
||||
"apikey": self._api_key,
|
||||
"outputsize": min(outputsize, 5000),
|
||||
"timezone": "UTC",
|
||||
"format": "json",
|
||||
}
|
||||
if start_date:
|
||||
params["start_date"] = start_date
|
||||
kwargs["start_date"] = start_date
|
||||
if end_date:
|
||||
params["end_date"] = end_date
|
||||
kwargs["end_date"] = end_date
|
||||
|
||||
resp = self._session.get(
|
||||
f"{self.TD_BASE_URL}/time_series",
|
||||
params=params,
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
if data.get("status") == "error":
|
||||
try:
|
||||
df = self._td.time_series(**kwargs).as_pandas()
|
||||
except Exception as e:
|
||||
raise DataIntegrityError(
|
||||
f"Twelve Data error for {td_symbol}: {data.get('message', 'unknown')}"
|
||||
)
|
||||
f"Twelve Data fetch failed for {td_symbol}: {e}"
|
||||
) from e
|
||||
|
||||
values = data.get("values")
|
||||
if not values:
|
||||
if df is None or df.empty:
|
||||
raise DataIntegrityError(f"Empty response from Twelve Data for {td_symbol}")
|
||||
|
||||
records = [
|
||||
{
|
||||
"Datetime": pd.Timestamp(v["datetime"], tz="UTC"),
|
||||
"Open": float(v["open"]),
|
||||
"High": float(v["high"]),
|
||||
"Low": float(v["low"]),
|
||||
"Close": float(v["close"]),
|
||||
"Volume": float(v.get("volume") or 0),
|
||||
}
|
||||
for v in reversed(values)
|
||||
]
|
||||
# SDK returns lowercase columns; engine expects capitalised.
|
||||
df = df.rename(columns={
|
||||
"open": "Open", "high": "High", "low": "Low",
|
||||
"close": "Close", "volume": "Volume",
|
||||
})
|
||||
for col in ("Open", "High", "Low", "Close", "Volume"):
|
||||
if col in df.columns:
|
||||
df[col] = pd.to_numeric(df[col], errors="coerce")
|
||||
if "Volume" not in df.columns:
|
||||
df["Volume"] = 0.0
|
||||
|
||||
df = pd.DataFrame(records).set_index("Datetime")
|
||||
# SDK returns newest-first; engine + indicators want oldest-first.
|
||||
df = df.sort_index()
|
||||
if df.index.tz is None:
|
||||
df.index = df.index.tz_localize("UTC")
|
||||
df.index.name = "Datetime"
|
||||
|
||||
logger.debug(f"Fetched {len(df)} bars for {td_symbol} @ {interval}")
|
||||
return df
|
||||
|
||||
|
||||
Reference in New Issue
Block a user