From dcd98fffa2e31bdbb5e9f36aa9d0372a0950de01 Mon Sep 17 00:00:00 2001
From: Harwinder <hssekhon93@hotmail.com>
Date: Thu, 7 May 2026 18:38:38 -0400
Subject: [PATCH] Swap raw requests for Twelve Data Python SDK

data_provider now uses TDClient.time_series().as_pandas() instead of
hand-rolled REST calls. Throttle, validation, and parquet caching are
unchanged. Adds twelvedata>=1.2.0 to requirements.txt.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
---
 data_provider.py | 83 +++++++++++++++++++++---------------------------
 requirements.txt |  1 +
 2 files changed, 38 insertions(+), 46 deletions(-)

diff --git a/data_provider.py b/data_provider.py
index 07cfd40..5b4cc5e 100644
--- a/data_provider.py
+++ b/data_provider.py
@@ -1,10 +1,10 @@
 """
-ForexBot2 — Data Provider (Twelve Data, USD/JPY only)
-=======================================================
-Fetches and validates OHLCV from Twelve Data. No synthetic fallback —
-if the API fails the caller learns about it. Adds a parquet on-disk
-cache so the backtest worker pool reuses one download instead of
-hammering Twelve Data 27 times during a parameter sweep.
+ForexBot2 — Data Provider (Twelve Data SDK, USD/JPY only)
+==========================================================
+Fetches and validates OHLCV via the official `twelvedata` Python SDK.
+No synthetic fallback — if the API fails the caller learns about it.
+Adds a parquet on-disk cache so the backtest worker pool reuses one
+download instead of hammering Twelve Data during a parameter sweep.
 """
 
 import logging
@@ -14,7 +14,7 @@ from collections import deque
 from typing import Dict, Optional
 
 import pandas as pd
-import requests
+from twelvedata import TDClient
 
 from config import PlatformConfig, Timeframe, USDJPY, DISPLAY, PIP_VALUE
 
@@ -27,9 +27,7 @@ class DataIntegrityError(Exception):
 
 
 class ForexDataProvider:
-    """Twelve Data REST client + on-disk parquet cache."""
-
-    TD_BASE_URL = "https://api.twelvedata.com"
+    """Twelve Data SDK client + on-disk parquet cache."""
 
     TD_INTERVALS = {
         Timeframe.M1:  "1min",
@@ -66,10 +64,9 @@ class ForexDataProvider:
         self._cache: Dict[str, pd.DataFrame] = {}
         self._cache_timestamps: Dict[str, float] = {}
         self._request_times: deque = deque()
-        self._session = requests.Session()
-        self._session.headers.update({
-            "User-Agent": "Mozilla/5.0 (compatible; ForexIntel/1.0)"
-        })
+        self._td: Optional[TDClient] = (
+            TDClient(apikey=self._api_key) if self._api_key else None
+        )
 
     @staticmethod
     def _to_td_symbol(symbol: str) -> str:
@@ -230,7 +227,7 @@ class ForexDataProvider:
         start_date: Optional[str] = None,
         end_date: Optional[str] = None,
     ) -> pd.DataFrame:
-        if not self._api_key:
+        if self._td is None:
             raise DataIntegrityError(
                 "No Twelve Data API key. Set TWELVE_DATA_API_KEY env var "
                 "or config.twelve_data_api_key."
@@ -241,50 +238,44 @@ class ForexDataProvider:
 
         self._throttle(credits=1)
 
-        params = {
+        kwargs = {
             "symbol": td_symbol,
             "interval": interval,
-            "outputsize": outputsize,
-            "apikey": self._api_key,
+            "outputsize": min(outputsize, 5000),
             "timezone": "UTC",
-            "format": "json",
         }
         if start_date:
-            params["start_date"] = start_date
+            kwargs["start_date"] = start_date
         if end_date:
-            params["end_date"] = end_date
+            kwargs["end_date"] = end_date
 
-        resp = self._session.get(
-            f"{self.TD_BASE_URL}/time_series",
-            params=params,
-            timeout=30,
-        )
-        resp.raise_for_status()
-        data = resp.json()
-
-        if data.get("status") == "error":
+        try:
+            df = self._td.time_series(**kwargs).as_pandas()
+        except Exception as e:
             raise DataIntegrityError(
-                f"Twelve Data error for {td_symbol}: {data.get('message', 'unknown')}"
-            )
+                f"Twelve Data fetch failed for {td_symbol}: {e}"
+            ) from e
 
-        values = data.get("values")
-        if not values:
+        if df is None or df.empty:
             raise DataIntegrityError(f"Empty response from Twelve Data for {td_symbol}")
 
-        records = [
-            {
-                "Datetime": pd.Timestamp(v["datetime"], tz="UTC"),
-                "Open":   float(v["open"]),
-                "High":   float(v["high"]),
-                "Low":    float(v["low"]),
-                "Close":  float(v["close"]),
-                "Volume": float(v.get("volume") or 0),
-            }
-            for v in reversed(values)
-        ]
+        # SDK returns lowercase columns; engine expects capitalised.
+        df = df.rename(columns={
+            "open": "Open", "high": "High", "low": "Low",
+            "close": "Close", "volume": "Volume",
+        })
+        for col in ("Open", "High", "Low", "Close", "Volume"):
+            if col in df.columns:
+                df[col] = pd.to_numeric(df[col], errors="coerce")
+        if "Volume" not in df.columns:
+            df["Volume"] = 0.0
 
-        df = pd.DataFrame(records).set_index("Datetime")
+        # SDK returns newest-first; engine + indicators want oldest-first.
+        df = df.sort_index()
+        if df.index.tz is None:
+            df.index = df.index.tz_localize("UTC")
         df.index.name = "Datetime"
+
         logger.debug(f"Fetched {len(df)} bars for {td_symbol} @ {interval}")
         return df
 
diff --git a/requirements.txt b/requirements.txt
index 45a3851..b86e51e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,3 +3,4 @@ pandas>=2.0
 requests>=2.28
 pyarrow>=14.0
 tqdm>=4.66
+twelvedata>=1.2.0