Remove yfinance/XAU/XAG fallback — focus on currency pairs only
- data_provider: strip yfinance imports, _YF_FALLBACK_MAP, _YF_INTERVALS, _YF_SESSION, _fetch_yfinance_fallback, _resample_h1_to_h4. Twelve Data is the sole data source. Raises DataUnavailableError if a pair isn't on the configured plan. - config: remove XAU/USD and XAG/USD from SYMBOL_DISPLAY and PIP_VALUES. PIP_VALUES is now empty (defaults: 0.01 JPY pairs, 0.0001 forex). - signal_aggregator: clear _PER_SYMBOL_TFS overrides (was XAU/XAG-only). - telegram_bot: replace XAUUSD with AUDUSD in SCAN_PAIRS. Auto-load .env via python-dotenv so TELEGRAM_BOT_TOKEN/GEMINI_API_KEY are picked up. - requirements: drop yfinance and curl_cffi dependencies. End-to-end pipeline test passes: data freshness, news engine (Gemini live), signal aggregator, and the /signal+/scan code paths in telegram_bot. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Harwinder
+4
-151
@@ -1,9 +1,9 @@
|
||||
"""
|
||||
Forex Signal Intelligence Platform - Data Provider
|
||||
====================================================
|
||||
Retrieves and validates market data from Twelve Data API.
|
||||
Raises DataUnavailableError when the API is unreachable or credits are exhausted.
|
||||
No synthetic/simulated data fallback.
|
||||
Retrieves and validates currency-pair OHLCV from the Twelve Data API.
|
||||
Raises DataUnavailableError when the API is unreachable or credits exhausted.
|
||||
No fallback data source — forex pairs only.
|
||||
"""
|
||||
|
||||
import logging
|
||||
@@ -17,46 +17,11 @@ from typing import Dict, List, Optional
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
try:
|
||||
import yfinance as yf
|
||||
_YF_AVAILABLE = True
|
||||
except ImportError:
|
||||
yf = None
|
||||
_YF_AVAILABLE = False
|
||||
|
||||
# Insecure HTTPS session for yfinance — disables SSL verification.
|
||||
# Needed on servers behind a TLS-inspecting proxy that re-signs certs with
|
||||
# an untrusted root. yfinance requires a curl_cffi session, not requests.
|
||||
# Security trade-off: man-in-the-middle attacks on yfinance traffic are
|
||||
# possible. Acceptable here because the data is public market quotes.
|
||||
try:
|
||||
from curl_cffi import requests as _curl_cffi_requests
|
||||
_YF_SESSION = _curl_cffi_requests.Session(impersonate="chrome", verify=False)
|
||||
except ImportError:
|
||||
_YF_SESSION = None
|
||||
|
||||
from config import PlatformConfig, Timeframe, SYMBOL_DISPLAY, PIP_VALUES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_no_data_warned: set = set() # suppress duplicate "no data" warnings per symbol
|
||||
|
||||
# Symbols that Twelve Data free tier does not support — route to Yahoo Finance.
|
||||
# Gold/silver futures are the most reliable Yahoo symbols (full OHLCV history).
|
||||
_YF_FALLBACK_MAP = {
|
||||
"XAUUSD=X": "GC=F",
|
||||
"XAGUSD=X": "SI=F",
|
||||
}
|
||||
|
||||
_YF_INTERVALS = {
|
||||
Timeframe.M1: "1m",
|
||||
Timeframe.M5: "5m",
|
||||
Timeframe.M15: "15m",
|
||||
Timeframe.M30: "30m",
|
||||
Timeframe.H1: "1h",
|
||||
Timeframe.D1: "1d",
|
||||
Timeframe.W1: "1wk",
|
||||
}
|
||||
|
||||
|
||||
class DataIntegrityError(Exception):
|
||||
"""Raised when data fails validation checks."""
|
||||
@@ -168,35 +133,6 @@ class ForexDataProvider:
|
||||
if cached is not None and not cached.empty:
|
||||
return cached
|
||||
|
||||
# Symbols not on Twelve Data (gold, silver) always use yfinance.
|
||||
# D1 is skipped (yfinance daily for futures is unreliable).
|
||||
# H4 is resampled from H1 since yfinance has no native 4h candle.
|
||||
if symbol in _YF_FALLBACK_MAP:
|
||||
if timeframe == Timeframe.D1:
|
||||
raise DataUnavailableError(
|
||||
f"Daily chart skipped for {symbol} — gold/silver use M15/H1/H4 only."
|
||||
)
|
||||
if timeframe == Timeframe.H4:
|
||||
h1_df = self._fetch_yfinance_fallback(symbol, Timeframe.H1)
|
||||
if h1_df is None:
|
||||
raise DataUnavailableError(
|
||||
f"Yahoo Finance unavailable for {symbol} (H4 resample source H1)."
|
||||
)
|
||||
df = self._resample_h1_to_h4(h1_df)
|
||||
df = self._validate_and_clean(df, symbol, timeframe)
|
||||
self._cache[cache_key] = df
|
||||
self._cache_timestamps[cache_key] = now
|
||||
return df
|
||||
yf_df = self._fetch_yfinance_fallback(symbol, timeframe)
|
||||
if yf_df is None:
|
||||
raise DataUnavailableError(
|
||||
f"Yahoo Finance unavailable for {symbol} ({timeframe.value})."
|
||||
)
|
||||
df = self._validate_and_clean(yf_df, symbol, timeframe)
|
||||
self._cache[cache_key] = df
|
||||
self._cache_timestamps[cache_key] = now
|
||||
return df
|
||||
|
||||
df = self._fetch_twelve_data(symbol, timeframe, lookback_days)
|
||||
df = self._validate_and_clean(df, symbol, timeframe)
|
||||
|
||||
@@ -221,9 +157,7 @@ class ForexDataProvider:
|
||||
if lookback_days is None:
|
||||
lookback_days = self.config.data_lookback_days
|
||||
|
||||
# XAU/XAG aren't on Twelve Data — exclude them from the batch call,
|
||||
# they'll be fetched individually via yfinance on demand.
|
||||
td_symbols = [s for s in symbols if s not in _YF_FALLBACK_MAP]
|
||||
td_symbols = list(symbols)
|
||||
if not td_symbols:
|
||||
return
|
||||
|
||||
@@ -288,17 +222,6 @@ class ForexDataProvider:
|
||||
}
|
||||
return fallback_spreads.get(symbol, 0.00020)
|
||||
|
||||
@staticmethod
|
||||
def _resample_h1_to_h4(h1_df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Resample 1-hour OHLCV to 4-hour bars (yfinance has no native 4h)."""
|
||||
return h1_df.resample("4h", label="right", closed="right").agg({
|
||||
"Open": "first",
|
||||
"High": "max",
|
||||
"Low": "min",
|
||||
"Close": "last",
|
||||
"Volume": "sum",
|
||||
}).dropna()
|
||||
|
||||
def get_pair_info(self, symbol: str) -> Dict:
|
||||
"""Return display name and pip value for a pair."""
|
||||
display = SYMBOL_DISPLAY.get(symbol, symbol)
|
||||
@@ -314,60 +237,6 @@ class ForexDataProvider:
|
||||
|
||||
# ── Private Methods ────────────────────────────────────────────
|
||||
|
||||
def _fetch_yfinance_fallback(
|
||||
self, symbol: str, timeframe: Timeframe
|
||||
) -> Optional[pd.DataFrame]:
|
||||
"""
|
||||
Fetch OHLCV from Yahoo Finance for symbols Twelve Data doesn't cover
|
||||
on the free tier (XAUUSD, XAGUSD). Returns None if not mapped or fails.
|
||||
"""
|
||||
if not _YF_AVAILABLE:
|
||||
return None
|
||||
yf_symbol = _YF_FALLBACK_MAP.get(symbol)
|
||||
yf_interval = _YF_INTERVALS.get(timeframe)
|
||||
if yf_symbol is None or yf_interval is None:
|
||||
return None
|
||||
|
||||
# Yahoo intraday limits: 1m → last 7d, 15m/30m/1h → last 60d
|
||||
if yf_interval in ("1m", "2m"):
|
||||
period = "7d"
|
||||
elif yf_interval in ("5m", "15m", "30m", "60m", "1h"):
|
||||
period = "60d"
|
||||
else:
|
||||
period = "2y"
|
||||
|
||||
try:
|
||||
yf_kwargs = dict(
|
||||
interval=yf_interval, period=period,
|
||||
progress=False, auto_adjust=False, threads=False,
|
||||
)
|
||||
if _YF_SESSION is not None:
|
||||
yf_kwargs["session"] = _YF_SESSION
|
||||
df = yf.download(yf_symbol, **yf_kwargs)
|
||||
if df is None or df.empty:
|
||||
return None
|
||||
|
||||
# Single-symbol download can return MultiIndex columns in newer yfinance versions
|
||||
if isinstance(df.columns, pd.MultiIndex):
|
||||
df.columns = df.columns.get_level_values(0)
|
||||
|
||||
# Normalise to UTC tz-aware index
|
||||
if df.index.tz is None:
|
||||
df.index = df.index.tz_localize("UTC")
|
||||
else:
|
||||
df.index = df.index.tz_convert("UTC")
|
||||
df.index.name = "Datetime"
|
||||
|
||||
df = df[["Open", "High", "Low", "Close", "Volume"]].copy()
|
||||
logger.info(
|
||||
f"[yf] {yf_symbol} {yf_interval}: {len(df)} bars, "
|
||||
f"last close={df['Close'].iloc[-1]:.5f}"
|
||||
)
|
||||
return df
|
||||
except Exception as e:
|
||||
logger.warning(f"yfinance fallback failed for {symbol} {timeframe.value}: {e}")
|
||||
return None
|
||||
|
||||
def _throttle(self, credits: int = 1) -> None:
|
||||
"""
|
||||
Block until `credits` per-minute API credits are available.
|
||||
@@ -484,14 +353,6 @@ class ForexDataProvider:
|
||||
|
||||
values = sym_data.get("values")
|
||||
if not values:
|
||||
# Try yfinance fallback for unsupported symbols (XAU, XAG)
|
||||
yf_df = self._fetch_yfinance_fallback(symbol, timeframe)
|
||||
if yf_df is not None:
|
||||
yf_df = self._validate_and_clean(yf_df, symbol, timeframe)
|
||||
cache_key = f"{symbol}_{timeframe.value}"
|
||||
self._cache[cache_key] = yf_df
|
||||
self._cache_timestamps[cache_key] = now
|
||||
continue
|
||||
if td_sym not in _no_data_warned:
|
||||
logger.warning(f"No data from Twelve Data for {td_sym} — symbol may require a paid plan.")
|
||||
_no_data_warned.add(td_sym)
|
||||
@@ -527,13 +388,8 @@ class ForexDataProvider:
|
||||
"""
|
||||
Fetch OHLCV from Twelve Data REST API.
|
||||
Raises DataUnavailableError when data cannot be retrieved.
|
||||
Falls back to yfinance for symbols not on the Twelve Data free tier.
|
||||
"""
|
||||
if not self._api_key:
|
||||
# No Twelve Data key — try yfinance fallback for mapped symbols
|
||||
yf_df = self._fetch_yfinance_fallback(symbol, timeframe)
|
||||
if yf_df is not None:
|
||||
return yf_df
|
||||
raise DataUnavailableError(
|
||||
"No Twelve Data API key — set the TWELVE_DATA_API_KEY environment variable."
|
||||
)
|
||||
@@ -571,9 +427,6 @@ class ForexDataProvider:
|
||||
|
||||
values = data.get("values")
|
||||
if not values:
|
||||
yf_df = self._fetch_yfinance_fallback(symbol, timeframe)
|
||||
if yf_df is not None:
|
||||
return yf_df
|
||||
raise DataUnavailableError(
|
||||
f"No data returned for {td_symbol} — symbol may require a paid plan."
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user