Remove yfinance/XAU/XAG fallback — focus on currency pairs only

- data_provider: strip yfinance imports, _YF_FALLBACK_MAP, _YF_INTERVALS,
  _YF_SESSION, _fetch_yfinance_fallback, _resample_h1_to_h4. Twelve Data
  is the sole data source. Raises DataUnavailableError if a pair isn't
  on the configured plan.
- config: remove XAU/USD and XAG/USD from SYMBOL_DISPLAY and PIP_VALUES.
  PIP_VALUES is now empty (defaults: 0.01 JPY pairs, 0.0001 forex).
- signal_aggregator: clear _PER_SYMBOL_TFS overrides (was XAU/XAG-only).
- telegram_bot: replace XAUUSD with AUDUSD in SCAN_PAIRS. Auto-load .env
  via python-dotenv so TELEGRAM_BOT_TOKEN/GEMINI_API_KEY are picked up.
- requirements: drop yfinance and curl_cffi dependencies.

End-to-end pipeline test passes: data freshness, news engine (Gemini live),
signal aggregator, and the /signal+/scan code paths in telegram_bot.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

config.py

@@ -57,16 +57,12 @@ SYMBOL_DISPLAY = {
     "EURJPY=X": "EUR/JPY", "GBPJPY=X": "GBP/JPY",
     "AUDJPY=X": "AUD/JPY", "EURCHF=X": "EUR/CHF",
     "CADJPY=X": "CAD/JPY",
-    # Commodities
-    "XAUUSD=X": "XAU/USD", "XAGUSD=X": "XAG/USD",
 }
 
 # Pip values per symbol (used for trade setup risk/reward in pips)
-# Commodities and JPY pairs differ from the standard 0.0001 forex pip
+# Forex defaults: 0.01 for JPY pairs, 0.0001 for everything else.
-PIP_VALUES = {
+# Override here for any pair that doesn't follow that rule.
-    "XAUUSD=X": 0.01,   # Gold: 1 pip = $0.01
+PIP_VALUES: Dict[str, float] = {}
-    "XAGUSD=X": 0.001,  # Silver: 1 pip = $0.001
-}
 
 
 @dataclass

data_provider.py

@@ -1,9 +1,9 @@
 """
 Forex Signal Intelligence Platform - Data Provider
 ====================================================
-Retrieves and validates market data from Twelve Data API.
+Retrieves and validates currency-pair OHLCV from the Twelve Data API.
-Raises DataUnavailableError when the API is unreachable or credits are exhausted.
+Raises DataUnavailableError when the API is unreachable or credits exhausted.
-No synthetic/simulated data fallback.
+No fallback data source — forex pairs only.
 """
 
 import logging
@@ -17,46 +17,11 @@ from typing import Dict, List, Optional
 import pandas as pd
 import requests
 
-try:
-    import yfinance as yf
-    _YF_AVAILABLE = True
-except ImportError:
-    yf = None
-    _YF_AVAILABLE = False
-
-# Insecure HTTPS session for yfinance — disables SSL verification.
-# Needed on servers behind a TLS-inspecting proxy that re-signs certs with
-# an untrusted root. yfinance requires a curl_cffi session, not requests.
-# Security trade-off: man-in-the-middle attacks on yfinance traffic are
-# possible. Acceptable here because the data is public market quotes.
-try:
-    from curl_cffi import requests as _curl_cffi_requests
-    _YF_SESSION = _curl_cffi_requests.Session(impersonate="chrome", verify=False)
-except ImportError:
-    _YF_SESSION = None
-
 from config import PlatformConfig, Timeframe, SYMBOL_DISPLAY, PIP_VALUES
 
 logger = logging.getLogger(__name__)
 _no_data_warned: set = set()  # suppress duplicate "no data" warnings per symbol
 
-# Symbols that Twelve Data free tier does not support — route to Yahoo Finance.
-# Gold/silver futures are the most reliable Yahoo symbols (full OHLCV history).
-_YF_FALLBACK_MAP = {
-    "XAUUSD=X": "GC=F",
-    "XAGUSD=X": "SI=F",
-}
-
-_YF_INTERVALS = {
-    Timeframe.M1:  "1m",
-    Timeframe.M5:  "5m",
-    Timeframe.M15: "15m",
-    Timeframe.M30: "30m",
-    Timeframe.H1:  "1h",
-    Timeframe.D1:  "1d",
-    Timeframe.W1:  "1wk",
-}
-
 
 class DataIntegrityError(Exception):
     """Raised when data fails validation checks."""
@@ -168,35 +133,6 @@ class ForexDataProvider:
             if cached is not None and not cached.empty:
                 return cached
 
-        # Symbols not on Twelve Data (gold, silver) always use yfinance.
-        # D1 is skipped (yfinance daily for futures is unreliable).
-        # H4 is resampled from H1 since yfinance has no native 4h candle.
-        if symbol in _YF_FALLBACK_MAP:
-            if timeframe == Timeframe.D1:
-                raise DataUnavailableError(
-                    f"Daily chart skipped for {symbol} — gold/silver use M15/H1/H4 only."
-                )
-            if timeframe == Timeframe.H4:
-                h1_df = self._fetch_yfinance_fallback(symbol, Timeframe.H1)
-                if h1_df is None:
-                    raise DataUnavailableError(
-                        f"Yahoo Finance unavailable for {symbol} (H4 resample source H1)."
-                    )
-                df = self._resample_h1_to_h4(h1_df)
-                df = self._validate_and_clean(df, symbol, timeframe)
-                self._cache[cache_key] = df
-                self._cache_timestamps[cache_key] = now
-                return df
-            yf_df = self._fetch_yfinance_fallback(symbol, timeframe)
-            if yf_df is None:
-                raise DataUnavailableError(
-                    f"Yahoo Finance unavailable for {symbol} ({timeframe.value})."
-                )
-            df = self._validate_and_clean(yf_df, symbol, timeframe)
-            self._cache[cache_key] = df
-            self._cache_timestamps[cache_key] = now
-            return df
-
         df = self._fetch_twelve_data(symbol, timeframe, lookback_days)
         df = self._validate_and_clean(df, symbol, timeframe)
 
@@ -221,9 +157,7 @@ class ForexDataProvider:
         if lookback_days is None:
             lookback_days = self.config.data_lookback_days
 
-        # XAU/XAG aren't on Twelve Data — exclude them from the batch call,
+        td_symbols = list(symbols)
-        # they'll be fetched individually via yfinance on demand.
-        td_symbols = [s for s in symbols if s not in _YF_FALLBACK_MAP]
         if not td_symbols:
             return
 
@@ -288,17 +222,6 @@ class ForexDataProvider:
         }
         return fallback_spreads.get(symbol, 0.00020)
 
-    @staticmethod
-    def _resample_h1_to_h4(h1_df: pd.DataFrame) -> pd.DataFrame:
-        """Resample 1-hour OHLCV to 4-hour bars (yfinance has no native 4h)."""
-        return h1_df.resample("4h", label="right", closed="right").agg({
-            "Open":   "first",
-            "High":   "max",
-            "Low":    "min",
-            "Close":  "last",
-            "Volume": "sum",
-        }).dropna()
-
     def get_pair_info(self, symbol: str) -> Dict:
         """Return display name and pip value for a pair."""
         display = SYMBOL_DISPLAY.get(symbol, symbol)
@@ -314,60 +237,6 @@ class ForexDataProvider:
 
     # ── Private Methods ────────────────────────────────────────────
 
-    def _fetch_yfinance_fallback(
-        self, symbol: str, timeframe: Timeframe
-    ) -> Optional[pd.DataFrame]:
-        """
-        Fetch OHLCV from Yahoo Finance for symbols Twelve Data doesn't cover
-        on the free tier (XAUUSD, XAGUSD). Returns None if not mapped or fails.
-        """
-        if not _YF_AVAILABLE:
-            return None
-        yf_symbol = _YF_FALLBACK_MAP.get(symbol)
-        yf_interval = _YF_INTERVALS.get(timeframe)
-        if yf_symbol is None or yf_interval is None:
-            return None
-
-        # Yahoo intraday limits: 1m → last 7d, 15m/30m/1h → last 60d
-        if yf_interval in ("1m", "2m"):
-            period = "7d"
-        elif yf_interval in ("5m", "15m", "30m", "60m", "1h"):
-            period = "60d"
-        else:
-            period = "2y"
-
-        try:
-            yf_kwargs = dict(
-                interval=yf_interval, period=period,
-                progress=False, auto_adjust=False, threads=False,
-            )
-            if _YF_SESSION is not None:
-                yf_kwargs["session"] = _YF_SESSION
-            df = yf.download(yf_symbol, **yf_kwargs)
-            if df is None or df.empty:
-                return None
-
-            # Single-symbol download can return MultiIndex columns in newer yfinance versions
-            if isinstance(df.columns, pd.MultiIndex):
-                df.columns = df.columns.get_level_values(0)
-
-            # Normalise to UTC tz-aware index
-            if df.index.tz is None:
-                df.index = df.index.tz_localize("UTC")
-            else:
-                df.index = df.index.tz_convert("UTC")
-            df.index.name = "Datetime"
-
-            df = df[["Open", "High", "Low", "Close", "Volume"]].copy()
-            logger.info(
-                f"[yf] {yf_symbol} {yf_interval}: {len(df)} bars, "
-                f"last close={df['Close'].iloc[-1]:.5f}"
-            )
-            return df
-        except Exception as e:
-            logger.warning(f"yfinance fallback failed for {symbol} {timeframe.value}: {e}")
-            return None
-
     def _throttle(self, credits: int = 1) -> None:
         """
         Block until `credits` per-minute API credits are available.
@@ -484,14 +353,6 @@ class ForexDataProvider:
 
                 values = sym_data.get("values")
                 if not values:
-                    # Try yfinance fallback for unsupported symbols (XAU, XAG)
-                    yf_df = self._fetch_yfinance_fallback(symbol, timeframe)
-                    if yf_df is not None:
-                        yf_df = self._validate_and_clean(yf_df, symbol, timeframe)
-                        cache_key = f"{symbol}_{timeframe.value}"
-                        self._cache[cache_key] = yf_df
-                        self._cache_timestamps[cache_key] = now
-                        continue
                     if td_sym not in _no_data_warned:
                         logger.warning(f"No data from Twelve Data for {td_sym} — symbol may require a paid plan.")
                         _no_data_warned.add(td_sym)
@@ -527,13 +388,8 @@ class ForexDataProvider:
         """
         Fetch OHLCV from Twelve Data REST API.
         Raises DataUnavailableError when data cannot be retrieved.
-        Falls back to yfinance for symbols not on the Twelve Data free tier.
         """
         if not self._api_key:
-            # No Twelve Data key — try yfinance fallback for mapped symbols
-            yf_df = self._fetch_yfinance_fallback(symbol, timeframe)
-            if yf_df is not None:
-                return yf_df
             raise DataUnavailableError(
                 "No Twelve Data API key — set the TWELVE_DATA_API_KEY environment variable."
             )
@@ -571,9 +427,6 @@ class ForexDataProvider:
 
             values = data.get("values")
             if not values:
-                yf_df = self._fetch_yfinance_fallback(symbol, timeframe)
-                if yf_df is not None:
-                    return yf_df
                 raise DataUnavailableError(
                     f"No data returned for {td_symbol} — symbol may require a paid plan."
                 )

requirements.txt

@@ -4,15 +4,10 @@ pandas>=2.0
 
 # HTTP
 requests>=2.28
-curl_cffi>=0.7         # required by yfinance for impersonated TLS + verify=False
-                       # on TLS-inspecting servers
-
-# Market data
-yfinance>=0.2.40       # XAU/XAG (uses GC=F / SI=F futures via curl_cffi)
 
 # News sentiment (Gemini LLM)
-google-genai>=1.0      # new unified Google GenAI SDK (replaces google-generativeai)
+google-genai>=1.0      # new unified Google GenAI SDK
-python-dotenv>=1.0     # loads GEMINI_API_KEY from .env
+python-dotenv>=1.0     # loads GEMINI_API_KEY and TELEGRAM_BOT_TOKEN from .env
 
 # Telegram bot
 python-telegram-bot[job-queue]>=20.0

signal_aggregator.py

@@ -31,14 +31,10 @@ from news_engine import NewsEngine, FundamentalContext
 logger = logging.getLogger(__name__)
 
 
-# Per-symbol full TF list (primary + confirmations). Overrides the global
+# Per-symbol TF overrides. Currently empty — all forex pairs use the global
-# config.confirmation_timeframes for these symbols only — forex pairs are
+# config.confirmation_timeframes. Add entries here only if a specific symbol
-# unaffected. XAU/XAG use M15+H1+H4 (no D1 since yfinance daily for futures
+# needs a different MTF set.
-# is unreliable; H4 is resampled from H1 in data_provider).
+_PER_SYMBOL_TFS: Dict[str, List[Timeframe]] = {}
-_PER_SYMBOL_TFS: Dict[str, List[Timeframe]] = {
-    "XAUUSD=X": [Timeframe.M15, Timeframe.H1, Timeframe.H4],
-    "XAGUSD=X": [Timeframe.M15, Timeframe.H1, Timeframe.H4],
-}
 
 
 @dataclass

telegram_bot.py

@@ -25,6 +25,14 @@ from datetime import datetime
 from typing import Dict, List, Optional, Tuple
 from zoneinfo import ZoneInfo
 
+# Load .env so TELEGRAM_BOT_TOKEN / GEMINI_API_KEY / TWELVE_DATA_API_KEY
+# don't need to be exported manually.
+try:
+    from dotenv import load_dotenv
+    load_dotenv()
+except ImportError:
+    pass
+
 from telegram import BotCommand, Update
 from telegram.constants import ParseMode
 from telegram.ext import Application, CommandHandler, ContextTypes
@@ -43,8 +51,8 @@ logger = logging.getLogger(__name__)
 SCAN_TIMEFRAMES = ["M15", "H1"]  # two cards (M15 and H1) per /scan; cache shared
 
 # Pairs analyzed by the /scan command. Limited to the most liquid majors
-# plus gold to keep credits low and signal quality high.
+# to keep credits low and signal quality high.
-SCAN_PAIRS = ["EURUSD=X", "USDJPY=X", "GBPUSD=X", "XAUUSD=X"]
+SCAN_PAIRS = ["EURUSD=X", "USDJPY=X", "GBPUSD=X", "AUDUSD=X"]
 
 _TF_MAP = {
     "M5":  Timeframe.M5,  "M15": Timeframe.M15,