diff --git a/config.py b/config.py index 48cb0b3..29314b6 100644 --- a/config.py +++ b/config.py @@ -57,16 +57,12 @@ SYMBOL_DISPLAY = { "EURJPY=X": "EUR/JPY", "GBPJPY=X": "GBP/JPY", "AUDJPY=X": "AUD/JPY", "EURCHF=X": "EUR/CHF", "CADJPY=X": "CAD/JPY", - # Commodities - "XAUUSD=X": "XAU/USD", "XAGUSD=X": "XAG/USD", } # Pip values per symbol (used for trade setup risk/reward in pips) -# Commodities and JPY pairs differ from the standard 0.0001 forex pip -PIP_VALUES = { - "XAUUSD=X": 0.01, # Gold: 1 pip = $0.01 - "XAGUSD=X": 0.001, # Silver: 1 pip = $0.001 -} +# Forex defaults: 0.01 for JPY pairs, 0.0001 for everything else. +# Override here for any pair that doesn't follow that rule. +PIP_VALUES: Dict[str, float] = {} @dataclass diff --git a/data_provider.py b/data_provider.py index 325d9b0..97677ae 100644 --- a/data_provider.py +++ b/data_provider.py @@ -1,9 +1,9 @@ """ Forex Signal Intelligence Platform - Data Provider ==================================================== -Retrieves and validates market data from Twelve Data API. -Raises DataUnavailableError when the API is unreachable or credits are exhausted. -No synthetic/simulated data fallback. +Retrieves and validates currency-pair OHLCV from the Twelve Data API. +Raises DataUnavailableError when the API is unreachable or credits exhausted. +No fallback data source — forex pairs only. """ import logging @@ -17,46 +17,11 @@ from typing import Dict, List, Optional import pandas as pd import requests -try: - import yfinance as yf - _YF_AVAILABLE = True -except ImportError: - yf = None - _YF_AVAILABLE = False - -# Insecure HTTPS session for yfinance — disables SSL verification. -# Needed on servers behind a TLS-inspecting proxy that re-signs certs with -# an untrusted root. yfinance requires a curl_cffi session, not requests. -# Security trade-off: man-in-the-middle attacks on yfinance traffic are -# possible. Acceptable here because the data is public market quotes. -try: - from curl_cffi import requests as _curl_cffi_requests - _YF_SESSION = _curl_cffi_requests.Session(impersonate="chrome", verify=False) -except ImportError: - _YF_SESSION = None - from config import PlatformConfig, Timeframe, SYMBOL_DISPLAY, PIP_VALUES logger = logging.getLogger(__name__) _no_data_warned: set = set() # suppress duplicate "no data" warnings per symbol -# Symbols that Twelve Data free tier does not support — route to Yahoo Finance. -# Gold/silver futures are the most reliable Yahoo symbols (full OHLCV history). -_YF_FALLBACK_MAP = { - "XAUUSD=X": "GC=F", - "XAGUSD=X": "SI=F", -} - -_YF_INTERVALS = { - Timeframe.M1: "1m", - Timeframe.M5: "5m", - Timeframe.M15: "15m", - Timeframe.M30: "30m", - Timeframe.H1: "1h", - Timeframe.D1: "1d", - Timeframe.W1: "1wk", -} - class DataIntegrityError(Exception): """Raised when data fails validation checks.""" @@ -168,35 +133,6 @@ class ForexDataProvider: if cached is not None and not cached.empty: return cached - # Symbols not on Twelve Data (gold, silver) always use yfinance. - # D1 is skipped (yfinance daily for futures is unreliable). - # H4 is resampled from H1 since yfinance has no native 4h candle. - if symbol in _YF_FALLBACK_MAP: - if timeframe == Timeframe.D1: - raise DataUnavailableError( - f"Daily chart skipped for {symbol} — gold/silver use M15/H1/H4 only." - ) - if timeframe == Timeframe.H4: - h1_df = self._fetch_yfinance_fallback(symbol, Timeframe.H1) - if h1_df is None: - raise DataUnavailableError( - f"Yahoo Finance unavailable for {symbol} (H4 resample source H1)." - ) - df = self._resample_h1_to_h4(h1_df) - df = self._validate_and_clean(df, symbol, timeframe) - self._cache[cache_key] = df - self._cache_timestamps[cache_key] = now - return df - yf_df = self._fetch_yfinance_fallback(symbol, timeframe) - if yf_df is None: - raise DataUnavailableError( - f"Yahoo Finance unavailable for {symbol} ({timeframe.value})." - ) - df = self._validate_and_clean(yf_df, symbol, timeframe) - self._cache[cache_key] = df - self._cache_timestamps[cache_key] = now - return df - df = self._fetch_twelve_data(symbol, timeframe, lookback_days) df = self._validate_and_clean(df, symbol, timeframe) @@ -221,9 +157,7 @@ class ForexDataProvider: if lookback_days is None: lookback_days = self.config.data_lookback_days - # XAU/XAG aren't on Twelve Data — exclude them from the batch call, - # they'll be fetched individually via yfinance on demand. - td_symbols = [s for s in symbols if s not in _YF_FALLBACK_MAP] + td_symbols = list(symbols) if not td_symbols: return @@ -288,17 +222,6 @@ class ForexDataProvider: } return fallback_spreads.get(symbol, 0.00020) - @staticmethod - def _resample_h1_to_h4(h1_df: pd.DataFrame) -> pd.DataFrame: - """Resample 1-hour OHLCV to 4-hour bars (yfinance has no native 4h).""" - return h1_df.resample("4h", label="right", closed="right").agg({ - "Open": "first", - "High": "max", - "Low": "min", - "Close": "last", - "Volume": "sum", - }).dropna() - def get_pair_info(self, symbol: str) -> Dict: """Return display name and pip value for a pair.""" display = SYMBOL_DISPLAY.get(symbol, symbol) @@ -314,60 +237,6 @@ class ForexDataProvider: # ── Private Methods ──────────────────────────────────────────── - def _fetch_yfinance_fallback( - self, symbol: str, timeframe: Timeframe - ) -> Optional[pd.DataFrame]: - """ - Fetch OHLCV from Yahoo Finance for symbols Twelve Data doesn't cover - on the free tier (XAUUSD, XAGUSD). Returns None if not mapped or fails. - """ - if not _YF_AVAILABLE: - return None - yf_symbol = _YF_FALLBACK_MAP.get(symbol) - yf_interval = _YF_INTERVALS.get(timeframe) - if yf_symbol is None or yf_interval is None: - return None - - # Yahoo intraday limits: 1m → last 7d, 15m/30m/1h → last 60d - if yf_interval in ("1m", "2m"): - period = "7d" - elif yf_interval in ("5m", "15m", "30m", "60m", "1h"): - period = "60d" - else: - period = "2y" - - try: - yf_kwargs = dict( - interval=yf_interval, period=period, - progress=False, auto_adjust=False, threads=False, - ) - if _YF_SESSION is not None: - yf_kwargs["session"] = _YF_SESSION - df = yf.download(yf_symbol, **yf_kwargs) - if df is None or df.empty: - return None - - # Single-symbol download can return MultiIndex columns in newer yfinance versions - if isinstance(df.columns, pd.MultiIndex): - df.columns = df.columns.get_level_values(0) - - # Normalise to UTC tz-aware index - if df.index.tz is None: - df.index = df.index.tz_localize("UTC") - else: - df.index = df.index.tz_convert("UTC") - df.index.name = "Datetime" - - df = df[["Open", "High", "Low", "Close", "Volume"]].copy() - logger.info( - f"[yf] {yf_symbol} {yf_interval}: {len(df)} bars, " - f"last close={df['Close'].iloc[-1]:.5f}" - ) - return df - except Exception as e: - logger.warning(f"yfinance fallback failed for {symbol} {timeframe.value}: {e}") - return None - def _throttle(self, credits: int = 1) -> None: """ Block until `credits` per-minute API credits are available. @@ -484,14 +353,6 @@ class ForexDataProvider: values = sym_data.get("values") if not values: - # Try yfinance fallback for unsupported symbols (XAU, XAG) - yf_df = self._fetch_yfinance_fallback(symbol, timeframe) - if yf_df is not None: - yf_df = self._validate_and_clean(yf_df, symbol, timeframe) - cache_key = f"{symbol}_{timeframe.value}" - self._cache[cache_key] = yf_df - self._cache_timestamps[cache_key] = now - continue if td_sym not in _no_data_warned: logger.warning(f"No data from Twelve Data for {td_sym} — symbol may require a paid plan.") _no_data_warned.add(td_sym) @@ -527,13 +388,8 @@ class ForexDataProvider: """ Fetch OHLCV from Twelve Data REST API. Raises DataUnavailableError when data cannot be retrieved. - Falls back to yfinance for symbols not on the Twelve Data free tier. """ if not self._api_key: - # No Twelve Data key — try yfinance fallback for mapped symbols - yf_df = self._fetch_yfinance_fallback(symbol, timeframe) - if yf_df is not None: - return yf_df raise DataUnavailableError( "No Twelve Data API key — set the TWELVE_DATA_API_KEY environment variable." ) @@ -571,9 +427,6 @@ class ForexDataProvider: values = data.get("values") if not values: - yf_df = self._fetch_yfinance_fallback(symbol, timeframe) - if yf_df is not None: - return yf_df raise DataUnavailableError( f"No data returned for {td_symbol} — symbol may require a paid plan." ) diff --git a/requirements.txt b/requirements.txt index 5d71650..dd22b83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,15 +4,10 @@ pandas>=2.0 # HTTP requests>=2.28 -curl_cffi>=0.7 # required by yfinance for impersonated TLS + verify=False - # on TLS-inspecting servers - -# Market data -yfinance>=0.2.40 # XAU/XAG (uses GC=F / SI=F futures via curl_cffi) # News sentiment (Gemini LLM) -google-genai>=1.0 # new unified Google GenAI SDK (replaces google-generativeai) -python-dotenv>=1.0 # loads GEMINI_API_KEY from .env +google-genai>=1.0 # new unified Google GenAI SDK +python-dotenv>=1.0 # loads GEMINI_API_KEY and TELEGRAM_BOT_TOKEN from .env # Telegram bot python-telegram-bot[job-queue]>=20.0 diff --git a/signal_aggregator.py b/signal_aggregator.py index 7cec105..aba7ed2 100644 --- a/signal_aggregator.py +++ b/signal_aggregator.py @@ -31,14 +31,10 @@ from news_engine import NewsEngine, FundamentalContext logger = logging.getLogger(__name__) -# Per-symbol full TF list (primary + confirmations). Overrides the global -# config.confirmation_timeframes for these symbols only — forex pairs are -# unaffected. XAU/XAG use M15+H1+H4 (no D1 since yfinance daily for futures -# is unreliable; H4 is resampled from H1 in data_provider). -_PER_SYMBOL_TFS: Dict[str, List[Timeframe]] = { - "XAUUSD=X": [Timeframe.M15, Timeframe.H1, Timeframe.H4], - "XAGUSD=X": [Timeframe.M15, Timeframe.H1, Timeframe.H4], -} +# Per-symbol TF overrides. Currently empty — all forex pairs use the global +# config.confirmation_timeframes. Add entries here only if a specific symbol +# needs a different MTF set. +_PER_SYMBOL_TFS: Dict[str, List[Timeframe]] = {} @dataclass diff --git a/telegram_bot.py b/telegram_bot.py index 2de480f..8fd324e 100644 --- a/telegram_bot.py +++ b/telegram_bot.py @@ -25,6 +25,14 @@ from datetime import datetime from typing import Dict, List, Optional, Tuple from zoneinfo import ZoneInfo +# Load .env so TELEGRAM_BOT_TOKEN / GEMINI_API_KEY / TWELVE_DATA_API_KEY +# don't need to be exported manually. +try: + from dotenv import load_dotenv + load_dotenv() +except ImportError: + pass + from telegram import BotCommand, Update from telegram.constants import ParseMode from telegram.ext import Application, CommandHandler, ContextTypes @@ -43,8 +51,8 @@ logger = logging.getLogger(__name__) SCAN_TIMEFRAMES = ["M15", "H1"] # two cards (M15 and H1) per /scan; cache shared # Pairs analyzed by the /scan command. Limited to the most liquid majors -# plus gold to keep credits low and signal quality high. -SCAN_PAIRS = ["EURUSD=X", "USDJPY=X", "GBPUSD=X", "XAUUSD=X"] +# to keep credits low and signal quality high. +SCAN_PAIRS = ["EURUSD=X", "USDJPY=X", "GBPUSD=X", "AUDUSD=X"] _TF_MAP = { "M5": Timeframe.M5, "M15": Timeframe.M15,