harry/ForexBot2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove yfinance/XAU/XAG fallback — focus on currency pairs only

Browse Source 
- data_provider: strip yfinance imports, _YF_FALLBACK_MAP, _YF_INTERVALS,
  _YF_SESSION, _fetch_yfinance_fallback, _resample_h1_to_h4. Twelve Data
  is the sole data source. Raises DataUnavailableError if a pair isn't
  on the configured plan.
- config: remove XAU/USD and XAG/USD from SYMBOL_DISPLAY and PIP_VALUES.
  PIP_VALUES is now empty (defaults: 0.01 JPY pairs, 0.0001 forex).
- signal_aggregator: clear _PER_SYMBOL_TFS overrides (was XAU/XAG-only).
- telegram_bot: replace XAUUSD with AUDUSD in SCAN_PAIRS. Auto-load .env
  via python-dotenv so TELEGRAM_BOT_TOKEN/GEMINI_API_KEY are picked up.
- requirements: drop yfinance and curl_cffi dependencies.

End-to-end pipeline test passes: data freshness, news engine (Gemini live),
signal aggregator, and the /signal+/scan code paths in telegram_bot.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Harwinder
2026-05-06 12:13:09 -04:00
parent bf6c890029
commit 405ddffc9c
5 changed files with 23 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config.py
+3 -7
View File
@@ -57,16 +57,12 @@ SYMBOL_DISPLAY = {
"EURJPY=X": "EUR/JPY", "GBPJPY=X": "GBP/JPY",
"AUDJPY=X": "AUD/JPY", "EURCHF=X": "EUR/CHF",
"CADJPY=X": "CAD/JPY",
# Commodities
"XAUUSD=X": "XAU/USD", "XAGUSD=X": "XAG/USD",
}
# Pip values per symbol (used for trade setup risk/reward in pips)
# Commodities and JPY pairs differ from the standard 0.0001 forex pip
PIP_VALUES = {
"XAUUSD=X": 0.01, # Gold: 1 pip = $0.01
"XAGUSD=X": 0.001, # Silver: 1 pip = $0.001
}
# Forex defaults: 0.01 for JPY pairs, 0.0001 for everything else.
# Override here for any pair that doesn't follow that rule.
PIP_VALUES: Dict[str, float] = {}
@dataclass
data_provider.py
+4 -151
View File
@@ -1,9 +1,9 @@
"""
Forex Signal Intelligence Platform - Data Provider
====================================================
Retrieves and validates market data from Twelve Data API.
Raises DataUnavailableError when the API is unreachable or credits are exhausted.
No synthetic/simulated data fallback.
Retrieves and validates currency-pair OHLCV from the Twelve Data API.
Raises DataUnavailableError when the API is unreachable or credits exhausted.
No fallback data source — forex pairs only.
"""
import logging
@@ -17,46 +17,11 @@ from typing import Dict, List, Optional
import pandas as pd
import requests
try:
import yfinance as yf
_YF_AVAILABLE = True
except ImportError:
yf = None
_YF_AVAILABLE = False
# Insecure HTTPS session for yfinance — disables SSL verification.
# Needed on servers behind a TLS-inspecting proxy that re-signs certs with
# an untrusted root. yfinance requires a curl_cffi session, not requests.
# Security trade-off: man-in-the-middle attacks on yfinance traffic are
# possible. Acceptable here because the data is public market quotes.
try:
from curl_cffi import requests as _curl_cffi_requests
_YF_SESSION = _curl_cffi_requests.Session(impersonate="chrome", verify=False)
except ImportError:
_YF_SESSION = None
from config import PlatformConfig, Timeframe, SYMBOL_DISPLAY, PIP_VALUES
logger = logging.getLogger(__name__)
_no_data_warned: set = set() # suppress duplicate "no data" warnings per symbol
# Symbols that Twelve Data free tier does not support — route to Yahoo Finance.
# Gold/silver futures are the most reliable Yahoo symbols (full OHLCV history).
_YF_FALLBACK_MAP = {
"XAUUSD=X": "GC=F",
"XAGUSD=X": "SI=F",
}
_YF_INTERVALS = {
Timeframe.M1: "1m",
Timeframe.M5: "5m",
Timeframe.M15: "15m",
Timeframe.M30: "30m",
Timeframe.H1: "1h",
Timeframe.D1: "1d",
Timeframe.W1: "1wk",
}
class DataIntegrityError(Exception):
"""Raised when data fails validation checks."""
@@ -168,35 +133,6 @@ class ForexDataProvider:
if cached is not None and not cached.empty:
return cached
# Symbols not on Twelve Data (gold, silver) always use yfinance.
# D1 is skipped (yfinance daily for futures is unreliable).
# H4 is resampled from H1 since yfinance has no native 4h candle.
if symbol in _YF_FALLBACK_MAP:
if timeframe == Timeframe.D1:
raise DataUnavailableError(
f"Daily chart skipped for {symbol} — gold/silver use M15/H1/H4 only."
)
if timeframe == Timeframe.H4:
h1_df = self._fetch_yfinance_fallback(symbol, Timeframe.H1)
if h1_df is None:
raise DataUnavailableError(
f"Yahoo Finance unavailable for {symbol} (H4 resample source H1)."
)
df = self._resample_h1_to_h4(h1_df)
df = self._validate_and_clean(df, symbol, timeframe)
self._cache[cache_key] = df
self._cache_timestamps[cache_key] = now
return df
yf_df = self._fetch_yfinance_fallback(symbol, timeframe)
if yf_df is None:
raise DataUnavailableError(
f"Yahoo Finance unavailable for {symbol} ({timeframe.value})."
)
df = self._validate_and_clean(yf_df, symbol, timeframe)
self._cache[cache_key] = df
self._cache_timestamps[cache_key] = now
return df
df = self._fetch_twelve_data(symbol, timeframe, lookback_days)
df = self._validate_and_clean(df, symbol, timeframe)
@@ -221,9 +157,7 @@ class ForexDataProvider:
if lookback_days is None:
lookback_days = self.config.data_lookback_days
# XAU/XAG aren't on Twelve Data — exclude them from the batch call,
# they'll be fetched individually via yfinance on demand.
td_symbols = [s for s in symbols if s not in _YF_FALLBACK_MAP]
td_symbols = list(symbols)
if not td_symbols:
return
@@ -288,17 +222,6 @@ class ForexDataProvider:
}
return fallback_spreads.get(symbol, 0.00020)
@staticmethod
def _resample_h1_to_h4(h1_df: pd.DataFrame) -> pd.DataFrame:
"""Resample 1-hour OHLCV to 4-hour bars (yfinance has no native 4h)."""
return h1_df.resample("4h", label="right", closed="right").agg({
"Open": "first",
"High": "max",
"Low": "min",
"Close": "last",
"Volume": "sum",
}).dropna()
def get_pair_info(self, symbol: str) -> Dict:
"""Return display name and pip value for a pair."""
display = SYMBOL_DISPLAY.get(symbol, symbol)
@@ -314,60 +237,6 @@ class ForexDataProvider:
# ── Private Methods ────────────────────────────────────────────
def _fetch_yfinance_fallback(
self, symbol: str, timeframe: Timeframe
) -> Optional[pd.DataFrame]:
"""
Fetch OHLCV from Yahoo Finance for symbols Twelve Data doesn't cover
on the free tier (XAUUSD, XAGUSD). Returns None if not mapped or fails.
"""
if not _YF_AVAILABLE:
return None
yf_symbol = _YF_FALLBACK_MAP.get(symbol)
yf_interval = _YF_INTERVALS.get(timeframe)
if yf_symbol is None or yf_interval is None:
return None
# Yahoo intraday limits: 1m → last 7d, 15m/30m/1h → last 60d
if yf_interval in ("1m", "2m"):
period = "7d"
elif yf_interval in ("5m", "15m", "30m", "60m", "1h"):
period = "60d"
else:
period = "2y"
try:
yf_kwargs = dict(
interval=yf_interval, period=period,
progress=False, auto_adjust=False, threads=False,
)
if _YF_SESSION is not None:
yf_kwargs["session"] = _YF_SESSION
df = yf.download(yf_symbol, **yf_kwargs)
if df is None or df.empty:
return None
# Single-symbol download can return MultiIndex columns in newer yfinance versions
if isinstance(df.columns, pd.MultiIndex):
df.columns = df.columns.get_level_values(0)
# Normalise to UTC tz-aware index
if df.index.tz is None:
df.index = df.index.tz_localize("UTC")
else:
df.index = df.index.tz_convert("UTC")
df.index.name = "Datetime"
df = df[["Open", "High", "Low", "Close", "Volume"]].copy()
logger.info(
f"[yf] {yf_symbol} {yf_interval}: {len(df)} bars, "
f"last close={df['Close'].iloc[-1]:.5f}"
)
return df
except Exception as e:
logger.warning(f"yfinance fallback failed for {symbol} {timeframe.value}: {e}")
return None
def _throttle(self, credits: int = 1) -> None:
"""
Block until `credits` per-minute API credits are available.
@@ -484,14 +353,6 @@ class ForexDataProvider:
values = sym_data.get("values")
if not values:
# Try yfinance fallback for unsupported symbols (XAU, XAG)
yf_df = self._fetch_yfinance_fallback(symbol, timeframe)
if yf_df is not None:
yf_df = self._validate_and_clean(yf_df, symbol, timeframe)
cache_key = f"{symbol}_{timeframe.value}"
self._cache[cache_key] = yf_df
self._cache_timestamps[cache_key] = now
continue
if td_sym not in _no_data_warned:
logger.warning(f"No data from Twelve Data for {td_sym} — symbol may require a paid plan.")
_no_data_warned.add(td_sym)
@@ -527,13 +388,8 @@ class ForexDataProvider:
"""
Fetch OHLCV from Twelve Data REST API.
Raises DataUnavailableError when data cannot be retrieved.
Falls back to yfinance for symbols not on the Twelve Data free tier.
"""
if not self._api_key:
# No Twelve Data key — try yfinance fallback for mapped symbols
yf_df = self._fetch_yfinance_fallback(symbol, timeframe)
if yf_df is not None:
return yf_df
raise DataUnavailableError(
"No Twelve Data API key — set the TWELVE_DATA_API_KEY environment variable."
)
@@ -571,9 +427,6 @@ class ForexDataProvider:
values = data.get("values")
if not values:
yf_df = self._fetch_yfinance_fallback(symbol, timeframe)
if yf_df is not None:
return yf_df
raise DataUnavailableError(
f"No data returned for {td_symbol} — symbol may require a paid plan."
)
requirements.txt
+2 -7
View File
@@ -4,15 +4,10 @@ pandas>=2.0
# HTTP
requests>=2.28
curl_cffi>=0.7 # required by yfinance for impersonated TLS + verify=False
# on TLS-inspecting servers
# Market data
yfinance>=0.2.40 # XAU/XAG (uses GC=F / SI=F futures via curl_cffi)
# News sentiment (Gemini LLM)
google-genai>=1.0 # new unified Google GenAI SDK (replaces google-generativeai)
python-dotenv>=1.0 # loads GEMINI_API_KEY from .env
google-genai>=1.0 # new unified Google GenAI SDK
python-dotenv>=1.0 # loads GEMINI_API_KEY and TELEGRAM_BOT_TOKEN from .env
# Telegram bot
python-telegram-bot[job-queue]>=20.0
signal_aggregator.py
+4 -8
View File
@@ -31,14 +31,10 @@ from news_engine import NewsEngine, FundamentalContext
logger = logging.getLogger(__name__)
# Per-symbol full TF list (primary + confirmations). Overrides the global
# config.confirmation_timeframes for these symbols only — forex pairs are
# unaffected. XAU/XAG use M15+H1+H4 (no D1 since yfinance daily for futures
# is unreliable; H4 is resampled from H1 in data_provider).
_PER_SYMBOL_TFS: Dict[str, List[Timeframe]] = {
"XAUUSD=X": [Timeframe.M15, Timeframe.H1, Timeframe.H4],
"XAGUSD=X": [Timeframe.M15, Timeframe.H1, Timeframe.H4],
}
# Per-symbol TF overrides. Currently empty — all forex pairs use the global
# config.confirmation_timeframes. Add entries here only if a specific symbol
# needs a different MTF set.
_PER_SYMBOL_TFS: Dict[str, List[Timeframe]] = {}
@dataclass
telegram_bot.py
+10 -2
View File
@@ -25,6 +25,14 @@ from datetime import datetime
from typing import Dict, List, Optional, Tuple
from zoneinfo import ZoneInfo
# Load .env so TELEGRAM_BOT_TOKEN / GEMINI_API_KEY / TWELVE_DATA_API_KEY
# don't need to be exported manually.
try:
from dotenv import load_dotenv
load_dotenv()
except ImportError:
pass
from telegram import BotCommand, Update
from telegram.constants import ParseMode
from telegram.ext import Application, CommandHandler, ContextTypes
@@ -43,8 +51,8 @@ logger = logging.getLogger(__name__)
SCAN_TIMEFRAMES = ["M15", "H1"] # two cards (M15 and H1) per /scan; cache shared
# Pairs analyzed by the /scan command. Limited to the most liquid majors
# plus gold to keep credits low and signal quality high.
SCAN_PAIRS = ["EURUSD=X", "USDJPY=X", "GBPUSD=X", "XAUUSD=X"]
# to keep credits low and signal quality high.
SCAN_PAIRS = ["EURUSD=X", "USDJPY=X", "GBPUSD=X", "AUDUSD=X"]
_TF_MAP = {
"M5": Timeframe.M5, "M15": Timeframe.M15,