harry/ForexBot2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
988adf9a3db3c6838bdeb29a2d96155d68cc5ae9
ForexBot2/news_engine.py
T
Harwinder bf6c890029 Implement Gemini batch sentiment scoring; replace FinBERT/transformers 
- Migrate news_engine to google-genai SDK with gemini-2.5-flash-lite
- Batch all headlines into one API call per scan (30 calls -> 1)
- In-memory cache keyed by headline hash; fallback to keyword scoring
- Update requirements.txt: add google-genai, python-dotenv; remove transformers
- Add .gitignore; update CLAUDE.md with Gemini live section

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-05-06 09:58:44 -04:00

558 lines
22 KiB
Python
Raw Blame History

"""
Monitors economic calendar events and news sentiment to provide
fundamental context for technical signals.
Key principle: High-impact news events override technical signals.
A perfect technical setup can be destroyed by an unexpected NFP print.
"""
import json
import logging
import os
import xml.etree.ElementTree as ET
from dataclasses import dataclass, field
from datetime import datetime, timedelta, timezone
from typing import Dict, List, Optional, Tuple
import requests
# Load .env so GEMINI_API_KEY is available without exporting it manually.
try:
from dotenv import load_dotenv
load_dotenv()
except ImportError:
pass
# New unified Google GenAI SDK (replaces google-generativeai).
try:
from google import genai
from google.genai import types as genai_types
_GENAI_AVAILABLE = True
except ImportError:
genai = None
genai_types = None
_GENAI_AVAILABLE = False
from config import NewsConfig
logger = logging.getLogger(__name__)
@dataclass
class EconomicEvent:
"""Single economic calendar event."""
time: datetime
currency: str
impact: str # "low", "medium", "high"
event: str
forecast: Optional[str] = None
previous: Optional[str] = None
actual: Optional[str] = None
surprise: Optional[str] = None # "positive", "negative", "inline", None
@property
def is_upcoming(self) -> bool:
return self.time > datetime.now(timezone.utc)
@property
def minutes_until(self) -> float:
delta = self.time - datetime.now(timezone.utc)
return delta.total_seconds() / 60
@property
def is_high_impact(self) -> bool:
return self.impact == "high"
@dataclass
class NewsSentiment:
"""News sentiment analysis result."""
headline: str
source: str
timestamp: datetime
currencies_mentioned: List[str]
sentiment_score: float # -1.0 (bearish) to +1.0 (bullish)
keywords: List[str]
@dataclass
class FundamentalContext:
"""Complete fundamental/news context for signal decisions."""
events_today: List[EconomicEvent] = field(default_factory=list)
events_upcoming_1h: List[EconomicEvent] = field(default_factory=list)
high_impact_warning: bool = False
warning_message: str = ""
news_sentiment: Dict[str, float] = field(default_factory=dict) # currency -> sentiment
news_headlines: List[NewsSentiment] = field(default_factory=list)
event_impact_on_pairs: Dict[str, str] = field(default_factory=dict)
calendar_available: bool = False # True if live calendar data was fetched
class NewsEngine:
"""
Economic calendar and news sentiment analyzer using Google Gemini API.
Provides fundamental context that modifies technical signal confidence.
"""
# Keywords that indicate directional sentiment (kept for fallback and keyword extraction)
BULLISH_KEYWORDS = {
"rate hike", "hawkish", "strong growth", "beat expectations",
"above forecast", "surplus", "expansion", "employment gains",
"strong jobs", "gdp growth", "inflation rising", "tightening",
"positive surprise", "record high", "bullish", "rally",
"stronger than expected", "upbeat",
}
BEARISH_KEYWORDS = {
"rate cut", "dovish", "recession", "slowdown", "contraction",
"below forecast", "deficit", "miss expectations", "weak data",
"unemployment rise", "negative surprise", "bearish", "selloff",
"weaker than expected", "downbeat", "easing", "stimulus",
}
# Currency-specific keywords for headline parsing
CURRENCY_KEYWORDS = {
"USD": ["dollar", "fed", "fomc", "treasury", "us ", "american", "nonfarm", "nfp"],
"EUR": ["euro", "ecb", "eurozone", "eu ", "german", "french"],
"GBP": ["pound", "sterling", "boe", "bank of england", "uk ", "british"],
"JPY": ["yen", "boj", "bank of japan", "japanese"],
"CHF": ["franc", "snb", "swiss"],
"AUD": ["aussie", "rba", "australian", "australia"],
"CAD": ["loonie", "boc", "canadian", "canada"],
"NZD": ["kiwi", "rbnz", "new zealand"],
}
# High-impact event types (these move markets significantly)
HIGH_IMPACT_EVENTS = {
"interest rate decision", "nonfarm payrolls", "non-farm",
"gdp", "cpi", "inflation", "employment change",
"retail sales", "pmi", "fomc", "ecb press conference",
"boe rate", "rba rate", "snb rate", "boc rate",
}
# gemini-2.5-flash-lite has a much higher free-tier quota (1000 RPD vs 20
# for gemini-2.5-flash) and is more than capable for headline classification.
GEMINI_MODEL = "gemini-2.5-flash-lite"
BATCH_MAX_HEADLINES = 60 # cap per Gemini call to keep prompts small
def __init__(self, config: NewsConfig):
self.config = config
self._session = requests.Session()
self._session.headers.update({
"User-Agent": "Mozilla/5.0 (compatible; ForexIntel/1.0)"
})
# In-memory cache for Gemini-scored headlines so repeat fetches across
# /scan calls don't re-spend tokens on the same RSS items.
self._sentiment_cache: Dict[int, float] = {}
# Initialize Google Gemini via the new unified google-genai SDK.
api_key = os.environ.get("GEMINI_API_KEY")
self.gemini_client = None
if not _GENAI_AVAILABLE:
logger.warning("google-genai SDK not installed — falling back to keyword sentiment.")
elif not api_key:
logger.warning("GEMINI_API_KEY not set — falling back to keyword sentiment.")
else:
try:
self.gemini_client = genai.Client(api_key=api_key)
logger.info(f"Google Gemini ({self.GEMINI_MODEL}) configured.")
except Exception as e:
logger.error(f"Failed to configure Gemini: {e}")
self.gemini_client = None
def get_fundamental_context(
self, currencies: Optional[List[str]] = None
) -> FundamentalContext:
"""
Get complete fundamental context for signal decisions.
Combines economic calendar + news sentiment into actionable context.
"""
if currencies is None:
currencies = self.config.tracked_currencies
context = FundamentalContext()
# 1. Economic calendar events
try:
events = self._fetch_economic_calendar(currencies)
now = datetime.now(timezone.utc)
context.events_today = [e for e in events if e.time.date() == now.date()]
context.events_upcoming_1h = [
e for e in events
if 0 < e.minutes_until <= 60
]
context.calendar_available = bool(events)
# Check for high-impact warnings
for event in events:
if event.is_high_impact and 0 < event.minutes_until <= self.config.pre_event_buffer_minutes:
context.high_impact_warning = True
context.warning_message = (
f"HIGH IMPACT: {event.currency} {event.event} "
f"in {event.minutes_until:.0f} min — signals may be unreliable"
)
break
# Map upcoming high-impact events to affected pairs
for event in context.events_upcoming_1h:
if event.is_high_impact:
affected = self._get_affected_pairs(event.currency)
for pair in affected:
context.event_impact_on_pairs[pair] = (
f"{event.currency} {event.event} in {event.minutes_until:.0f}min"
)
except Exception as e:
logger.warning(f"Economic calendar fetch failed: {e}")
# 2. News sentiment
try:
headlines = self._fetch_news_sentiment()
context.news_headlines = headlines
# Aggregate sentiment per currency
for currency in currencies:
scores = [
h.sentiment_score for h in headlines
if currency in h.currencies_mentioned
]
if scores:
context.news_sentiment[currency] = sum(scores) / len(scores)
except Exception as e:
logger.warning(f"News sentiment fetch failed: {e}")
return context
def get_event_risk_level(self, pair_symbol: str, context: FundamentalContext) -> Tuple[str, str]:
"""
Get event risk level for a specific pair.
Returns (risk_level, description).
"""
clean = pair_symbol.replace("=X", "")
base = clean[:3]
quote = clean[3:]
# Check upcoming high-impact events
for event in context.events_upcoming_1h:
if event.is_high_impact and event.currency in (base, quote):
mins = event.minutes_until
if mins <= self.config.pre_event_buffer_minutes:
return "danger", f"{event.currency} {event.event} in {mins:.0f}min"
else:
return "caution", f"{event.currency} {event.event} in {mins:.0f}min"
# Check for recent high-impact releases
for event in context.events_today:
if event.is_high_impact and event.currency in (base, quote):
if not event.is_upcoming and event.minutes_until > -self.config.post_event_buffer_minutes:
return "caution", f"{event.currency} {event.event} just released"
return "clear", "No significant events"
def get_pair_sentiment(
self, pair_symbol: str, context: FundamentalContext
) -> Tuple[float, str]:
"""
Get net sentiment for a pair based on news.
Returns (score, description) where score is -1 to +1.
"""
clean = pair_symbol.replace("=X", "")
base = clean[:3]
quote = clean[3:]
base_sent = context.news_sentiment.get(base, 0.0)
quote_sent = context.news_sentiment.get(quote, 0.0)
# Net sentiment: positive base or negative quote = bullish for pair
net = base_sent - quote_sent
if abs(net) < 0.1:
desc = "Neutral news sentiment"
elif net > 0.3:
desc = f"Bullish sentiment ({base} positive / {quote} negative)"
elif net > 0:
desc = "Slightly bullish sentiment"
elif net < -0.3:
desc = f"Bearish sentiment ({base} negative / {quote} positive)"
else:
desc = "Slightly bearish sentiment"
return net, desc
# ── Private Methods ────────────────────────────────────────────
def _fetch_economic_calendar(self, currencies: List[str]) -> List[EconomicEvent]:
try:
events = self._try_fetch_calendar_api()
if events:
return [e for e in events if e.currency in currencies]
except Exception as e:
logger.debug(f"Calendar API fetch exception: {e}")
logger.info("Economic calendar data unavailable. Risk checks skipped.")
return []
def _try_fetch_calendar_api(self) -> List[EconomicEvent]:
events = []
url = "https://nfs.faireconomy.media/ff_calendar_thisweek.json"
resp = self._session.get(url, timeout=10)
if resp.status_code != 200:
return []
data = resp.json()
impact_map = {"High": "high", "Medium": "medium", "Low": "low"}
for item in data:
try:
impact = impact_map.get(item.get("impact", ""), "low")
date_str = item.get("date", "")
if not date_str:
continue
event_time = datetime.fromisoformat(date_str.replace("Z", "+00:00"))
events.append(EconomicEvent(
time=event_time,
currency=item.get("country", ""),
impact=impact,
event=item.get("title", ""),
forecast=item.get("forecast") or None,
previous=item.get("previous") or None,
actual=item.get("actual") or None,
))
except Exception:
continue
return events
def _fetch_news_sentiment(self) -> List[NewsSentiment]:
# Stage 1: collect all candidate (title, pub_date, source) from RSS feeds
# without scoring yet.
candidates: List[Tuple[str, datetime, str]] = []
for feed_url in self.config.rss_feeds:
try:
resp = self._session.get(feed_url, timeout=10)
if resp.status_code != 200:
continue
root = ET.fromstring(resp.content)
for item in root.iter("item"):
title_el = item.find("title")
pub_date_el = item.find("pubDate")
if title_el is None:
continue
title = title_el.text or ""
pub_date = self._parse_rss_date(
pub_date_el.text if pub_date_el is not None else None
)
if pub_date and (datetime.now(timezone.utc) - pub_date).total_seconds() > 86400:
continue
if not self._extract_currencies(title):
continue # skip headlines that don't mention any tracked currency
candidates.append((title, pub_date or datetime.now(timezone.utc), feed_url))
except Exception as e:
logger.debug(f"Failed to fetch RSS from {feed_url}: {e}")
# Stage 2: batch-score all relevant titles in one Gemini call (or as
# few as possible). Cached titles are served from memory; only the
# uncached subset is sent to the API.
unique_titles = list({t for t, _, _ in candidates})
scores = self._score_headlines_batch(unique_titles)
# Stage 3: build NewsSentiment objects using the scored map.
return [
NewsSentiment(
headline=title,
source=source,
timestamp=ts,
currencies_mentioned=self._extract_currencies(title),
sentiment_score=scores.get(title, 0.0),
keywords=self._extract_keywords(title),
)
for title, ts, source in candidates
]
def _score_headlines_batch(self, headlines: List[str]) -> Dict[str, float]:
"""
Score a list of headlines with a single Gemini API call.
Cached titles are returned from memory; only the uncached subset is
sent to the model. Falls back to keyword scoring on API failure.
"""
out: Dict[str, float] = {}
to_score: List[str] = []
for h in headlines:
key = hash(h)
if key in self._sentiment_cache:
out[h] = self._sentiment_cache[key]
else:
to_score.append(h)
if not to_score:
return out
if self.gemini_client is None:
for h in to_score:
s = self._fallback_keyword_sentiment(h)
self._sentiment_cache[hash(h)] = s
out[h] = s
return out
# Process in chunks to keep the prompt reasonable.
for i in range(0, len(to_score), self.BATCH_MAX_HEADLINES):
chunk = to_score[i : i + self.BATCH_MAX_HEADLINES]
chunk_scores = self._gemini_score_chunk(chunk)
for h in chunk:
s = chunk_scores.get(h)
if s is None:
s = self._fallback_keyword_sentiment(h)
self._sentiment_cache[hash(h)] = s
out[h] = s
return out
def _gemini_score_chunk(self, headlines: List[str]) -> Dict[str, float]:
"""Single Gemini call returning {headline: score} for the chunk."""
numbered = "\n".join(f'{i + 1}. "{h}"' for i, h in enumerate(headlines))
prompt = (
"You are an expert Forex fundamental analyst. Score each headline "
"from -1.0 (extremely bearish for its primary currency) to +1.0 "
"(extremely bullish). Use 0.0 if neutral or off-topic. Consider "
"context: surprises beating forecast = bullish; misses = bearish; "
'central bank "pause" or "patient" = dovish/bearish; "hawkish" or '
'"hike" = bullish.\n\n'
f"Headlines:\n{numbered}\n\n"
'Respond ONLY with a JSON array, one object per headline, in order:\n'
'[{"id": 1, "score": 0.5}, {"id": 2, "score": -0.3}, ...]'
)
try:
response = self.gemini_client.models.generate_content(
model=self.GEMINI_MODEL,
contents=prompt,
config=genai_types.GenerateContentConfig(
response_mime_type="application/json",
temperature=0.0,
),
)
data = json.loads(response.text)
scores: Dict[str, float] = {}
for entry in data:
idx = int(entry.get("id", 0)) - 1
if 0 <= idx < len(headlines):
raw = float(entry.get("score", 0.0))
scores[headlines[idx]] = max(-1.0, min(1.0, raw))
return scores
except Exception as e:
logger.warning(f"Gemini batch scoring failed ({len(headlines)} headlines): {e}")
return {}
def _analyze_headline_sentiment(self, headline: str) -> float:
"""
Score a single headline -1.0 to +1.0 via Google Gemini.
Falls back to keyword sentiment if the API is unavailable or fails.
Cached per-headline so repeat calls cost zero tokens.
"""
cache_key = hash(headline)
if cache_key in self._sentiment_cache:
return self._sentiment_cache[cache_key]
if self.gemini_client is None:
score = self._fallback_keyword_sentiment(headline)
self._sentiment_cache[cache_key] = score
return score
prompt = (
"You are an expert Forex fundamental analyst. "
"Analyze this financial news headline:\n"
f'"{headline}"\n\n'
"Determine the sentiment score on the primary currency mentioned.\n"
"-1.0 means extremely bearish (rate cuts, recession, weak data).\n"
" 1.0 means extremely bullish (rate hikes, strong growth, beats).\n"
" 0.0 means neutral or irrelevant.\n\n"
'Respond ONLY with valid JSON: {"score": 0.85}'
)
try:
response = self.gemini_client.models.generate_content(
model=self.GEMINI_MODEL,
contents=prompt,
config=genai_types.GenerateContentConfig(
response_mime_type="application/json",
temperature=0.0,
),
)
data = json.loads(response.text)
score = max(-1.0, min(1.0, float(data.get("score", 0.0))))
except Exception as e:
logger.debug(f"Gemini API failed for '{headline[:60]}': {e}")
score = self._fallback_keyword_sentiment(headline)
self._sentiment_cache[cache_key] = score
return score
def _fallback_keyword_sentiment(self, headline: str) -> float:
"""Original keyword-based fallback if Gemini API is unavailable or fails."""
headline_lower = headline.lower()
bullish_score = sum(1 for kw in self.BULLISH_KEYWORDS if kw in headline_lower)
bearish_score = sum(1 for kw in self.BEARISH_KEYWORDS if kw in headline_lower)
total = bullish_score + bearish_score
if total == 0:
return 0.0
return (bullish_score - bearish_score) / total
def _extract_currencies(self, headline: str) -> List[str]:
headline_lower = headline.lower()
found = []
for currency, keywords in self.CURRENCY_KEYWORDS.items():
if currency.lower() in headline_lower:
found.append(currency)
continue
for kw in keywords:
if kw in headline_lower:
found.append(currency)
break
return list(set(found))
def _extract_keywords(self, headline: str) -> List[str]:
headline_lower = headline.lower()
return [kw for kw in self.BULLISH_KEYWORDS | self.BEARISH_KEYWORDS if kw in headline_lower]
@staticmethod
def _parse_rss_date(date_str: Optional[str]) -> Optional[datetime]:
if not date_str:
return None
formats = [
"%a, %d %b %Y %H:%M:%S %z",
"%a, %d %b %Y %H:%M:%S %Z",
"%Y-%m-%dT%H:%M:%S%z",
"%Y-%m-%d %H:%M:%S",
]
for fmt in formats:
try:
dt = datetime.strptime(date_str.strip(), fmt)
if dt.tzinfo is not None:
return dt
return dt.replace(tzinfo=timezone.utc)
except ValueError:
continue
return None
@staticmethod
def _get_affected_pairs(currency: str) -> List[str]:
pairs = {
"USD": ["EURUSD=X", "GBPUSD=X", "USDJPY=X", "USDCHF=X", "AUDUSD=X", "USDCAD=X", "NZDUSD=X"],
"EUR": ["EURUSD=X", "EURGBP=X", "EURJPY=X", "EURCHF=X"],
"GBP": ["GBPUSD=X", "EURGBP=X", "GBPJPY=X"],
"JPY": ["USDJPY=X", "EURJPY=X", "GBPJPY=X", "AUDJPY=X", "CADJPY=X"],
"CHF": ["USDCHF=X", "EURCHF=X"],
"AUD": ["AUDUSD=X", "AUDJPY=X"],
"CAD": ["USDCAD=X", "CADJPY=X"],
"NZD": ["NZDUSD=X"],
}
return pairs.get(currency, [])
Reference in New Issue View Git Blame Copy Permalink