From ea306df48464c234f38b691a8a8b7a5110038d4a Mon Sep 17 00:00:00 2001 From: Harwinder Sekhon Date: Fri, 10 Jul 2026 09:43:11 -0400 Subject: [PATCH] Overhaul news engine: fix false signals, add surprise detection + caching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Improvements from a live-verified audit of the news/calendar pipeline: - Currency tagging: keyword matching now uses letter-boundary regexes and drops ambiguous "us "/"eu "/"uk " tokens — "versus" no longer tags headlines as USD (confirmed live false positive polluting the XAU/USD sentiment average). - Fetch caching: calendar events and headlines are cached class-level (10 min TTL, 60s retry after empty/failed fetch, lock-guarded for concurrent scans). Stops per-scan refetching that was tripping the Forex Factory rate limit; TTL stays under the 30-min pre-event buffer so the danger window can't go stale. - Surprise detection: EconomicEvent.surprise is computed from parsed actual vs forecast (handles %, K/M/B/T suffixes, negatives). A surprise print widens the post-release caution window to 45 min; inline releases keep the original 15 min. - Gold-aware sentiment: headlines are scored per currency, with monetary-policy keywords (hawkish/rate hike/tightening vs dovish/ rate cut/easing/stimulus) inverted for XAU — "hawkish Fed" headlines scored +1.0 for gold before, -1.0 now (verified on live feed). - New sentiment sources: Google News gold-focused query and CNBC Economy RSS (verified live; 40 → 74 unique headlines). Added cross-feed title dedup so syndicated stories count once. - events_today now buckets by UTC date (feed timestamps carry US- Eastern offsets); --json output includes news_description. - Removed dead NewsConfig.min_impact and unused HIGH_IMPACT_EVENTS. Tests: 142 passed (7 new covering boundary matching, surprise classification, post-event windows, cache reuse, gold inversion). --- config.py | 13 +++- main.py | 1 + news_engine.py | 200 ++++++++++++++++++++++++++++++++++++++++--------- test_all.py | 118 +++++++++++++++++++++++++++++ 4 files changed, 297 insertions(+), 35 deletions(-) diff --git a/config.py b/config.py index 599b668..3e7b17f 100755 --- a/config.py +++ b/config.py @@ -159,13 +159,24 @@ class SignalConfig: class NewsConfig: """Economic calendar & sentiment.""" tracked_currencies: List[str] = field(default_factory=lambda: ["USD", "XAU"]) - min_impact: str = "medium" pre_event_buffer_minutes: int = 30 post_event_buffer_minutes: int = 15 + # Wider stay-out window after a high-impact release that surprised + # vs forecast (inline releases use post_event_buffer_minutes). + post_event_surprise_buffer_minutes: int = 45 + # How long fetched calendar/headline data is reused across scans. + # Must stay below pre_event_buffer_minutes so an event can't cross + # into the danger window while a stale cache says it's clear. + context_cache_ttl_seconds: int = 600 rss_feeds: List[str] = field(default_factory=lambda: [ "https://investinglive.com/feed/", "https://www.fxstreet.com/rss", + # Google News query targeted at gold drivers — also carries Kitco, + # Reuters etc. whose direct feeds are unavailable. + "https://news.google.com/rss/search?q=gold+price+OR+fed+OR+inflation&hl=en-US&gl=US&ceid=US:en", + # CNBC "Economy" section + "https://search.cnbc.com/rs/search/combinedcms/view.xml?partnerId=wrss01&id=20910258", ]) diff --git a/main.py b/main.py index f6766f5..bf6965f 100755 --- a/main.py +++ b/main.py @@ -200,6 +200,7 @@ def report_to_dict(report: SignalReport) -> dict: "event_risk": report.event_risk, "event_description": report.event_description, "news_sentiment": round(report.news_sentiment, 3), + "news_description": report.news_description, "bullish_reasons": report.bullish_reasons, "bearish_reasons": report.bearish_reasons, "warnings": report.warnings, diff --git a/news_engine.py b/news_engine.py index 1d284f2..1b7c4f0 100755 --- a/news_engine.py +++ b/news_engine.py @@ -10,6 +10,7 @@ A perfect technical setup can be destroyed by an unexpected NFP print. import logging import re +import threading import time import xml.etree.ElementTree as ET from dataclasses import dataclass, field @@ -23,6 +24,21 @@ from config import NewsConfig logger = logging.getLogger(__name__) +_RELEASE_VALUE_RE = re.compile(r"(-?\d+(?:\.\d+)?)\s*([kmbt])?", re.IGNORECASE) +_RELEASE_MULTIPLIERS = {"k": 1e3, "m": 1e6, "b": 1e9, "t": 1e12} + + +def _parse_release_value(raw: Optional[str]) -> Optional[float]: + """Parse a calendar release value like '0.3%', '-1.7%', '228K', '1.1M'.""" + if not raw: + return None + match = _RELEASE_VALUE_RE.search(raw.replace(",", "")) + if not match: + return None + multiplier = _RELEASE_MULTIPLIERS.get((match.group(2) or "").lower(), 1.0) + return float(match.group(1)) * multiplier + + @dataclass class EconomicEvent: """Single economic calendar event.""" @@ -33,7 +49,21 @@ class EconomicEvent: forecast: Optional[str] = None previous: Optional[str] = None actual: Optional[str] = None - surprise: Optional[str] = None # "positive", "negative", "inline", None + + @property + def surprise(self) -> Optional[str]: + """Actual vs forecast: "positive"/"negative"/"inline", or None when + either value is missing or non-numeric. Direction is numeric only — + it does not know whether higher is market-bullish for this event.""" + actual = _parse_release_value(self.actual) + forecast = _parse_release_value(self.forecast) + if actual is None or forecast is None: + return None + if actual > forecast: + return "positive" + if actual < forecast: + return "negative" + return "inline" @property def is_upcoming(self) -> bool: @@ -58,6 +88,9 @@ class NewsSentiment: currencies_mentioned: List[str] sentiment_score: float # -1.0 (bearish) to +1.0 (bullish) keywords: List[str] + # Per-currency score: monetary-policy keywords flip sign for XAU + # (hawkish is dollar-bullish but gold-bearish). + sentiment_by_currency: Dict[str, float] = field(default_factory=dict) @dataclass @@ -105,29 +138,52 @@ class NewsEngine: "disappointing", "soft data", } - # Currency-specific keywords for headline parsing + # Monetary-policy keywords whose direction inverts for gold: tighter + # policy lifts the dollar and yields, weighing on non-yielding gold; + # easing does the opposite. Only unambiguous policy words — economic + # data words (beats/misses) keep their normal direction. + GOLD_INVERSE_KEYWORDS = { + "rate hike", "hawkish", "tightening", + "rate cut", "dovish", "easing", "stimulus", + } + + # Currency-specific keywords for headline parsing. Matched with word + # boundaries — bare "us"/"eu"/"uk" are deliberately excluded because as + # substrings they false-positive constantly ("versus" → USD). CURRENCY_KEYWORDS = { - "USD": ["dollar", "fed", "fomc", "treasury", "us ", "american", "nonfarm", "nfp"], - "EUR": ["euro", "ecb", "eurozone", "eu ", "german", "french"], - "GBP": ["pound", "sterling", "boe", "bank of england", "uk ", "british"], + "USD": ["dollar", "fed", "federal reserve", "fomc", "treasury", "american", "nonfarm", "nfp"], + "EUR": ["euro", "ecb", "eurozone", "german", "french"], + "GBP": ["pound", "sterling", "boe", "bank of england", "british"], "JPY": ["yen", "boj", "bank of japan", "japanese"], "CHF": ["franc", "snb", "swiss"], "AUD": ["aussie", "rba", "australian", "australia"], "CAD": ["loonie", "boc", "canadian", "canada"], "NZD": ["kiwi", "rbnz", "new zealand"], - "XAU": ["gold", "xau", "precious metal", "safe haven", "bullion"], + "XAU": ["gold", "precious metal", "safe haven", "bullion"], } - # High-impact event types (these move markets significantly) - HIGH_IMPACT_EVENTS = { - "interest rate decision", "nonfarm payrolls", "non-farm", - "gdp", "cpi", "inflation", "employment change", - "retail sales", "pmi", "fomc", "ecb press conference", - "boe rate", "rba rate", "snb rate", "boc rate", + # One compiled alternation per currency (code + keywords). (? -self.config.post_event_buffer_minutes: + if event.is_high_impact and event.currency in (base, quote) and not event.is_upcoming: + surprise = event.surprise + surprised = surprise in ("positive", "negative") + window = ( + self.config.post_event_surprise_buffer_minutes + if surprised + else self.config.post_event_buffer_minutes + ) + if event.minutes_until > -window: + if surprised: + return "caution", ( + f"{event.currency} {event.event} released — {surprise} surprise " + f"(actual {event.actual} vs forecast {event.forecast})" + ) return "caution", f"{event.currency} {event.event} just released" return "clear", "No significant events" @@ -272,7 +350,32 @@ class NewsEngine: "canada": "CAD", "new zealand": "NZD", } + def _cached_fetch(self, cache_attr: str, fetch_fn) -> list: + """Return cached data for `cache_attr` or refetch when the TTL + expired. Successful results live for context_cache_ttl_seconds; + empty results only _EMPTY_RESULT_TTL so outages retry quickly. + The lock is held across the fetch so concurrent scans share one + request instead of stampeding the source.""" + ts, data = getattr(NewsEngine, cache_attr) + ttl = self.config.context_cache_ttl_seconds if data else self._EMPTY_RESULT_TTL + if time.time() - ts < ttl: + return data + + with NewsEngine._cache_lock: + ts, data = getattr(NewsEngine, cache_attr) # re-check under lock + ttl = self.config.context_cache_ttl_seconds if data else self._EMPTY_RESULT_TTL + if time.time() - ts < ttl: + return data + data = fetch_fn() + setattr(NewsEngine, cache_attr, (time.time(), data)) + return data + def _fetch_economic_calendar(self, currencies: List[str]) -> List[EconomicEvent]: + """Cached calendar fetch, filtered to the tracked currencies.""" + events = self._cached_fetch("_calendar_cache", self._fetch_calendar_raw) + return [e for e in events if e.currency in currencies] + + def _fetch_calendar_raw(self) -> List[EconomicEvent]: """ Fetch economic calendar events from a free API. Tries Forex Factory first, falls back to MyFXBook RSS. @@ -283,7 +386,7 @@ class NewsEngine: try: events = fetcher() if events: - return [e for e in events if e.currency in currencies] + return events except Exception as e: logger.debug(f"Calendar fetch exception ({fetcher.__name__}): {e}") @@ -426,8 +529,13 @@ class NewsEngine: return events def _fetch_news_sentiment(self) -> List[NewsSentiment]: + """Cached headline fetch + sentiment analysis.""" + return self._cached_fetch("_headlines_cache", self._fetch_news_sentiment_raw) + + def _fetch_news_sentiment_raw(self) -> List[NewsSentiment]: """Fetch and analyze news headlines for sentiment.""" headlines = [] + seen_titles = set() for feed_url in self.config.rss_feeds: try: @@ -453,6 +561,14 @@ class NewsEngine: if pub_date and (datetime.now(timezone.utc) - pub_date).total_seconds() > 86400: continue + # Dedup within and across feeds so a syndicated story + # doesn't count twice in the sentiment average. Google + # News appends " - Publisher", so compare without it. + title_key = title.lower().rsplit(" - ", 1)[0].strip() + if title_key in seen_titles: + continue + seen_titles.add(title_key) + sentiment = self._analyze_headline_sentiment(title) currencies = self._extract_currencies(title) @@ -464,6 +580,10 @@ class NewsEngine: currencies_mentioned=currencies, sentiment_score=sentiment, keywords=self._extract_keywords(title), + sentiment_by_currency={ + cur: self._analyze_headline_sentiment(title, cur) + for cur in currencies + }, )) except Exception as e: @@ -471,19 +591,38 @@ class NewsEngine: return headlines - def _analyze_headline_sentiment(self, headline: str) -> float: + def _analyze_headline_sentiment( + self, headline: str, for_currency: Optional[str] = None + ) -> float: """ Simple keyword-based sentiment analysis. Returns -1.0 (very bearish) to +1.0 (very bullish). + When for_currency is "XAU", monetary-policy keywords contribute + with inverted sign (see GOLD_INVERSE_KEYWORDS). + Note: This is intentionally simple. NLP models like FinBERT would be more accurate but require large dependencies. This catches the most impactful sentiment signals from headlines. """ headline_lower = headline.lower() - bullish_score = sum(1 for kw in self.BULLISH_KEYWORDS if kw in headline_lower) - bearish_score = sum(1 for kw in self.BEARISH_KEYWORDS if kw in headline_lower) + invert_policy = for_currency == "XAU" + + bullish_score = 0 + bearish_score = 0 + for kw in self.BULLISH_KEYWORDS: + if kw in headline_lower: + if invert_policy and kw in self.GOLD_INVERSE_KEYWORDS: + bearish_score += 1 + else: + bullish_score += 1 + for kw in self.BEARISH_KEYWORDS: + if kw in headline_lower: + if invert_policy and kw in self.GOLD_INVERSE_KEYWORDS: + bullish_score += 1 + else: + bearish_score += 1 total = bullish_score + bearish_score if total == 0: @@ -494,18 +633,11 @@ class NewsEngine: def _extract_currencies(self, headline: str) -> List[str]: """Extract mentioned currencies from a headline.""" headline_lower = headline.lower() - found = [] - - for currency, keywords in self.CURRENCY_KEYWORDS.items(): - if currency.lower() in headline_lower: - found.append(currency) - continue - for kw in keywords: - if kw in headline_lower: - found.append(currency) - break - - return list(set(found)) + return [ + currency + for currency, pattern in self._CURRENCY_PATTERNS.items() + if pattern.search(headline_lower) + ] def _extract_keywords(self, headline: str) -> List[str]: """Extract relevant trading keywords from headline.""" diff --git a/test_all.py b/test_all.py index 96d8899..e0b3bed 100644 --- a/test_all.py +++ b/test_all.py @@ -953,6 +953,124 @@ def _(): assert "neutral" in desc.lower() +@test("NewsEngine._extract_currencies no USD false positive from 'versus'") +def _(): + ne = NewsEngine(NewsConfig()) + currencies = ne._extract_currencies( + "Canada building permits for May -1.7% versus 2.4% estimate" + ) + assert "USD" not in currencies, f"'versus' must not tag USD, got {currencies}" + assert "CAD" in currencies + + +@test("NewsEngine._extract_currencies no AUD false positive from 'audit'") +def _(): + ne = NewsEngine(NewsConfig()) + currencies = ne._extract_currencies("Treasury audit finds accounting errors") + assert "AUD" not in currencies + + +@test("EconomicEvent.surprise from actual vs forecast") +def _(): + now = datetime.now(timezone.utc) + beat = EconomicEvent(time=now, currency="USD", impact="high", + event="NFP", forecast="900K", actual="1.1M") + assert beat.surprise == "positive" + + miss = EconomicEvent(time=now, currency="USD", impact="high", + event="CPI y/y", forecast="0.3%", actual="0.2%") + assert miss.surprise == "negative" + + inline = EconomicEvent(time=now, currency="USD", impact="high", + event="CPI y/y", forecast="0.3%", actual="0.3%") + assert inline.surprise == "inline" + + pending = EconomicEvent(time=now, currency="USD", impact="high", + event="CPI y/y", forecast="0.3%") + assert pending.surprise is None + + negative = EconomicEvent(time=now, currency="USD", impact="high", + event="Permits m/m", forecast="2.4%", actual="-1.7%") + assert negative.surprise == "negative" + + +@test("NewsEngine.get_event_risk_level surprise widens post-event window") +def _(): + ne = NewsEngine(NewsConfig()) + released_at = datetime.now(timezone.utc) - timedelta(minutes=30) + + # Surprise print 30 min ago: outside the 15-min inline buffer but + # inside the 45-min surprise buffer → still caution. + context = FundamentalContext() + context.events_today = [EconomicEvent( + time=released_at, currency="USD", impact="high", + event="CPI m/m", forecast="0.3%", actual="0.5%", + )] + risk, desc = ne.get_event_risk_level("XAU/USD", context) + assert risk == "caution", f"Expected caution after surprise, got {risk}" + assert "surprise" in desc + + # Same timing but inline print → clear. + context.events_today = [EconomicEvent( + time=released_at, currency="USD", impact="high", + event="CPI m/m", forecast="0.3%", actual="0.3%", + )] + risk, _ = ne.get_event_risk_level("XAU/USD", context) + assert risk == "clear", f"Inline release 30min ago should be clear, got {risk}" + + +@test("NewsEngine gold-inverted sentiment for policy keywords") +def _(): + ne = NewsEngine(NewsConfig()) + headline = "Fed signals rate hike as hawkish stance hardens" + # Currency-bullish for the dollar... + assert ne._analyze_headline_sentiment(headline) > 0 + assert ne._analyze_headline_sentiment(headline, "USD") > 0 + # ...but bearish for gold. + assert ne._analyze_headline_sentiment(headline, "XAU") < 0 + + dovish = "Fed turns dovish, rate cut expected" + assert ne._analyze_headline_sentiment(dovish) < 0 + assert ne._analyze_headline_sentiment(dovish, "XAU") > 0 + + # Non-policy keywords keep their direction for gold. + data = "Gold surges to record high" + assert ne._analyze_headline_sentiment(data, "XAU") > 0 + + +@test("NewsEngine calendar cache avoids refetch within TTL") +def _(): + NewsEngine._calendar_cache = (0.0, []) + try: + ne = NewsEngine(NewsConfig()) + calls = {"n": 0} + + def fake_fetch(): + calls["n"] += 1 + return [EconomicEvent( + time=datetime.now(timezone.utc), currency="USD", + impact="low", event="Test Event", + )] + + ne._try_fetch_calendar_api = fake_fetch + ne._try_fetch_myfxbook_rss = lambda: [] + + first = ne._fetch_economic_calendar(["USD", "XAU"]) + second = ne._fetch_economic_calendar(["USD", "XAU"]) + assert calls["n"] == 1, f"Expected 1 fetch, got {calls['n']}" + assert len(first) == 1 and len(second) == 1 + + # A second engine instance (new scan) must also hit the cache. + ne2 = NewsEngine(NewsConfig()) + ne2._try_fetch_calendar_api = fake_fetch + ne2._try_fetch_myfxbook_rss = lambda: [] + third = ne2._fetch_economic_calendar(["USD", "XAU"]) + assert calls["n"] == 1, "New engine instance should reuse class-level cache" + assert len(third) == 1 + finally: + NewsEngine._calendar_cache = (0.0, []) + + @test("FundamentalContext defaults") def _(): ctx = FundamentalContext()