Overhaul news engine: fix false signals, add surprise detection + caching

Improvements from a live-verified audit of the news/calendar pipeline:

- Currency tagging: keyword matching now uses letter-boundary regexes
  and drops ambiguous "us "/"eu "/"uk " tokens — "versus" no longer
  tags headlines as USD (confirmed live false positive polluting the
  XAU/USD sentiment average).

- Fetch caching: calendar events and headlines are cached class-level
  (10 min TTL, 60s retry after empty/failed fetch, lock-guarded for
  concurrent scans). Stops per-scan refetching that was tripping the
  Forex Factory rate limit; TTL stays under the 30-min pre-event
  buffer so the danger window can't go stale.

- Surprise detection: EconomicEvent.surprise is computed from parsed
  actual vs forecast (handles %, K/M/B/T suffixes, negatives). A
  surprise print widens the post-release caution window to 45 min;
  inline releases keep the original 15 min.

- Gold-aware sentiment: headlines are scored per currency, with
  monetary-policy keywords (hawkish/rate hike/tightening vs dovish/
  rate cut/easing/stimulus) inverted for XAU — "hawkish Fed" headlines
  scored +1.0 for gold before, -1.0 now (verified on live feed).

- New sentiment sources: Google News gold-focused query and CNBC
  Economy RSS (verified live; 40 → 74 unique headlines). Added
  cross-feed title dedup so syndicated stories count once.

- events_today now buckets by UTC date (feed timestamps carry US-
  Eastern offsets); --json output includes news_description.

- Removed dead NewsConfig.min_impact and unused HIGH_IMPACT_EVENTS.

Tests: 142 passed (7 new covering boundary matching, surprise
classification, post-event windows, cache reuse, gold inversion).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

config.py

@@ -159,13 +159,24 @@ class SignalConfig:
 class NewsConfig:
     """Economic calendar & sentiment."""
     tracked_currencies: List[str] = field(default_factory=lambda: ["USD", "XAU"])
-    min_impact: str = "medium"
     pre_event_buffer_minutes: int = 30
     post_event_buffer_minutes: int = 15
+    # Wider stay-out window after a high-impact release that surprised
+    # vs forecast (inline releases use post_event_buffer_minutes).
+    post_event_surprise_buffer_minutes: int = 45
+    # How long fetched calendar/headline data is reused across scans.
+    # Must stay below pre_event_buffer_minutes so an event can't cross
+    # into the danger window while a stale cache says it's clear.
+    context_cache_ttl_seconds: int = 600
 
     rss_feeds: List[str] = field(default_factory=lambda: [
         "https://investinglive.com/feed/",
         "https://www.fxstreet.com/rss",
+        # Google News query targeted at gold drivers — also carries Kitco,
+        # Reuters etc. whose direct feeds are unavailable.
+        "https://news.google.com/rss/search?q=gold+price+OR+fed+OR+inflation&hl=en-US&gl=US&ceid=US:en",
+        # CNBC "Economy" section
+        "https://search.cnbc.com/rs/search/combinedcms/view.xml?partnerId=wrss01&id=20910258",
     ])
 
 

main.py

@@ -200,6 +200,7 @@ def report_to_dict(report: SignalReport) -> dict:
         "event_risk": report.event_risk,
         "event_description": report.event_description,
         "news_sentiment": round(report.news_sentiment, 3),
+        "news_description": report.news_description,
         "bullish_reasons": report.bullish_reasons,
         "bearish_reasons": report.bearish_reasons,
         "warnings": report.warnings,

news_engine.py

@@ -10,6 +10,7 @@ A perfect technical setup can be destroyed by an unexpected NFP print.
 
 import logging
 import re
+import threading
 import time
 import xml.etree.ElementTree as ET
 from dataclasses import dataclass, field
@@ -23,6 +24,21 @@ from config import NewsConfig
 logger = logging.getLogger(__name__)
 
 
+_RELEASE_VALUE_RE = re.compile(r"(-?\d+(?:\.\d+)?)\s*([kmbt])?", re.IGNORECASE)
+_RELEASE_MULTIPLIERS = {"k": 1e3, "m": 1e6, "b": 1e9, "t": 1e12}
+
+
+def _parse_release_value(raw: Optional[str]) -> Optional[float]:
+    """Parse a calendar release value like '0.3%', '-1.7%', '228K', '1.1M'."""
+    if not raw:
+        return None
+    match = _RELEASE_VALUE_RE.search(raw.replace(",", ""))
+    if not match:
+        return None
+    multiplier = _RELEASE_MULTIPLIERS.get((match.group(2) or "").lower(), 1.0)
+    return float(match.group(1)) * multiplier
+
+
 @dataclass
 class EconomicEvent:
     """Single economic calendar event."""
@@ -33,7 +49,21 @@ class EconomicEvent:
     forecast: Optional[str] = None
     previous: Optional[str] = None
     actual: Optional[str] = None
-    surprise: Optional[str] = None  # "positive", "negative", "inline", None
+
+    @property
+    def surprise(self) -> Optional[str]:
+        """Actual vs forecast: "positive"/"negative"/"inline", or None when
+        either value is missing or non-numeric. Direction is numeric only —
+        it does not know whether higher is market-bullish for this event."""
+        actual = _parse_release_value(self.actual)
+        forecast = _parse_release_value(self.forecast)
+        if actual is None or forecast is None:
+            return None
+        if actual > forecast:
+            return "positive"
+        if actual < forecast:
+            return "negative"
+        return "inline"
 
     @property
     def is_upcoming(self) -> bool:
@@ -58,6 +88,9 @@ class NewsSentiment:
     currencies_mentioned: List[str]
     sentiment_score: float  # -1.0 (bearish) to +1.0 (bullish)
     keywords: List[str]
+    # Per-currency score: monetary-policy keywords flip sign for XAU
+    # (hawkish is dollar-bullish but gold-bearish).
+    sentiment_by_currency: Dict[str, float] = field(default_factory=dict)
 
 
 @dataclass
@@ -105,29 +138,52 @@ class NewsEngine:
         "disappointing", "soft data",
     }
 
-    # Currency-specific keywords for headline parsing
+    # Monetary-policy keywords whose direction inverts for gold: tighter
+    # policy lifts the dollar and yields, weighing on non-yielding gold;
+    # easing does the opposite. Only unambiguous policy words — economic
+    # data words (beats/misses) keep their normal direction.
+    GOLD_INVERSE_KEYWORDS = {
+        "rate hike", "hawkish", "tightening",
+        "rate cut", "dovish", "easing", "stimulus",
+    }
+
+    # Currency-specific keywords for headline parsing. Matched with word
+    # boundaries — bare "us"/"eu"/"uk" are deliberately excluded because as
+    # substrings they false-positive constantly ("versus" → USD).
     CURRENCY_KEYWORDS = {
-        "USD": ["dollar", "fed", "fomc", "treasury", "us ", "american", "nonfarm", "nfp"],
+        "USD": ["dollar", "fed", "federal reserve", "fomc", "treasury", "american", "nonfarm", "nfp"],
-        "EUR": ["euro", "ecb", "eurozone", "eu ", "german", "french"],
+        "EUR": ["euro", "ecb", "eurozone", "german", "french"],
-        "GBP": ["pound", "sterling", "boe", "bank of england", "uk ", "british"],
+        "GBP": ["pound", "sterling", "boe", "bank of england", "british"],
         "JPY": ["yen", "boj", "bank of japan", "japanese"],
         "CHF": ["franc", "snb", "swiss"],
         "AUD": ["aussie", "rba", "australian", "australia"],
         "CAD": ["loonie", "boc", "canadian", "canada"],
         "NZD": ["kiwi", "rbnz", "new zealand"],
-        "XAU": ["gold", "xau", "precious metal", "safe haven", "bullion"],
+        "XAU": ["gold", "precious metal", "safe haven", "bullion"],
     }
 
-    # High-impact event types (these move markets significantly)
+    # One compiled alternation per currency (code + keywords). (?<![a-z]) /
-    HIGH_IMPACT_EVENTS = {
+    # (?![a-z]) instead of \b so "0.3%fed" still matches but "versus",
-        "interest rate decision", "nonfarm payrolls", "non-farm",
+    # "audit", "cascade" don't.
-        "gdp", "cpi", "inflation", "employment change",
+    _CURRENCY_PATTERNS = {
-        "retail sales", "pmi", "fomc", "ecb press conference",
+        cur: re.compile(
-        "boe rate", "rba rate", "snb rate", "boc rate",
+            r"(?<![a-z])(?:"
+            + "|".join(re.escape(kw) for kw in [cur.lower()] + kws)
+            + r")(?![a-z])"
+        )
+        for cur, kws in CURRENCY_KEYWORDS.items()
     }
 
     _calendar_backoff_until: float = 0.0
 
+    # Fetched-data cache, class-level: a fresh NewsEngine is built per scan,
+    # so instance state would never be reused. Guarded by _cache_lock since
+    # scans run concurrently in a thread pool.
+    _cache_lock = threading.Lock()
+    _calendar_cache: Tuple[float, List[EconomicEvent]] = (0.0, [])
+    _headlines_cache: Tuple[float, List[NewsSentiment]] = (0.0, [])
+    _EMPTY_RESULT_TTL = 60.0  # retry sooner after a failed/empty fetch
+
     def __init__(self, config: NewsConfig):
         self.config = config
         self._session = requests.Session()
@@ -153,7 +209,13 @@ class NewsEngine:
             events = self._fetch_economic_calendar(currencies)
             now = datetime.now(timezone.utc)
 
-            context.events_today = [e for e in events if e.time.date() == now.date()]
+            # Compare dates in UTC — feed timestamps carry their own offset
+            # (FF sends US-Eastern), so a late-evening ET event would
+            # otherwise land on the wrong day.
+            context.events_today = [
+                e for e in events
+                if e.time.astimezone(timezone.utc).date() == now.date()
+            ]
             context.events_upcoming_1h = [
                 e for e in events
                 if 0 < e.minutes_until <= 60
@@ -187,10 +249,12 @@ class NewsEngine:
             headlines = self._fetch_news_sentiment()
             context.news_headlines = headlines
 
-            # Aggregate sentiment per currency
+            # Aggregate sentiment per currency (per-currency score where
+            # available, so gold-inverted policy keywords are respected)
             for currency in currencies:
                 scores = [
-                    h.sentiment_score for h in headlines
+                    h.sentiment_by_currency.get(currency, h.sentiment_score)
+                    for h in headlines
                     if currency in h.currencies_mentioned
                 ]
                 if scores:
@@ -226,10 +290,24 @@ class NewsEngine:
                 else:
                     return "caution", f"{event.currency} {event.event} in {mins:.0f}min"
 
-        # Check for recent high-impact releases (within post-event buffer)
+        # Check for recent high-impact releases. A surprise print (actual
+        # away from forecast) keeps the market unstable longer than an
+        # inline release, so it gets a wider post-event window.
         for event in context.events_today:
-            if event.is_high_impact and event.currency in (base, quote):
+            if event.is_high_impact and event.currency in (base, quote) and not event.is_upcoming:
-                if not event.is_upcoming and event.minutes_until > -self.config.post_event_buffer_minutes:
+                surprise = event.surprise
+                surprised = surprise in ("positive", "negative")
+                window = (
+                    self.config.post_event_surprise_buffer_minutes
+                    if surprised
+                    else self.config.post_event_buffer_minutes
+                )
+                if event.minutes_until > -window:
+                    if surprised:
+                        return "caution", (
+                            f"{event.currency} {event.event} released — {surprise} surprise "
+                            f"(actual {event.actual} vs forecast {event.forecast})"
+                        )
                     return "caution", f"{event.currency} {event.event} just released"
 
         return "clear", "No significant events"
@@ -272,7 +350,32 @@ class NewsEngine:
         "canada": "CAD", "new zealand": "NZD",
     }
 
+    def _cached_fetch(self, cache_attr: str, fetch_fn) -> list:
+        """Return cached data for `cache_attr` or refetch when the TTL
+        expired. Successful results live for context_cache_ttl_seconds;
+        empty results only _EMPTY_RESULT_TTL so outages retry quickly.
+        The lock is held across the fetch so concurrent scans share one
+        request instead of stampeding the source."""
+        ts, data = getattr(NewsEngine, cache_attr)
+        ttl = self.config.context_cache_ttl_seconds if data else self._EMPTY_RESULT_TTL
+        if time.time() - ts < ttl:
+            return data
+
+        with NewsEngine._cache_lock:
+            ts, data = getattr(NewsEngine, cache_attr)  # re-check under lock
+            ttl = self.config.context_cache_ttl_seconds if data else self._EMPTY_RESULT_TTL
+            if time.time() - ts < ttl:
+                return data
+            data = fetch_fn()
+            setattr(NewsEngine, cache_attr, (time.time(), data))
+            return data
+
     def _fetch_economic_calendar(self, currencies: List[str]) -> List[EconomicEvent]:
+        """Cached calendar fetch, filtered to the tracked currencies."""
+        events = self._cached_fetch("_calendar_cache", self._fetch_calendar_raw)
+        return [e for e in events if e.currency in currencies]
+
+    def _fetch_calendar_raw(self) -> List[EconomicEvent]:
         """
         Fetch economic calendar events from a free API.
         Tries Forex Factory first, falls back to MyFXBook RSS.
@@ -283,7 +386,7 @@ class NewsEngine:
             try:
                 events = fetcher()
                 if events:
-                    return [e for e in events if e.currency in currencies]
+                    return events
             except Exception as e:
                 logger.debug(f"Calendar fetch exception ({fetcher.__name__}): {e}")
 
@@ -426,8 +529,13 @@ class NewsEngine:
         return events
 
     def _fetch_news_sentiment(self) -> List[NewsSentiment]:
+        """Cached headline fetch + sentiment analysis."""
+        return self._cached_fetch("_headlines_cache", self._fetch_news_sentiment_raw)
+
+    def _fetch_news_sentiment_raw(self) -> List[NewsSentiment]:
         """Fetch and analyze news headlines for sentiment."""
         headlines = []
+        seen_titles = set()
 
         for feed_url in self.config.rss_feeds:
             try:
@@ -453,6 +561,14 @@ class NewsEngine:
                     if pub_date and (datetime.now(timezone.utc) - pub_date).total_seconds() > 86400:
                         continue
 
+                    # Dedup within and across feeds so a syndicated story
+                    # doesn't count twice in the sentiment average. Google
+                    # News appends " - Publisher", so compare without it.
+                    title_key = title.lower().rsplit(" - ", 1)[0].strip()
+                    if title_key in seen_titles:
+                        continue
+                    seen_titles.add(title_key)
+
                     sentiment = self._analyze_headline_sentiment(title)
                     currencies = self._extract_currencies(title)
 
@@ -464,6 +580,10 @@ class NewsEngine:
                             currencies_mentioned=currencies,
                             sentiment_score=sentiment,
                             keywords=self._extract_keywords(title),
+                            sentiment_by_currency={
+                                cur: self._analyze_headline_sentiment(title, cur)
+                                for cur in currencies
+                            },
                         ))
 
             except Exception as e:
@@ -471,19 +591,38 @@ class NewsEngine:
 
         return headlines
 
-    def _analyze_headline_sentiment(self, headline: str) -> float:
+    def _analyze_headline_sentiment(
+        self, headline: str, for_currency: Optional[str] = None
+    ) -> float:
         """
         Simple keyword-based sentiment analysis.
 
         Returns -1.0 (very bearish) to +1.0 (very bullish).
 
+        When for_currency is "XAU", monetary-policy keywords contribute
+        with inverted sign (see GOLD_INVERSE_KEYWORDS).
+
         Note: This is intentionally simple. NLP models like FinBERT
         would be more accurate but require large dependencies. This catches
         the most impactful sentiment signals from headlines.
         """
         headline_lower = headline.lower()
-        bullish_score = sum(1 for kw in self.BULLISH_KEYWORDS if kw in headline_lower)
+        invert_policy = for_currency == "XAU"
-        bearish_score = sum(1 for kw in self.BEARISH_KEYWORDS if kw in headline_lower)
+
+        bullish_score = 0
+        bearish_score = 0
+        for kw in self.BULLISH_KEYWORDS:
+            if kw in headline_lower:
+                if invert_policy and kw in self.GOLD_INVERSE_KEYWORDS:
+                    bearish_score += 1
+                else:
+                    bullish_score += 1
+        for kw in self.BEARISH_KEYWORDS:
+            if kw in headline_lower:
+                if invert_policy and kw in self.GOLD_INVERSE_KEYWORDS:
+                    bullish_score += 1
+                else:
+                    bearish_score += 1
 
         total = bullish_score + bearish_score
         if total == 0:
@@ -494,18 +633,11 @@ class NewsEngine:
     def _extract_currencies(self, headline: str) -> List[str]:
         """Extract mentioned currencies from a headline."""
         headline_lower = headline.lower()
-        found = []
+        return [
-
+            currency
-        for currency, keywords in self.CURRENCY_KEYWORDS.items():
+            for currency, pattern in self._CURRENCY_PATTERNS.items()
-            if currency.lower() in headline_lower:
+            if pattern.search(headline_lower)
-                found.append(currency)
+        ]
-                continue
-            for kw in keywords:
-                if kw in headline_lower:
-                    found.append(currency)
-                    break
-
-        return list(set(found))
 
     def _extract_keywords(self, headline: str) -> List[str]:
         """Extract relevant trading keywords from headline."""

test_all.py

@@ -953,6 +953,124 @@ def _():
     assert "neutral" in desc.lower()
 
 
+@test("NewsEngine._extract_currencies no USD false positive from 'versus'")
+def _():
+    ne = NewsEngine(NewsConfig())
+    currencies = ne._extract_currencies(
+        "Canada building permits for May -1.7% versus 2.4% estimate"
+    )
+    assert "USD" not in currencies, f"'versus' must not tag USD, got {currencies}"
+    assert "CAD" in currencies
+
+
+@test("NewsEngine._extract_currencies no AUD false positive from 'audit'")
+def _():
+    ne = NewsEngine(NewsConfig())
+    currencies = ne._extract_currencies("Treasury audit finds accounting errors")
+    assert "AUD" not in currencies
+
+
+@test("EconomicEvent.surprise from actual vs forecast")
+def _():
+    now = datetime.now(timezone.utc)
+    beat = EconomicEvent(time=now, currency="USD", impact="high",
+                         event="NFP", forecast="900K", actual="1.1M")
+    assert beat.surprise == "positive"
+
+    miss = EconomicEvent(time=now, currency="USD", impact="high",
+                         event="CPI y/y", forecast="0.3%", actual="0.2%")
+    assert miss.surprise == "negative"
+
+    inline = EconomicEvent(time=now, currency="USD", impact="high",
+                           event="CPI y/y", forecast="0.3%", actual="0.3%")
+    assert inline.surprise == "inline"
+
+    pending = EconomicEvent(time=now, currency="USD", impact="high",
+                            event="CPI y/y", forecast="0.3%")
+    assert pending.surprise is None
+
+    negative = EconomicEvent(time=now, currency="USD", impact="high",
+                             event="Permits m/m", forecast="2.4%", actual="-1.7%")
+    assert negative.surprise == "negative"
+
+
+@test("NewsEngine.get_event_risk_level surprise widens post-event window")
+def _():
+    ne = NewsEngine(NewsConfig())
+    released_at = datetime.now(timezone.utc) - timedelta(minutes=30)
+
+    # Surprise print 30 min ago: outside the 15-min inline buffer but
+    # inside the 45-min surprise buffer → still caution.
+    context = FundamentalContext()
+    context.events_today = [EconomicEvent(
+        time=released_at, currency="USD", impact="high",
+        event="CPI m/m", forecast="0.3%", actual="0.5%",
+    )]
+    risk, desc = ne.get_event_risk_level("XAU/USD", context)
+    assert risk == "caution", f"Expected caution after surprise, got {risk}"
+    assert "surprise" in desc
+
+    # Same timing but inline print → clear.
+    context.events_today = [EconomicEvent(
+        time=released_at, currency="USD", impact="high",
+        event="CPI m/m", forecast="0.3%", actual="0.3%",
+    )]
+    risk, _ = ne.get_event_risk_level("XAU/USD", context)
+    assert risk == "clear", f"Inline release 30min ago should be clear, got {risk}"
+
+
+@test("NewsEngine gold-inverted sentiment for policy keywords")
+def _():
+    ne = NewsEngine(NewsConfig())
+    headline = "Fed signals rate hike as hawkish stance hardens"
+    # Currency-bullish for the dollar...
+    assert ne._analyze_headline_sentiment(headline) > 0
+    assert ne._analyze_headline_sentiment(headline, "USD") > 0
+    # ...but bearish for gold.
+    assert ne._analyze_headline_sentiment(headline, "XAU") < 0
+
+    dovish = "Fed turns dovish, rate cut expected"
+    assert ne._analyze_headline_sentiment(dovish) < 0
+    assert ne._analyze_headline_sentiment(dovish, "XAU") > 0
+
+    # Non-policy keywords keep their direction for gold.
+    data = "Gold surges to record high"
+    assert ne._analyze_headline_sentiment(data, "XAU") > 0
+
+
+@test("NewsEngine calendar cache avoids refetch within TTL")
+def _():
+    NewsEngine._calendar_cache = (0.0, [])
+    try:
+        ne = NewsEngine(NewsConfig())
+        calls = {"n": 0}
+
+        def fake_fetch():
+            calls["n"] += 1
+            return [EconomicEvent(
+                time=datetime.now(timezone.utc), currency="USD",
+                impact="low", event="Test Event",
+            )]
+
+        ne._try_fetch_calendar_api = fake_fetch
+        ne._try_fetch_myfxbook_rss = lambda: []
+
+        first = ne._fetch_economic_calendar(["USD", "XAU"])
+        second = ne._fetch_economic_calendar(["USD", "XAU"])
+        assert calls["n"] == 1, f"Expected 1 fetch, got {calls['n']}"
+        assert len(first) == 1 and len(second) == 1
+
+        # A second engine instance (new scan) must also hit the cache.
+        ne2 = NewsEngine(NewsConfig())
+        ne2._try_fetch_calendar_api = fake_fetch
+        ne2._try_fetch_myfxbook_rss = lambda: []
+        third = ne2._fetch_economic_calendar(["USD", "XAU"])
+        assert calls["n"] == 1, "New engine instance should reuse class-level cache"
+        assert len(third) == 1
+    finally:
+        NewsEngine._calendar_cache = (0.0, [])
+
+
 @test("FundamentalContext defaults")
 def _():
     ctx = FundamentalContext()