Overhaul news engine: fix false signals, add surprise detection + caching

Improvements from a live-verified audit of the news/calendar pipeline:

- Currency tagging: keyword matching now uses letter-boundary regexes
  and drops ambiguous "us "/"eu "/"uk " tokens — "versus" no longer
  tags headlines as USD (confirmed live false positive polluting the
  XAU/USD sentiment average).

- Fetch caching: calendar events and headlines are cached class-level
  (10 min TTL, 60s retry after empty/failed fetch, lock-guarded for
  concurrent scans). Stops per-scan refetching that was tripping the
  Forex Factory rate limit; TTL stays under the 30-min pre-event
  buffer so the danger window can't go stale.

- Surprise detection: EconomicEvent.surprise is computed from parsed
  actual vs forecast (handles %, K/M/B/T suffixes, negatives). A
  surprise print widens the post-release caution window to 45 min;
  inline releases keep the original 15 min.

- Gold-aware sentiment: headlines are scored per currency, with
  monetary-policy keywords (hawkish/rate hike/tightening vs dovish/
  rate cut/easing/stimulus) inverted for XAU — "hawkish Fed" headlines
  scored +1.0 for gold before, -1.0 now (verified on live feed).

- New sentiment sources: Google News gold-focused query and CNBC
  Economy RSS (verified live; 40 → 74 unique headlines). Added
  cross-feed title dedup so syndicated stories count once.

- events_today now buckets by UTC date (feed timestamps carry US-
  Eastern offsets); --json output includes news_description.

- Removed dead NewsConfig.min_impact and unused HIGH_IMPACT_EVENTS.

Tests: 142 passed (7 new covering boundary matching, surprise
classification, post-event windows, cache reuse, gold inversion).
This commit is contained in:
Harwinder Sekhon
2026-07-10 09:43:11 -04:00
parent 1038a53bac
commit ea306df484
4 changed files with 297 additions and 35 deletions
+118
View File
@@ -953,6 +953,124 @@ def _():
assert "neutral" in desc.lower()
@test("NewsEngine._extract_currencies no USD false positive from 'versus'")
def _():
ne = NewsEngine(NewsConfig())
currencies = ne._extract_currencies(
"Canada building permits for May -1.7% versus 2.4% estimate"
)
assert "USD" not in currencies, f"'versus' must not tag USD, got {currencies}"
assert "CAD" in currencies
@test("NewsEngine._extract_currencies no AUD false positive from 'audit'")
def _():
ne = NewsEngine(NewsConfig())
currencies = ne._extract_currencies("Treasury audit finds accounting errors")
assert "AUD" not in currencies
@test("EconomicEvent.surprise from actual vs forecast")
def _():
now = datetime.now(timezone.utc)
beat = EconomicEvent(time=now, currency="USD", impact="high",
event="NFP", forecast="900K", actual="1.1M")
assert beat.surprise == "positive"
miss = EconomicEvent(time=now, currency="USD", impact="high",
event="CPI y/y", forecast="0.3%", actual="0.2%")
assert miss.surprise == "negative"
inline = EconomicEvent(time=now, currency="USD", impact="high",
event="CPI y/y", forecast="0.3%", actual="0.3%")
assert inline.surprise == "inline"
pending = EconomicEvent(time=now, currency="USD", impact="high",
event="CPI y/y", forecast="0.3%")
assert pending.surprise is None
negative = EconomicEvent(time=now, currency="USD", impact="high",
event="Permits m/m", forecast="2.4%", actual="-1.7%")
assert negative.surprise == "negative"
@test("NewsEngine.get_event_risk_level surprise widens post-event window")
def _():
ne = NewsEngine(NewsConfig())
released_at = datetime.now(timezone.utc) - timedelta(minutes=30)
# Surprise print 30 min ago: outside the 15-min inline buffer but
# inside the 45-min surprise buffer → still caution.
context = FundamentalContext()
context.events_today = [EconomicEvent(
time=released_at, currency="USD", impact="high",
event="CPI m/m", forecast="0.3%", actual="0.5%",
)]
risk, desc = ne.get_event_risk_level("XAU/USD", context)
assert risk == "caution", f"Expected caution after surprise, got {risk}"
assert "surprise" in desc
# Same timing but inline print → clear.
context.events_today = [EconomicEvent(
time=released_at, currency="USD", impact="high",
event="CPI m/m", forecast="0.3%", actual="0.3%",
)]
risk, _ = ne.get_event_risk_level("XAU/USD", context)
assert risk == "clear", f"Inline release 30min ago should be clear, got {risk}"
@test("NewsEngine gold-inverted sentiment for policy keywords")
def _():
ne = NewsEngine(NewsConfig())
headline = "Fed signals rate hike as hawkish stance hardens"
# Currency-bullish for the dollar...
assert ne._analyze_headline_sentiment(headline) > 0
assert ne._analyze_headline_sentiment(headline, "USD") > 0
# ...but bearish for gold.
assert ne._analyze_headline_sentiment(headline, "XAU") < 0
dovish = "Fed turns dovish, rate cut expected"
assert ne._analyze_headline_sentiment(dovish) < 0
assert ne._analyze_headline_sentiment(dovish, "XAU") > 0
# Non-policy keywords keep their direction for gold.
data = "Gold surges to record high"
assert ne._analyze_headline_sentiment(data, "XAU") > 0
@test("NewsEngine calendar cache avoids refetch within TTL")
def _():
NewsEngine._calendar_cache = (0.0, [])
try:
ne = NewsEngine(NewsConfig())
calls = {"n": 0}
def fake_fetch():
calls["n"] += 1
return [EconomicEvent(
time=datetime.now(timezone.utc), currency="USD",
impact="low", event="Test Event",
)]
ne._try_fetch_calendar_api = fake_fetch
ne._try_fetch_myfxbook_rss = lambda: []
first = ne._fetch_economic_calendar(["USD", "XAU"])
second = ne._fetch_economic_calendar(["USD", "XAU"])
assert calls["n"] == 1, f"Expected 1 fetch, got {calls['n']}"
assert len(first) == 1 and len(second) == 1
# A second engine instance (new scan) must also hit the cache.
ne2 = NewsEngine(NewsConfig())
ne2._try_fetch_calendar_api = fake_fetch
ne2._try_fetch_myfxbook_rss = lambda: []
third = ne2._fetch_economic_calendar(["USD", "XAU"])
assert calls["n"] == 1, "New engine instance should reuse class-level cache"
assert len(third) == 1
finally:
NewsEngine._calendar_cache = (0.0, [])
@test("FundamentalContext defaults")
def _():
ctx = FundamentalContext()