harry/ForexBot2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Overhaul news engine: fix false signals, add surprise detection + caching

Browse Source 
Improvements from a live-verified audit of the news/calendar pipeline:

- Currency tagging: keyword matching now uses letter-boundary regexes
  and drops ambiguous "us "/"eu "/"uk " tokens — "versus" no longer
  tags headlines as USD (confirmed live false positive polluting the
  XAU/USD sentiment average).

- Fetch caching: calendar events and headlines are cached class-level
  (10 min TTL, 60s retry after empty/failed fetch, lock-guarded for
  concurrent scans). Stops per-scan refetching that was tripping the
  Forex Factory rate limit; TTL stays under the 30-min pre-event
  buffer so the danger window can't go stale.

- Surprise detection: EconomicEvent.surprise is computed from parsed
  actual vs forecast (handles %, K/M/B/T suffixes, negatives). A
  surprise print widens the post-release caution window to 45 min;
  inline releases keep the original 15 min.

- Gold-aware sentiment: headlines are scored per currency, with
  monetary-policy keywords (hawkish/rate hike/tightening vs dovish/
  rate cut/easing/stimulus) inverted for XAU — "hawkish Fed" headlines
  scored +1.0 for gold before, -1.0 now (verified on live feed).

- New sentiment sources: Google News gold-focused query and CNBC
  Economy RSS (verified live; 40 → 74 unique headlines). Added
  cross-feed title dedup so syndicated stories count once.

- events_today now buckets by UTC date (feed timestamps carry US-
  Eastern offsets); --json output includes news_description.

- Removed dead NewsConfig.min_impact and unused HIGH_IMPACT_EVENTS.

Tests: 142 passed (7 new covering boundary matching, surprise
classification, post-event windows, cache reuse, gold inversion).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Harwinder Sekhon
2026-07-10 09:43:11 -04:00
parent dad2432d4d
commit ec33f8b718
4 changed files with 297 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config.py
+12 -1
View File
@@ -159,13 +159,24 @@ class SignalConfig:
class NewsConfig:
"""Economic calendar & sentiment."""
tracked_currencies: List[str] = field(default_factory=lambda: ["USD", "XAU"])
min_impact: str = "medium"
pre_event_buffer_minutes: int = 30
post_event_buffer_minutes: int = 15
# Wider stay-out window after a high-impact release that surprised
# vs forecast (inline releases use post_event_buffer_minutes).
post_event_surprise_buffer_minutes: int = 45
# How long fetched calendar/headline data is reused across scans.
# Must stay below pre_event_buffer_minutes so an event can't cross
# into the danger window while a stale cache says it's clear.
context_cache_ttl_seconds: int = 600
rss_feeds: List[str] = field(default_factory=lambda: [
"https://investinglive.com/feed/",
"https://www.fxstreet.com/rss",
# Google News query targeted at gold drivers — also carries Kitco,
# Reuters etc. whose direct feeds are unavailable.
"https://news.google.com/rss/search?q=gold+price+OR+fed+OR+inflation&hl=en-US&gl=US&ceid=US:en",
# CNBC "Economy" section
"https://search.cnbc.com/rs/search/combinedcms/view.xml?partnerId=wrss01&id=20910258",
])
main.py
+1
View File
@@ -200,6 +200,7 @@ def report_to_dict(report: SignalReport) -> dict:
"event_risk": report.event_risk,
"event_description": report.event_description,
"news_sentiment": round(report.news_sentiment, 3),
"news_description": report.news_description,
"bullish_reasons": report.bullish_reasons,
"bearish_reasons": report.bearish_reasons,
"warnings": report.warnings,
news_engine.py
+166 -34
View File
@@ -10,6 +10,7 @@ A perfect technical setup can be destroyed by an unexpected NFP print.
import logging
import re
import threading
import time
import xml.etree.ElementTree as ET
from dataclasses import dataclass, field
@@ -23,6 +24,21 @@ from config import NewsConfig
logger = logging.getLogger(__name__)
_RELEASE_VALUE_RE = re.compile(r"(-?\d+(?:\.\d+)?)\s*([kmbt])?", re.IGNORECASE)
_RELEASE_MULTIPLIERS = {"k": 1e3, "m": 1e6, "b": 1e9, "t": 1e12}
def _parse_release_value(raw: Optional[str]) -> Optional[float]:
"""Parse a calendar release value like '0.3%', '-1.7%', '228K', '1.1M'."""
if not raw:
return None
match = _RELEASE_VALUE_RE.search(raw.replace(",", ""))
if not match:
return None
multiplier = _RELEASE_MULTIPLIERS.get((match.group(2) or "").lower(), 1.0)
return float(match.group(1)) * multiplier
@dataclass
class EconomicEvent:
"""Single economic calendar event."""
@@ -33,7 +49,21 @@ class EconomicEvent:
forecast: Optional[str] = None
previous: Optional[str] = None
actual: Optional[str] = None
surprise: Optional[str] = None # "positive", "negative", "inline", None
@property
def surprise(self) -> Optional[str]:
"""Actual vs forecast: "positive"/"negative"/"inline", or None when
either value is missing or non-numeric. Direction is numeric only —
it does not know whether higher is market-bullish for this event."""
actual = _parse_release_value(self.actual)
forecast = _parse_release_value(self.forecast)
if actual is None or forecast is None:
return None
if actual > forecast:
return "positive"
if actual < forecast:
return "negative"
return "inline"
@property
def is_upcoming(self) -> bool:
@@ -58,6 +88,9 @@ class NewsSentiment:
currencies_mentioned: List[str]
sentiment_score: float # -1.0 (bearish) to +1.0 (bullish)
keywords: List[str]
# Per-currency score: monetary-policy keywords flip sign for XAU
# (hawkish is dollar-bullish but gold-bearish).
sentiment_by_currency: Dict[str, float] = field(default_factory=dict)
@dataclass
@@ -105,29 +138,52 @@ class NewsEngine:
"disappointing", "soft data",
}
# Currency-specific keywords for headline parsing
# Monetary-policy keywords whose direction inverts for gold: tighter
# policy lifts the dollar and yields, weighing on non-yielding gold;
# easing does the opposite. Only unambiguous policy words — economic
# data words (beats/misses) keep their normal direction.
GOLD_INVERSE_KEYWORDS = {
"rate hike", "hawkish", "tightening",
"rate cut", "dovish", "easing", "stimulus",
}
# Currency-specific keywords for headline parsing. Matched with word
# boundaries — bare "us"/"eu"/"uk" are deliberately excluded because as
# substrings they false-positive constantly ("versus" → USD).
CURRENCY_KEYWORDS = {
"USD": ["dollar", "fed", "fomc", "treasury", "us ", "american", "nonfarm", "nfp"],
"EUR": ["euro", "ecb", "eurozone", "eu ", "german", "french"],
"GBP": ["pound", "sterling", "boe", "bank of england", "uk ", "british"],
"USD": ["dollar", "fed", "federal reserve", "fomc", "treasury", "american", "nonfarm", "nfp"],
"EUR": ["euro", "ecb", "eurozone", "german", "french"],
"GBP": ["pound", "sterling", "boe", "bank of england", "british"],
"JPY": ["yen", "boj", "bank of japan", "japanese"],
"CHF": ["franc", "snb", "swiss"],
"AUD": ["aussie", "rba", "australian", "australia"],
"CAD": ["loonie", "boc", "canadian", "canada"],
"NZD": ["kiwi", "rbnz", "new zealand"],
"XAU": ["gold", "xau", "precious metal", "safe haven", "bullion"],
"XAU": ["gold", "precious metal", "safe haven", "bullion"],
}
# High-impact event types (these move markets significantly)
HIGH_IMPACT_EVENTS = {
"interest rate decision", "nonfarm payrolls", "non-farm",
"gdp", "cpi", "inflation", "employment change",
"retail sales", "pmi", "fomc", "ecb press conference",
"boe rate", "rba rate", "snb rate", "boc rate",
# One compiled alternation per currency (code + keywords). (?<![a-z]) /
# (?![a-z]) instead of \b so "0.3%fed" still matches but "versus",
# "audit", "cascade" don't.
_CURRENCY_PATTERNS = {
cur: re.compile(
r"(?<![a-z])(?:"
+ "|".join(re.escape(kw) for kw in [cur.lower()] + kws)
+ r")(?![a-z])"
)
for cur, kws in CURRENCY_KEYWORDS.items()
}
_calendar_backoff_until: float = 0.0
# Fetched-data cache, class-level: a fresh NewsEngine is built per scan,
# so instance state would never be reused. Guarded by _cache_lock since
# scans run concurrently in a thread pool.
_cache_lock = threading.Lock()
_calendar_cache: Tuple[float, List[EconomicEvent]] = (0.0, [])
_headlines_cache: Tuple[float, List[NewsSentiment]] = (0.0, [])
_EMPTY_RESULT_TTL = 60.0 # retry sooner after a failed/empty fetch
def __init__(self, config: NewsConfig):
self.config = config
self._session = requests.Session()
@@ -153,7 +209,13 @@ class NewsEngine:
events = self._fetch_economic_calendar(currencies)
now = datetime.now(timezone.utc)
context.events_today = [e for e in events if e.time.date() == now.date()]
# Compare dates in UTC — feed timestamps carry their own offset
# (FF sends US-Eastern), so a late-evening ET event would
# otherwise land on the wrong day.
context.events_today = [
e for e in events
if e.time.astimezone(timezone.utc).date() == now.date()
]
context.events_upcoming_1h = [
e for e in events
if 0 < e.minutes_until <= 60
@@ -187,10 +249,12 @@ class NewsEngine:
headlines = self._fetch_news_sentiment()
context.news_headlines = headlines
# Aggregate sentiment per currency
# Aggregate sentiment per currency (per-currency score where
# available, so gold-inverted policy keywords are respected)
for currency in currencies:
scores = [
h.sentiment_score for h in headlines
h.sentiment_by_currency.get(currency, h.sentiment_score)
for h in headlines
if currency in h.currencies_mentioned
]
if scores:
@@ -226,10 +290,24 @@ class NewsEngine:
else:
return "caution", f"{event.currency} {event.event} in {mins:.0f}min"
# Check for recent high-impact releases (within post-event buffer)
# Check for recent high-impact releases. A surprise print (actual
# away from forecast) keeps the market unstable longer than an
# inline release, so it gets a wider post-event window.
for event in context.events_today:
if event.is_high_impact and event.currency in (base, quote):
if not event.is_upcoming and event.minutes_until > -self.config.post_event_buffer_minutes:
if event.is_high_impact and event.currency in (base, quote) and not event.is_upcoming:
surprise = event.surprise
surprised = surprise in ("positive", "negative")
window = (
self.config.post_event_surprise_buffer_minutes
if surprised
else self.config.post_event_buffer_minutes
)
if event.minutes_until > -window:
if surprised:
return "caution", (
f"{event.currency} {event.event} released — {surprise} surprise "
f"(actual {event.actual} vs forecast {event.forecast})"
)
return "caution", f"{event.currency} {event.event} just released"
return "clear", "No significant events"
@@ -272,7 +350,32 @@ class NewsEngine:
"canada": "CAD", "new zealand": "NZD",
}
def _cached_fetch(self, cache_attr: str, fetch_fn) -> list:
"""Return cached data for `cache_attr` or refetch when the TTL
expired. Successful results live for context_cache_ttl_seconds;
empty results only _EMPTY_RESULT_TTL so outages retry quickly.
The lock is held across the fetch so concurrent scans share one
request instead of stampeding the source."""
ts, data = getattr(NewsEngine, cache_attr)
ttl = self.config.context_cache_ttl_seconds if data else self._EMPTY_RESULT_TTL
if time.time() - ts < ttl:
return data
with NewsEngine._cache_lock:
ts, data = getattr(NewsEngine, cache_attr) # re-check under lock
ttl = self.config.context_cache_ttl_seconds if data else self._EMPTY_RESULT_TTL
if time.time() - ts < ttl:
return data
data = fetch_fn()
setattr(NewsEngine, cache_attr, (time.time(), data))
return data
def _fetch_economic_calendar(self, currencies: List[str]) -> List[EconomicEvent]:
"""Cached calendar fetch, filtered to the tracked currencies."""
events = self._cached_fetch("_calendar_cache", self._fetch_calendar_raw)
return [e for e in events if e.currency in currencies]
def _fetch_calendar_raw(self) -> List[EconomicEvent]:
"""
Fetch economic calendar events from a free API.
Tries Forex Factory first, falls back to MyFXBook RSS.
@@ -283,7 +386,7 @@ class NewsEngine:
try:
events = fetcher()
if events:
return [e for e in events if e.currency in currencies]
return events
except Exception as e:
logger.debug(f"Calendar fetch exception ({fetcher.__name__}): {e}")
@@ -426,8 +529,13 @@ class NewsEngine:
return events
def _fetch_news_sentiment(self) -> List[NewsSentiment]:
"""Cached headline fetch + sentiment analysis."""
return self._cached_fetch("_headlines_cache", self._fetch_news_sentiment_raw)
def _fetch_news_sentiment_raw(self) -> List[NewsSentiment]:
"""Fetch and analyze news headlines for sentiment."""
headlines = []
seen_titles = set()
for feed_url in self.config.rss_feeds:
try:
@@ -453,6 +561,14 @@ class NewsEngine:
if pub_date and (datetime.now(timezone.utc) - pub_date).total_seconds() > 86400:
continue
# Dedup within and across feeds so a syndicated story
# doesn't count twice in the sentiment average. Google
# News appends " - Publisher", so compare without it.
title_key = title.lower().rsplit(" - ", 1)[0].strip()
if title_key in seen_titles:
continue
seen_titles.add(title_key)
sentiment = self._analyze_headline_sentiment(title)
currencies = self._extract_currencies(title)
@@ -464,6 +580,10 @@ class NewsEngine:
currencies_mentioned=currencies,
sentiment_score=sentiment,
keywords=self._extract_keywords(title),
sentiment_by_currency={
cur: self._analyze_headline_sentiment(title, cur)
for cur in currencies
},
))
except Exception as e:
@@ -471,19 +591,38 @@ class NewsEngine:
return headlines
def _analyze_headline_sentiment(self, headline: str) -> float:
def _analyze_headline_sentiment(
self, headline: str, for_currency: Optional[str] = None
) -> float:
"""
Simple keyword-based sentiment analysis.
Returns -1.0 (very bearish) to +1.0 (very bullish).
When for_currency is "XAU", monetary-policy keywords contribute
with inverted sign (see GOLD_INVERSE_KEYWORDS).
Note: This is intentionally simple. NLP models like FinBERT
would be more accurate but require large dependencies. This catches
the most impactful sentiment signals from headlines.
"""
headline_lower = headline.lower()
bullish_score = sum(1 for kw in self.BULLISH_KEYWORDS if kw in headline_lower)
bearish_score = sum(1 for kw in self.BEARISH_KEYWORDS if kw in headline_lower)
invert_policy = for_currency == "XAU"
bullish_score = 0
bearish_score = 0
for kw in self.BULLISH_KEYWORDS:
if kw in headline_lower:
if invert_policy and kw in self.GOLD_INVERSE_KEYWORDS:
bearish_score += 1
else:
bullish_score += 1
for kw in self.BEARISH_KEYWORDS:
if kw in headline_lower:
if invert_policy and kw in self.GOLD_INVERSE_KEYWORDS:
bullish_score += 1
else:
bearish_score += 1
total = bullish_score + bearish_score
if total == 0:
@@ -494,18 +633,11 @@ class NewsEngine:
def _extract_currencies(self, headline: str) -> List[str]:
"""Extract mentioned currencies from a headline."""
headline_lower = headline.lower()
found = []
for currency, keywords in self.CURRENCY_KEYWORDS.items():
if currency.lower() in headline_lower:
found.append(currency)
continue
for kw in keywords:
if kw in headline_lower:
found.append(currency)
break
return list(set(found))
return [
currency
for currency, pattern in self._CURRENCY_PATTERNS.items()
if pattern.search(headline_lower)
]
def _extract_keywords(self, headline: str) -> List[str]:
"""Extract relevant trading keywords from headline."""
test_all.py
+118
View File
@@ -953,6 +953,124 @@ def _():
assert "neutral" in desc.lower()
@test("NewsEngine._extract_currencies no USD false positive from 'versus'")
def _():
ne = NewsEngine(NewsConfig())
currencies = ne._extract_currencies(
"Canada building permits for May -1.7% versus 2.4% estimate"
)
assert "USD" not in currencies, f"'versus' must not tag USD, got {currencies}"
assert "CAD" in currencies
@test("NewsEngine._extract_currencies no AUD false positive from 'audit'")
def _():
ne = NewsEngine(NewsConfig())
currencies = ne._extract_currencies("Treasury audit finds accounting errors")
assert "AUD" not in currencies
@test("EconomicEvent.surprise from actual vs forecast")
def _():
now = datetime.now(timezone.utc)
beat = EconomicEvent(time=now, currency="USD", impact="high",
event="NFP", forecast="900K", actual="1.1M")
assert beat.surprise == "positive"
miss = EconomicEvent(time=now, currency="USD", impact="high",
event="CPI y/y", forecast="0.3%", actual="0.2%")
assert miss.surprise == "negative"
inline = EconomicEvent(time=now, currency="USD", impact="high",
event="CPI y/y", forecast="0.3%", actual="0.3%")
assert inline.surprise == "inline"
pending = EconomicEvent(time=now, currency="USD", impact="high",
event="CPI y/y", forecast="0.3%")
assert pending.surprise is None
negative = EconomicEvent(time=now, currency="USD", impact="high",
event="Permits m/m", forecast="2.4%", actual="-1.7%")
assert negative.surprise == "negative"
@test("NewsEngine.get_event_risk_level surprise widens post-event window")
def _():
ne = NewsEngine(NewsConfig())
released_at = datetime.now(timezone.utc) - timedelta(minutes=30)
# Surprise print 30 min ago: outside the 15-min inline buffer but
# inside the 45-min surprise buffer → still caution.
context = FundamentalContext()
context.events_today = [EconomicEvent(
time=released_at, currency="USD", impact="high",
event="CPI m/m", forecast="0.3%", actual="0.5%",
)]
risk, desc = ne.get_event_risk_level("XAU/USD", context)
assert risk == "caution", f"Expected caution after surprise, got {risk}"
assert "surprise" in desc
# Same timing but inline print → clear.
context.events_today = [EconomicEvent(
time=released_at, currency="USD", impact="high",
event="CPI m/m", forecast="0.3%", actual="0.3%",
)]
risk, _ = ne.get_event_risk_level("XAU/USD", context)
assert risk == "clear", f"Inline release 30min ago should be clear, got {risk}"
@test("NewsEngine gold-inverted sentiment for policy keywords")
def _():
ne = NewsEngine(NewsConfig())
headline = "Fed signals rate hike as hawkish stance hardens"
# Currency-bullish for the dollar...
assert ne._analyze_headline_sentiment(headline) > 0
assert ne._analyze_headline_sentiment(headline, "USD") > 0
# ...but bearish for gold.
assert ne._analyze_headline_sentiment(headline, "XAU") < 0
dovish = "Fed turns dovish, rate cut expected"
assert ne._analyze_headline_sentiment(dovish) < 0
assert ne._analyze_headline_sentiment(dovish, "XAU") > 0
# Non-policy keywords keep their direction for gold.
data = "Gold surges to record high"
assert ne._analyze_headline_sentiment(data, "XAU") > 0
@test("NewsEngine calendar cache avoids refetch within TTL")
def _():
NewsEngine._calendar_cache = (0.0, [])
try:
ne = NewsEngine(NewsConfig())
calls = {"n": 0}
def fake_fetch():
calls["n"] += 1
return [EconomicEvent(
time=datetime.now(timezone.utc), currency="USD",
impact="low", event="Test Event",
)]
ne._try_fetch_calendar_api = fake_fetch
ne._try_fetch_myfxbook_rss = lambda: []
first = ne._fetch_economic_calendar(["USD", "XAU"])
second = ne._fetch_economic_calendar(["USD", "XAU"])
assert calls["n"] == 1, f"Expected 1 fetch, got {calls['n']}"
assert len(first) == 1 and len(second) == 1
# A second engine instance (new scan) must also hit the cache.
ne2 = NewsEngine(NewsConfig())
ne2._try_fetch_calendar_api = fake_fetch
ne2._try_fetch_myfxbook_rss = lambda: []
third = ne2._fetch_economic_calendar(["USD", "XAU"])
assert calls["n"] == 1, "New engine instance should reuse class-level cache"
assert len(third) == 1
finally:
NewsEngine._calendar_cache = (0.0, [])
@test("FundamentalContext defaults")
def _():
ctx = FundamentalContext()