Harwinder Sekhon
ad14ea4fed
Agent panel v2: personas return p_win anchored to the book base rates with a code-computed decision context; verdict = equal-weight mean (verdict_source "mean"); only the macro persona runs live (agent_panel.personas ["macro"]); prompt version and exact payloads logged; offline replay harness scripts/panel_replay.py; base-rate table agent_panel_base_rates.json (scripts/build_panel_base_rates.py). Backtester `strategy` column now carries the generator's own stamp (label-only; five reference books re-pinned, all 62 other columns identical). News corpus (after evaluating the owner's ML-NEWS scikit model against DeepSeek: golden set 39/78 vs 78/78, out-of-sample agreement 64% against a 90% self-agreement ceiling; verdict: fallback only, not a replacement): - NewsSentiment.xau_score + archive column xau_score keep DeepSeek's gold score for every headline (930 of the first 1,546 DeepSeek rows had none); an existing newsdata.db gains the column in place on first open - headlines naming no tracked currency are scored and archived (currencies_json []) but never returned to the platform - _apply_llm_sentiment sends every new headline in its cycle, 25 per call; a failed call ends the cycle's sending Dashboard publisher never writes a database: main() calls app_config.set_db_writes_for_process(False), honoured by signal_log_enabled() and news_archive_enabled() (owner decision: the bot is the single writer). Test suite: the hermetic app config now points both writers at a suite-private temp dir — the RSS fixture tests were writing fixture headlines into the real logs/newsdata.db — with a Module 32 test pinning it. 529 tests.
1211 lines
52 KiB
Python
Executable File
1211 lines
52 KiB
Python
Executable File
"""
|
|
Forex Signal Intelligence Platform - News & Sentiment Engine
|
|
=============================================================
|
|
Monitors economic calendar events and news sentiment to provide
|
|
fundamental context for technical signals.
|
|
|
|
Key principle: High-impact news events override technical signals.
|
|
A perfect technical setup can be destroyed by an unexpected NFP print.
|
|
"""
|
|
|
|
import hashlib
|
|
import html
|
|
import logging
|
|
import math
|
|
import re
|
|
import threading
|
|
import time
|
|
import xml.etree.ElementTree as ET
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from email.utils import parsedate_to_datetime
|
|
from typing import Dict, List, Optional, Tuple
|
|
|
|
import requests
|
|
|
|
import news_archive
|
|
from config import NewsConfig
|
|
from llm_client import get_llm_client
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# ── DeepSeek prompts ─────────────────────────────────────────────────
|
|
# STATIC constants on purpose: DeepSeek prices a repeated byte-identical
|
|
# prompt prefix at the cache-hit rate (~50x cheaper), so nothing here may
|
|
# be interpolated or timestamped. All variable content travels in the
|
|
# user payload. LLM output is display/warning-only — it never gates
|
|
# signals (see NewsConfig.sentiment_affects_signals).
|
|
|
|
_LLM_SENTIMENT_SYSTEM = """\
|
|
You are a forex/gold news analyst. You receive JSON:
|
|
{"headlines": ["...", ...]}
|
|
For EACH headline, in the same order, score:
|
|
- "s": overall market sentiment of the headline, -1.0 (very bearish)
|
|
to 1.0 (very bullish), 0.0 if irrelevant or neutral.
|
|
- "xau": the effect on the GOLD (XAU/USD) price, -1.0 (gold down)
|
|
to 1.0 (gold up), 0.0 if irrelevant.
|
|
Gold doctrine: hawkish Fed / rate hikes / rising real yields / strong
|
|
dollar are gold-bearish; easing / rate cuts / safe-haven flows /
|
|
geopolitical stress / weak dollar are gold-bullish.
|
|
Handle negation correctly ("no rate hike" is not hawkish).
|
|
Reply with ONLY JSON: {"scores": [[s, xau], ...]} — exactly one
|
|
[s, xau] pair per input headline, in the same order, nothing else.
|
|
|
|
Examples:
|
|
"Fed signals faster rate hikes as inflation stays hot" -> [0.4, -0.7]
|
|
"Gold surges to record high on safe-haven demand" -> [0.8, 0.9]
|
|
"Dollar slips after soft jobs report; Treasury yields fall" -> [-0.4, 0.6]
|
|
"""
|
|
|
|
_LLM_EVENT_SYSTEM = """\
|
|
You are a forex/gold analyst. You receive one released economic event as
|
|
JSON: {"event", "currency", "actual", "forecast", "previous",
|
|
"vs_forecast"}. "vs_forecast" states whether the actual print came in
|
|
above or below the forecast; it is computed programmatically — trust it
|
|
over your own comparison of the numbers. If it is "unknown", compare
|
|
actual and forecast yourself.
|
|
Decide the likely effect of this RELEASE on the GOLD (XAU/USD) price.
|
|
A stronger-than-forecast USD economy or a hawkish implication is usually
|
|
gold-bearish; weaker data or a dovish implication is usually
|
|
gold-bullish.
|
|
Inverted metrics — for unemployment rate and unemployment/jobless
|
|
claims a LOWER number means a STRONGER labor market: below forecast =
|
|
strong economy = usually gold-bearish; above forecast = weak economy =
|
|
usually gold-bullish.
|
|
Example: {"event": "Unemployment Claims", "actual": "202K", "forecast":
|
|
"217K", "vs_forecast": "below forecast"} ->
|
|
{"gold_direction": "bearish", "reason": "fewer claims show strong labor market"}
|
|
Reply with ONLY JSON:
|
|
{"gold_direction": "bullish"|"bearish"|"neutral", "reason": "<max 10 words>"}
|
|
"""
|
|
|
|
_LLM_SUMMARY_SYSTEM = """\
|
|
You write a short news note about the gold price for someone who has
|
|
never traded. You receive JSON with recent "headlines" (newest first;
|
|
each carries "gold": "up" or "down", the platform's own reading of how
|
|
that story pushes the gold price — trust it) and today's high-impact
|
|
"events" (released: true means the number is out).
|
|
Say three things, in this order: what is happening to gold right now,
|
|
why it matters for the gold price, and the next scheduled report to
|
|
watch if one is listed. Use only the headlines and events given; if
|
|
they are thin, say so plainly.
|
|
Style: plain everyday English, short simple sentences, no acronyms, no
|
|
abbreviations, no trader words. Write "the US central bank", not "the
|
|
Fed"; "interest rates", not "yields" or "hawkish"; "a report on US
|
|
prices", not "CPI" or "PCE"; "gold", never "XAU". No advice, no hedging
|
|
boilerplate. 2-3 sentences, max 55 words.
|
|
Reply with ONLY JSON: {"summary": "..."}
|
|
|
|
Examples:
|
|
{"summary": "Gold is falling today. The US central bank hinted it may raise interest rates, which makes the dollar stronger, and a stronger dollar usually pushes gold down. The next thing to watch is Thursday's report on US prices."}
|
|
{"summary": "Gold is climbing. Fresh fighting in the Middle East has people moving money into gold as a safe place, and a weaker dollar is helping too. No major US reports are due today."}
|
|
"""
|
|
|
|
|
|
#: Identity of the labelling prompt, stored on every archived LLM score.
|
|
#: If the prompt changes, scores before and after are answers to different
|
|
#: questions and must not be pooled into one training set.
|
|
_LLM_SENTIMENT_PROMPT_SHA1 = hashlib.sha1(
|
|
_LLM_SENTIMENT_SYSTEM.encode("utf-8")).hexdigest()
|
|
|
|
|
|
_RELEASE_VALUE_RE = re.compile(r"(-?\d+(?:\.\d+)?)\s*([kmbt])?", re.IGNORECASE)
|
|
_RELEASE_MULTIPLIERS = {"k": 1e3, "m": 1e6, "b": 1e9, "t": 1e12}
|
|
|
|
|
|
def _parse_release_value(raw: Optional[str]) -> Optional[float]:
|
|
"""Parse a calendar release value like '0.3%', '-1.7%', '228K', '1.1M'."""
|
|
if not raw:
|
|
return None
|
|
match = _RELEASE_VALUE_RE.search(raw.replace(",", ""))
|
|
if not match:
|
|
return None
|
|
multiplier = _RELEASE_MULTIPLIERS.get((match.group(2) or "").lower(), 1.0)
|
|
return float(match.group(1)) * multiplier
|
|
|
|
|
|
_DESC_MAX_CHARS = 500
|
|
_MAX_LINK_CHARS = 2048
|
|
|
|
|
|
def _clean_html_text(raw: Optional[str],
|
|
limit: int = _DESC_MAX_CHARS) -> Optional[str]:
|
|
"""RSS <description> → plain text: unescape entities, strip tags,
|
|
collapse whitespace, truncate. Unescape and strip both run twice
|
|
because feeds routinely double-encode (&lt;p&gt; wrapping
|
|
real markup) — the second unescape can reveal fresh tags."""
|
|
if not raw:
|
|
return None
|
|
text = html.unescape(raw)
|
|
text = re.sub(r"<[^>]+>", " ", text)
|
|
text = html.unescape(text)
|
|
text = re.sub(r"<[^>]+>", " ", text)
|
|
text = re.sub(r"\s+", " ", text).strip()
|
|
if not text:
|
|
return None
|
|
if len(text) > limit:
|
|
text = text[:limit].rstrip() + "…"
|
|
return text
|
|
|
|
|
|
def _safe_link(raw: Optional[str]) -> Optional[str]:
|
|
"""Article URL for display: http(s) only (an RSS <link> is untrusted
|
|
and lands in an href), bounded length (Google News redirect URLs run
|
|
1-2 KB; anything past that is bloat, not a link)."""
|
|
if not raw:
|
|
return None
|
|
link = raw.strip()
|
|
if not link.lower().startswith(("http://", "https://")):
|
|
return None
|
|
if len(link) > _MAX_LINK_CHARS:
|
|
return None
|
|
return link
|
|
|
|
|
|
@dataclass
|
|
class EconomicEvent:
|
|
"""Single economic calendar event."""
|
|
time: datetime
|
|
currency: str
|
|
impact: str # "low", "medium", "high"
|
|
event: str
|
|
forecast: Optional[str] = None
|
|
previous: Optional[str] = None
|
|
actual: Optional[str] = None
|
|
|
|
@property
|
|
def surprise(self) -> Optional[str]:
|
|
"""Actual vs forecast: "positive"/"negative"/"inline", or None when
|
|
either value is missing or non-numeric. Direction is numeric only —
|
|
it does not know whether higher is market-bullish for this event."""
|
|
actual = _parse_release_value(self.actual)
|
|
forecast = _parse_release_value(self.forecast)
|
|
if actual is None or forecast is None:
|
|
return None
|
|
if actual > forecast:
|
|
return "positive"
|
|
if actual < forecast:
|
|
return "negative"
|
|
return "inline"
|
|
|
|
@property
|
|
def is_upcoming(self) -> bool:
|
|
return self.time > datetime.now(timezone.utc)
|
|
|
|
@property
|
|
def minutes_until(self) -> float:
|
|
delta = self.time - datetime.now(timezone.utc)
|
|
return delta.total_seconds() / 60
|
|
|
|
@property
|
|
def is_high_impact(self) -> bool:
|
|
return self.impact == "high"
|
|
|
|
|
|
@dataclass
|
|
class NewsSentiment:
|
|
"""News sentiment analysis result."""
|
|
headline: str
|
|
source: str
|
|
timestamp: datetime
|
|
currencies_mentioned: List[str]
|
|
sentiment_score: float # -1.0 (bearish) to +1.0 (bullish)
|
|
keywords: List[str]
|
|
# Per-currency score: monetary-policy keywords flip sign for XAU
|
|
# (hawkish is dollar-bullish but gold-bearish).
|
|
sentiment_by_currency: Dict[str, float] = field(default_factory=dict)
|
|
# Display-only enrichment (dashboard detail dialog). Optional with
|
|
# defaults so existing constructor calls keep working.
|
|
link: Optional[str] = None # article URL (Google News = redirect)
|
|
description: Optional[str] = None # plain text, tags stripped, ≤500 chars
|
|
publisher: Optional[str] = None # Google News <source> text, else None
|
|
# Which scorer produced sentiment_score / sentiment_by_currency:
|
|
# "keyword" (the built-in scorer, ~31% on the 2026-08-06 golden set),
|
|
# "llm" (DeepSeek, 78/78 on the same set) or "model" (a trained local
|
|
# artifact). Without it the two are indistinguishable in the archive
|
|
# and a model trained on the mix inherits the keyword scorer's errors.
|
|
score_source: str = "keyword"
|
|
# DeepSeek's gold-specific score for THIS headline, whatever currencies
|
|
# it names. sentiment_by_currency["XAU"] carries it only when the
|
|
# headline matched a gold keyword; for a pure Fed / dollar / yields
|
|
# headline it used to be discarded, which left 930 of the first 1,546
|
|
# archived DeepSeek rows without a gold label (eval 2026-09-17).
|
|
# Nothing displayed or averaged reads it — the archive stores it as
|
|
# training data for a local scorer. None until DeepSeek has scored
|
|
# the headline (the keyword scorer never sets it).
|
|
xau_score: Optional[float] = None
|
|
|
|
|
|
@dataclass
|
|
class FundamentalContext:
|
|
"""Complete fundamental/news context for signal decisions."""
|
|
events_today: List[EconomicEvent] = field(default_factory=list)
|
|
events_upcoming_1h: List[EconomicEvent] = field(default_factory=list)
|
|
high_impact_warning: bool = False
|
|
warning_message: str = ""
|
|
news_sentiment: Dict[str, float] = field(default_factory=dict) # currency -> sentiment
|
|
news_headlines: List[NewsSentiment] = field(default_factory=list)
|
|
calendar_available: bool = False # True if live calendar data was fetched
|
|
llm_summary: str = "" # DeepSeek market-context blurb (display-only)
|
|
|
|
|
|
# Headlines whose every relevant score is inside ±NEUTRAL_HEADLINE_THRESHOLD
|
|
# lean neither way: they carry no direction for display and only dilute the
|
|
# per-currency mean. Same threshold agent_panel uses for its macro payload.
|
|
NEUTRAL_HEADLINE_THRESHOLD = 0.1
|
|
|
|
|
|
def _is_neutral_headline(h) -> bool:
|
|
"""True when no per-currency score on the headline leans either way.
|
|
|
|
A headline whose overall score is 0.0 but whose gold-specific score is
|
|
-0.7 is NOT neutral. Missing, empty or non-finite scores count as
|
|
neutral (they cannot be shown as leaning one way)."""
|
|
by_currency = getattr(h, "sentiment_by_currency", None) or {}
|
|
overall = getattr(h, "sentiment_score", None)
|
|
values = []
|
|
for currency in (getattr(h, "currencies_mentioned", None) or []):
|
|
value = by_currency.get(currency, overall)
|
|
if isinstance(value, (int, float)) and math.isfinite(value):
|
|
values.append(abs(float(value)))
|
|
return not values or max(values) < NEUTRAL_HEADLINE_THRESHOLD
|
|
|
|
|
|
def _gold_direction_label(h) -> str:
|
|
""""up" / "down" / "flat" for the market-summary payload — the same
|
|
XAU-first, USD-inverted reading agent_panel uses for its macro persona."""
|
|
by_currency = getattr(h, "sentiment_by_currency", None) or {}
|
|
xau = by_currency.get("XAU")
|
|
usd = by_currency.get("USD")
|
|
if isinstance(xau, (int, float)) and math.isfinite(xau):
|
|
score = float(xau)
|
|
elif isinstance(usd, (int, float)) and math.isfinite(usd):
|
|
score = -float(usd)
|
|
else:
|
|
raw = getattr(h, "sentiment_score", None)
|
|
score = float(raw) if isinstance(raw, (int, float)) and math.isfinite(raw) else 0.0
|
|
if score > 0:
|
|
return "up"
|
|
if score < 0:
|
|
return "down"
|
|
return "flat"
|
|
|
|
|
|
class NewsEngine:
|
|
"""
|
|
Economic calendar and news sentiment analyzer.
|
|
|
|
Provides fundamental context that modifies technical signal confidence.
|
|
"""
|
|
|
|
# Keywords that indicate directional sentiment
|
|
BULLISH_KEYWORDS = {
|
|
"rate hike", "hawkish", "strong growth", "beat expectations",
|
|
"above forecast", "above estimate", "above expectation",
|
|
"surplus", "expansion", "employment gains",
|
|
"strong jobs", "gdp growth", "inflation rising", "tightening",
|
|
"positive surprise", "record high", "bullish", "rally",
|
|
"stronger than expected", "upbeat",
|
|
"beats", "surges", "soars", "jumps", "climbs", "rises",
|
|
"higher than expected", "better than expected",
|
|
"strong data", "robust", "resilient", "accelerates",
|
|
}
|
|
|
|
BEARISH_KEYWORDS = {
|
|
"rate cut", "dovish", "recession", "slowdown", "contraction",
|
|
"below forecast", "below estimate", "below expectation",
|
|
"deficit", "miss expectations", "weak data",
|
|
"unemployment rise", "negative surprise", "bearish", "selloff",
|
|
"weaker than expected", "downbeat", "easing", "stimulus",
|
|
"misses", "slides", "drops", "falls", "plunges", "tumbles",
|
|
"weakens", "slumps", "declines", "retreats",
|
|
"lower than expected", "worse than expected",
|
|
"disappointing", "soft data",
|
|
}
|
|
|
|
# Monetary-policy keywords whose direction inverts for gold: tighter
|
|
# policy lifts the dollar and yields, weighing on non-yielding gold;
|
|
# easing does the opposite. Only unambiguous policy words — economic
|
|
# data words (beats/misses) keep their normal direction.
|
|
GOLD_INVERSE_KEYWORDS = {
|
|
"rate hike", "hawkish", "tightening",
|
|
"rate cut", "dovish", "easing", "stimulus",
|
|
}
|
|
|
|
# Currency-specific keywords for headline parsing. Matched with word
|
|
# boundaries — bare "us"/"eu"/"uk" are deliberately excluded because as
|
|
# substrings they false-positive constantly ("versus" → USD).
|
|
CURRENCY_KEYWORDS = {
|
|
"USD": ["dollar", "fed", "federal reserve", "fomc", "treasury", "american", "nonfarm", "nfp"],
|
|
"EUR": ["euro", "ecb", "eurozone", "german", "french"],
|
|
"GBP": ["pound", "sterling", "boe", "bank of england", "british"],
|
|
"JPY": ["yen", "boj", "bank of japan", "japanese"],
|
|
"CHF": ["franc", "snb", "swiss"],
|
|
"AUD": ["aussie", "rba", "australian", "australia"],
|
|
"CAD": ["loonie", "boc", "canadian", "canada"],
|
|
"NZD": ["kiwi", "rbnz", "new zealand"],
|
|
"XAU": ["gold", "precious metal", "safe haven", "bullion"],
|
|
}
|
|
|
|
# One compiled alternation per currency (code + keywords). (?<![a-z]) /
|
|
# (?![a-z]) instead of \b so "0.3%fed" still matches but "versus",
|
|
# "audit", "cascade" don't.
|
|
_CURRENCY_PATTERNS = {
|
|
cur: re.compile(
|
|
r"(?<![a-z])(?:"
|
|
+ "|".join(re.escape(kw) for kw in [cur.lower()] + kws)
|
|
+ r")(?![a-z])"
|
|
)
|
|
for cur, kws in CURRENCY_KEYWORDS.items()
|
|
}
|
|
|
|
_calendar_backoff_until: float = 0.0
|
|
|
|
# Fetched-data cache, class-level: a fresh NewsEngine is built per scan,
|
|
# so instance state would never be reused. Guarded by _cache_lock since
|
|
# scans run concurrently in a thread pool.
|
|
_cache_lock = threading.Lock()
|
|
_calendar_cache: Tuple[float, List[EconomicEvent]] = (0.0, [])
|
|
_headlines_cache: Tuple[float, List[NewsSentiment]] = (0.0, [])
|
|
_EMPTY_RESULT_TTL = 60.0 # retry sooner after a failed/empty fetch
|
|
|
|
# DeepSeek result caches. Headlines and releases are immutable, so a
|
|
# scored one never needs re-scoring; bounded by dropping the oldest
|
|
# insertion. _llm_sent_cache is only touched under _cache_lock (its
|
|
# writer runs inside _cached_fetch); the other two take the lock
|
|
# around cache access but keep HTTP calls outside it.
|
|
_llm_sent_cache: Dict[str, Tuple[float, float]] = {} # sha1 → (s, xau)
|
|
_llm_event_cache: Dict[Tuple[str, str, str], str] = {}
|
|
_llm_summary_cache: Tuple[float, str] = (0.0, "")
|
|
_LLM_SENT_CACHE_MAX = 400
|
|
_LLM_EVENT_CACHE_MAX = 100
|
|
#: Headlines per DeepSeek sentiment call — the reply must fit the
|
|
#: client's max_tokens (~10 tokens per headline, 2000 max).
|
|
_LLM_BATCH_CAP = 25
|
|
|
|
# Per-source fetch health for status displays (telegram /status).
|
|
# name → {"ok": ts|None, "err": ts|None, "detail": str, "extra": str}.
|
|
# All writers run under _cache_lock (fetches happen inside
|
|
# _cached_fetch); readers render without the lock — worst case a
|
|
# display line is one cycle stale. A source is "failing" when its
|
|
# last error is newer than its last success.
|
|
_source_health: Dict[str, dict] = {}
|
|
|
|
def __init__(self, config: NewsConfig):
|
|
self.config = config
|
|
self._session = requests.Session()
|
|
self._session.headers.update({
|
|
"User-Agent": "Mozilla/5.0 (compatible; ForexIntel/1.0)"
|
|
})
|
|
|
|
def get_fundamental_context(
|
|
self, currencies: Optional[List[str]] = None
|
|
) -> FundamentalContext:
|
|
"""
|
|
Get complete fundamental context for signal decisions.
|
|
|
|
Combines economic calendar + news sentiment into actionable context.
|
|
"""
|
|
if currencies is None:
|
|
currencies = self.config.tracked_currencies
|
|
|
|
context = FundamentalContext()
|
|
|
|
# 1. Economic calendar events
|
|
try:
|
|
events = self._fetch_economic_calendar(currencies)
|
|
now = datetime.now(timezone.utc)
|
|
|
|
# Compare dates in UTC — feed timestamps carry their own offset
|
|
# (FF sends US-Eastern), so a late-evening ET event would
|
|
# otherwise land on the wrong day.
|
|
context.events_today = [
|
|
e for e in events
|
|
if e.time.astimezone(timezone.utc).date() == now.date()
|
|
]
|
|
context.events_upcoming_1h = [
|
|
e for e in events
|
|
if 0 < e.minutes_until <= 60
|
|
]
|
|
context.calendar_available = bool(events)
|
|
|
|
# Check for high-impact warnings
|
|
for event in events:
|
|
if event.is_high_impact and 0 < event.minutes_until <= self.config.pre_event_buffer_minutes:
|
|
context.high_impact_warning = True
|
|
context.warning_message = (
|
|
f"HIGH IMPACT: {event.currency} {event.event} "
|
|
f"in {event.minutes_until:.0f} min — signals may be unreliable"
|
|
)
|
|
break
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Economic calendar fetch failed: {e}")
|
|
|
|
# 2. News sentiment
|
|
try:
|
|
headlines = self._fetch_news_sentiment()
|
|
# Drop headlines that lean neither way (2026-09-04). Applied
|
|
# here rather than in the fetch so the class-level cache and
|
|
# the feed-health counts still describe what the feeds sent;
|
|
# every consumer (aggregate score, summary, dashboard, panel,
|
|
# signal log) reads context.news_headlines.
|
|
kept = [h for h in headlines if not _is_neutral_headline(h)]
|
|
if len(kept) != len(headlines):
|
|
logger.debug(
|
|
f"News: dropped {len(headlines) - len(kept)} neutral "
|
|
f"headlines, {len(kept)} kept")
|
|
headlines = kept
|
|
context.news_headlines = headlines
|
|
|
|
# Aggregate sentiment per currency (per-currency score where
|
|
# available, so gold-inverted policy keywords are respected)
|
|
for currency in currencies:
|
|
scores = [
|
|
h.sentiment_by_currency.get(currency, h.sentiment_score)
|
|
for h in headlines
|
|
if currency in h.currencies_mentioned
|
|
]
|
|
if scores:
|
|
context.news_sentiment[currency] = sum(scores) / len(scores)
|
|
|
|
except Exception as e:
|
|
logger.warning(f"News sentiment fetch failed: {e}")
|
|
|
|
# 3. LLM market-context summary (display-only)
|
|
try:
|
|
context.llm_summary = self._llm_market_summary(context)
|
|
except Exception as e:
|
|
logger.debug(f"LLM market summary failed: {e}")
|
|
|
|
return context
|
|
|
|
@staticmethod
|
|
def _parse_pair(pair_symbol: str) -> Tuple[str, str]:
|
|
base, sep, quote = pair_symbol.partition("/")
|
|
if not sep:
|
|
raise ValueError(f"pair symbol must be BASE/QUOTE, got {pair_symbol!r}")
|
|
return base, quote
|
|
|
|
def get_event_risk_level(self, pair_symbol: str, context: FundamentalContext) -> Tuple[str, str]:
|
|
"""
|
|
Get event risk level for a specific pair.
|
|
Returns (risk_level, description).
|
|
risk_level: "clear", "caution", "danger"
|
|
"""
|
|
base, quote = self._parse_pair(pair_symbol)
|
|
|
|
# Check upcoming high-impact events. Grade from the SOONEST
|
|
# matching event, not the first listed — an unsorted feed could
|
|
# otherwise report "caution" for a far event while a nearer one
|
|
# warrants "danger".
|
|
soonest = None
|
|
for event in context.events_upcoming_1h:
|
|
if event.is_high_impact and event.currency in (base, quote):
|
|
if soonest is None or event.minutes_until < soonest.minutes_until:
|
|
soonest = event
|
|
if soonest is not None:
|
|
mins = soonest.minutes_until
|
|
if mins <= self.config.pre_event_buffer_minutes:
|
|
return "danger", f"{soonest.currency} {soonest.event} in {mins:.0f}min"
|
|
return "caution", f"{soonest.currency} {soonest.event} in {mins:.0f}min"
|
|
|
|
# Check for recent high-impact releases. A surprise print (actual
|
|
# away from forecast) keeps the market unstable longer than an
|
|
# inline release, so it gets a wider post-event window.
|
|
for event in context.events_today:
|
|
if event.is_high_impact and event.currency in (base, quote) and not event.is_upcoming:
|
|
surprise = event.surprise
|
|
surprised = surprise in ("positive", "negative")
|
|
window = (
|
|
self.config.post_event_surprise_buffer_minutes
|
|
if surprised
|
|
else self.config.post_event_buffer_minutes
|
|
)
|
|
if event.minutes_until > -window:
|
|
if surprised:
|
|
desc = (
|
|
f"{event.currency} {event.event} released — {surprise} surprise "
|
|
f"(actual {event.actual} vs forecast {event.forecast})"
|
|
)
|
|
# LLM enrichment touches the DESCRIPTION only —
|
|
# `surprise` is numeric-direction and does not know
|
|
# whether a beat is gold-bullish; the LLM does.
|
|
gold_dir = self._llm_event_direction(event)
|
|
if gold_dir:
|
|
desc += f" — gold {gold_dir}"
|
|
return "caution", desc
|
|
return "caution", f"{event.currency} {event.event} just released"
|
|
|
|
return "clear", "No significant events"
|
|
|
|
def get_pair_sentiment(
|
|
self, pair_symbol: str, context: FundamentalContext
|
|
) -> Tuple[float, str]:
|
|
"""
|
|
Get net sentiment for a pair based on news.
|
|
Returns (score, description) where score is -1 to +1.
|
|
Positive = bullish for the pair, negative = bearish.
|
|
"""
|
|
base, quote = self._parse_pair(pair_symbol)
|
|
|
|
base_sent = context.news_sentiment.get(base, 0.0)
|
|
quote_sent = context.news_sentiment.get(quote, 0.0)
|
|
|
|
# Net sentiment: positive base or negative quote = bullish for pair
|
|
net = base_sent - quote_sent
|
|
|
|
if abs(net) < 0.1:
|
|
desc = "Neutral news sentiment"
|
|
elif net > 0.3:
|
|
desc = f"Bullish sentiment ({base} positive / {quote} negative)"
|
|
elif net > 0:
|
|
desc = "Slightly bullish sentiment"
|
|
elif net < -0.3:
|
|
desc = f"Bearish sentiment ({base} negative / {quote} positive)"
|
|
else:
|
|
desc = "Slightly bearish sentiment"
|
|
|
|
return net, desc
|
|
|
|
def get_week_events(
|
|
self, currencies: Optional[List[str]] = None
|
|
) -> List[EconomicEvent]:
|
|
"""Full cached calendar week (past and upcoming), filtered to
|
|
`currencies` (default: tracked_currencies), sorted by time.
|
|
|
|
Read-only accessor for display surfaces (the web dashboard) —
|
|
rides the same class-level cache as signal scans, so it adds no
|
|
network I/O. No production signal path calls this.
|
|
"""
|
|
if currencies is None:
|
|
currencies = self.config.tracked_currencies
|
|
events = self._cached_fetch("_calendar_cache", self._fetch_calendar_raw)
|
|
return sorted(
|
|
(e for e in events if e.currency in currencies),
|
|
key=lambda e: e.time,
|
|
)
|
|
|
|
# ── Private Methods ────────────────────────────────────────────
|
|
|
|
# Maps country names in MyFXBook titles to currency codes
|
|
_COUNTRY_TO_CURRENCY = {
|
|
"united states": "USD", "euro area": "EUR", "united kingdom": "GBP",
|
|
"japan": "JPY", "switzerland": "CHF", "australia": "AUD",
|
|
"canada": "CAD", "new zealand": "NZD",
|
|
}
|
|
|
|
@classmethod
|
|
def _record_source(cls, name: str, ok: bool, detail: str = "",
|
|
extra: str = "") -> None:
|
|
"""Track one dependency's fetch outcome for status displays.
|
|
Logs state TRANSITIONS so the log file carries the story without
|
|
a line every 10-min cycle: first failure WARNING (with detail),
|
|
repeat failures DEBUG, recovery INFO."""
|
|
h = cls._source_health.setdefault(
|
|
name, {"ok": None, "err": None, "detail": "", "extra": ""})
|
|
was_failing = h["err"] is not None and (
|
|
h["ok"] is None or h["err"] > h["ok"])
|
|
if ok:
|
|
h["ok"] = time.time()
|
|
h["extra"] = extra
|
|
if was_failing:
|
|
logger.info("Source recovered: %s", name)
|
|
else:
|
|
h["err"] = time.time()
|
|
h["detail"] = detail
|
|
if was_failing:
|
|
logger.debug("Source still failing: %s (%s)", name, detail)
|
|
else:
|
|
logger.warning("Source failing: %s (%s)", name, detail)
|
|
|
|
@staticmethod
|
|
def source_is_failing(entry: Optional[dict]) -> bool:
|
|
"""True when the entry's last error is newer than its last ok."""
|
|
return bool(entry and entry["err"] is not None and
|
|
(entry["ok"] is None or entry["err"] > entry["ok"]))
|
|
|
|
def _cached_fetch(self, cache_attr: str, fetch_fn) -> list:
|
|
"""Return cached data for `cache_attr` or refetch when the TTL
|
|
expired. Successful results live for context_cache_ttl_seconds;
|
|
empty results only _EMPTY_RESULT_TTL so outages retry quickly.
|
|
The lock is held across the fetch so concurrent scans share one
|
|
request instead of stampeding the source."""
|
|
ts, data = getattr(NewsEngine, cache_attr)
|
|
ttl = self.config.context_cache_ttl_seconds if data else self._EMPTY_RESULT_TTL
|
|
if time.time() - ts < ttl:
|
|
return data
|
|
|
|
with NewsEngine._cache_lock:
|
|
ts, data = getattr(NewsEngine, cache_attr) # re-check under lock
|
|
ttl = self.config.context_cache_ttl_seconds if data else self._EMPTY_RESULT_TTL
|
|
if time.time() - ts < ttl:
|
|
return data
|
|
data = fetch_fn()
|
|
setattr(NewsEngine, cache_attr, (time.time(), data))
|
|
return data
|
|
|
|
def _fetch_economic_calendar(self, currencies: List[str]) -> List[EconomicEvent]:
|
|
"""Cached calendar fetch, filtered to the tracked currencies."""
|
|
events = self._cached_fetch("_calendar_cache", self._fetch_calendar_raw)
|
|
return [e for e in events if e.currency in currencies]
|
|
|
|
def _fetch_calendar_raw(self) -> List[EconomicEvent]:
|
|
"""
|
|
Fetch economic calendar events from a free API.
|
|
Tries Forex Factory first, falls back to MyFXBook RSS.
|
|
Returns an empty list (with a log warning) if unavailable —
|
|
we never generate fake events since that causes constant false alarms.
|
|
"""
|
|
sources = (
|
|
(self._try_fetch_calendar_api, "calendar:forexfactory",
|
|
"ForexFactory"),
|
|
(self._try_fetch_myfxbook_rss, "calendar:myfxbook", "MyFXBook"),
|
|
)
|
|
for fetcher, src, label in sources:
|
|
try:
|
|
events = fetcher()
|
|
if events:
|
|
self._record_source(src, True)
|
|
self._record_source(
|
|
"calendar", True,
|
|
extra=f"{len(events)} events via {label}")
|
|
return events
|
|
self._record_source(src, False, "no events returned")
|
|
except Exception as e:
|
|
self._record_source(
|
|
src, False, f"{type(e).__name__}: {e}")
|
|
logger.debug(f"Calendar fetch exception ({fetcher.__name__}): {e}")
|
|
|
|
self._record_source("calendar", False, "all sources failed")
|
|
logger.info(
|
|
"Economic calendar data unavailable. "
|
|
"Event risk checks will be skipped for this cycle."
|
|
)
|
|
return []
|
|
|
|
def _try_fetch_calendar_api(self) -> List[EconomicEvent]:
|
|
"""Attempt to fetch from the Forex Factory free JSON calendar."""
|
|
now = time.time()
|
|
if now < NewsEngine._calendar_backoff_until:
|
|
wait = NewsEngine._calendar_backoff_until - now
|
|
logger.debug(f"Calendar API in backoff for {wait:.0f}s more")
|
|
return []
|
|
|
|
url = "https://nfs.faireconomy.media/ff_calendar_thisweek.json"
|
|
|
|
for attempt in range(3):
|
|
try:
|
|
resp = self._session.get(url, timeout=10)
|
|
except requests.RequestException as e:
|
|
logger.debug(f"Calendar request failed (attempt {attempt + 1}): {e}")
|
|
if attempt < 2:
|
|
time.sleep(2 ** attempt)
|
|
continue
|
|
|
|
if resp.status_code == 429:
|
|
retry_after = int(resp.headers.get("Retry-After", 300))
|
|
retry_after = min(retry_after, 600)
|
|
NewsEngine._calendar_backoff_until = time.time() + retry_after
|
|
logger.warning(
|
|
f"Calendar API rate-limited (429), backing off {retry_after}s"
|
|
)
|
|
return []
|
|
|
|
if resp.status_code != 200:
|
|
logger.debug(f"Calendar API returned {resp.status_code}")
|
|
return []
|
|
|
|
data = resp.json()
|
|
events = []
|
|
impact_map = {"High": "high", "Medium": "medium", "Low": "low"}
|
|
|
|
for item in data:
|
|
try:
|
|
impact = impact_map.get(item.get("impact", ""), "low")
|
|
|
|
date_str = item.get("date", "")
|
|
if not date_str:
|
|
continue
|
|
event_time = datetime.fromisoformat(date_str.replace("Z", "+00:00"))
|
|
|
|
events.append(EconomicEvent(
|
|
time=event_time,
|
|
currency=item.get("country", ""),
|
|
impact=impact,
|
|
event=item.get("title", ""),
|
|
forecast=item.get("forecast") or None,
|
|
previous=item.get("previous") or None,
|
|
actual=item.get("actual") or None,
|
|
))
|
|
except Exception:
|
|
continue
|
|
|
|
return events
|
|
|
|
return []
|
|
|
|
def _try_fetch_myfxbook_rss(self) -> List[EconomicEvent]:
|
|
"""Fallback calendar source: MyFXBook economic calendar RSS."""
|
|
url = "https://www.myfxbook.com/rss/forex-economic-calendar-events"
|
|
try:
|
|
resp = self._session.get(url, timeout=10)
|
|
except requests.RequestException as e:
|
|
logger.debug(f"MyFXBook calendar request failed: {e}")
|
|
return []
|
|
|
|
if resp.status_code != 200:
|
|
return []
|
|
|
|
try:
|
|
root = ET.fromstring(resp.content)
|
|
except ET.ParseError:
|
|
return []
|
|
|
|
events = []
|
|
for item in root.iter("item"):
|
|
try:
|
|
title_el = item.find("title")
|
|
pub_el = item.find("pubDate")
|
|
desc_el = item.find("description")
|
|
if title_el is None or title_el.text is None:
|
|
continue
|
|
|
|
title = title_el.text.strip()
|
|
|
|
currency = ""
|
|
title_lower = title.lower()
|
|
for country, cur in self._COUNTRY_TO_CURRENCY.items():
|
|
if title_lower.startswith(country):
|
|
currency = cur
|
|
title = title[len(country):].strip()
|
|
break
|
|
if not currency:
|
|
continue
|
|
|
|
event_time = self._parse_rss_date(
|
|
pub_el.text if pub_el is not None else None
|
|
)
|
|
if event_time is None:
|
|
continue
|
|
|
|
impact = "low"
|
|
desc_text = desc_el.text if desc_el is not None and desc_el.text else ""
|
|
impact_match = re.search(r"sprite-(high|medium|low)-impact", desc_text)
|
|
if impact_match:
|
|
impact = impact_match.group(1)
|
|
|
|
cells = re.findall(r"<td[^>]*>(.*?)</td>", desc_text, re.DOTALL)
|
|
cells = [re.sub(r"<[^>]+>", "", c).strip() for c in cells]
|
|
previous = cells[2] if len(cells) > 2 and cells[2] else None
|
|
forecast = cells[3] if len(cells) > 3 and cells[3] else None
|
|
actual = cells[4] if len(cells) > 4 and cells[4] else None
|
|
|
|
events.append(EconomicEvent(
|
|
time=event_time,
|
|
currency=currency,
|
|
impact=impact,
|
|
event=title,
|
|
forecast=forecast,
|
|
previous=previous,
|
|
actual=actual,
|
|
))
|
|
except Exception:
|
|
continue
|
|
|
|
logger.debug(f"MyFXBook calendar: parsed {len(events)} events")
|
|
return events
|
|
|
|
def _fetch_news_sentiment(self) -> List[NewsSentiment]:
|
|
"""Cached headline fetch + sentiment analysis."""
|
|
return self._cached_fetch("_headlines_cache", self._fetch_news_sentiment_raw)
|
|
|
|
def _fetch_news_sentiment_raw(self) -> List[NewsSentiment]:
|
|
"""Fetch and analyze news headlines for sentiment."""
|
|
headlines = [] # names a tracked currency: what the platform uses
|
|
unmatched = [] # names none: scored and archived, never shown
|
|
seen_titles = set()
|
|
ok_feeds = 0
|
|
undated = 0
|
|
|
|
for feed_url in self.config.rss_feeds:
|
|
host = feed_url.split("/")[2] if "//" in feed_url else feed_url
|
|
kept_before = len(headlines)
|
|
try:
|
|
resp = self._session.get(feed_url, timeout=10)
|
|
if resp.status_code != 200:
|
|
self._record_source(
|
|
f"rss:{host}", False, f"HTTP {resp.status_code}")
|
|
continue
|
|
|
|
root = ET.fromstring(resp.content)
|
|
|
|
for item in root.iter("item"):
|
|
title_el = item.find("title")
|
|
pub_date_el = item.find("pubDate")
|
|
|
|
if title_el is None:
|
|
continue
|
|
|
|
title = title_el.text or ""
|
|
pub_date = self._parse_rss_date(
|
|
pub_date_el.text if pub_date_el is not None else None
|
|
)
|
|
|
|
# Only recent headlines (configurable: headline_max_age_seconds).
|
|
# An item without a parsable pubDate cannot be proven
|
|
# fresh, so it is dropped too — it used to be stamped
|
|
# "now" and never aged out (2026-09-04).
|
|
if pub_date is None:
|
|
undated += 1
|
|
continue
|
|
if (datetime.now(timezone.utc) - pub_date).total_seconds() > self.config.headline_max_age_seconds:
|
|
continue
|
|
|
|
# Dedup within and across feeds so a syndicated story
|
|
# doesn't count twice in the sentiment average. Google
|
|
# News appends " - Publisher", so compare without it.
|
|
title_key = title.lower().rsplit(" - ", 1)[0].strip()
|
|
if title_key in seen_titles:
|
|
continue
|
|
seen_titles.add(title_key)
|
|
|
|
sentiment = self._analyze_headline_sentiment(title)
|
|
currencies = self._extract_currencies(title)
|
|
|
|
link_el = item.find("link")
|
|
desc_el = item.find("description")
|
|
src_el = item.find("source")
|
|
# itertext() survives CDATA and stray child
|
|
# elements where .text stops at the first child.
|
|
desc = _clean_html_text(
|
|
"".join(desc_el.itertext())
|
|
if desc_el is not None else None)
|
|
# Google News descriptions are often just the
|
|
# headline again inside an anchor — worthless.
|
|
if desc and desc.strip() == title.strip():
|
|
desc = None
|
|
publisher = None
|
|
if src_el is not None:
|
|
publisher = (src_el.text or "").strip() or None
|
|
entry = NewsSentiment(
|
|
headline=title,
|
|
source=feed_url,
|
|
timestamp=pub_date or datetime.now(timezone.utc),
|
|
currencies_mentioned=currencies,
|
|
sentiment_score=sentiment,
|
|
keywords=self._extract_keywords(title),
|
|
sentiment_by_currency={
|
|
cur: self._analyze_headline_sentiment(title, cur)
|
|
for cur in currencies
|
|
},
|
|
link=_safe_link(
|
|
link_el.text if link_el is not None else None),
|
|
description=desc,
|
|
publisher=publisher,
|
|
)
|
|
# A headline naming no tracked currency stays invisible
|
|
# to the platform (no per-currency mean, no display,
|
|
# no panel) but is scored and archived since
|
|
# 2026-09-17: it was 23% of a day's feed, and DeepSeek
|
|
# gave 10 of those 24 a gold direction — the Iran /
|
|
# Hormuz stories were the strongest gold headlines of
|
|
# the day. Training data only.
|
|
if currencies:
|
|
headlines.append(entry)
|
|
else:
|
|
unmatched.append(entry)
|
|
|
|
ok_feeds += 1
|
|
self._record_source(
|
|
f"rss:{host}", True,
|
|
extra=f"{len(headlines) - kept_before} kept")
|
|
except Exception as e:
|
|
self._record_source(
|
|
f"rss:{host}", False, f"{type(e).__name__}: {e}")
|
|
logger.debug(f"Failed to fetch RSS from {feed_url}: {e}")
|
|
|
|
if undated:
|
|
logger.debug(f"RSS: dropped {undated} headlines without a publish time")
|
|
if unmatched:
|
|
logger.debug(f"RSS: {len(unmatched)} headlines name no tracked "
|
|
"currency (scored and archived, not shown)")
|
|
self._record_source(
|
|
"headlines", ok_feeds > 0,
|
|
detail="all feeds failed" if ok_feeds == 0 else "",
|
|
extra=(f"{len(headlines)} headlines from "
|
|
f"{ok_feeds}/{len(self.config.rss_feeds)} feeds"))
|
|
|
|
# Score and archive EVERYTHING fetched; hand back only the
|
|
# currency-matched headlines so display, per-currency means, the
|
|
# panel and the summary see exactly what they saw before.
|
|
scored = headlines + unmatched
|
|
try:
|
|
self._apply_llm_sentiment(scored)
|
|
except Exception as e:
|
|
logger.debug(f"LLM sentiment failed, keyword scores stand: {e}")
|
|
|
|
# Archive BEFORE any downstream filtering. get_fundamental_context
|
|
# drops every headline inside ±NEUTRAL_HEADLINE_THRESHOLD, so a
|
|
# corpus built from what the context carries holds almost no
|
|
# genuinely neutral headlines and would teach a trained scorer
|
|
# that "neutral" barely exists. Never raises.
|
|
# Guarded at the call site as well as inside archive_headlines:
|
|
# arguments are evaluated BEFORE that function's own try/except, so
|
|
# a raising get_llm_client() would escape it. Archiving is a side
|
|
# effect — it must never cost us the headlines we just fetched.
|
|
try:
|
|
news_archive.archive_headlines(
|
|
scored,
|
|
teacher_model=getattr(get_llm_client(), "_model", None),
|
|
teacher_prompt_sha1=_LLM_SENTIMENT_PROMPT_SHA1)
|
|
except Exception as e:
|
|
logger.debug(f"news archive skipped, headlines unaffected: {e}")
|
|
|
|
return headlines
|
|
|
|
def _analyze_headline_sentiment(
|
|
self, headline: str, for_currency: Optional[str] = None
|
|
) -> float:
|
|
"""
|
|
Simple keyword-based sentiment analysis.
|
|
|
|
Returns -1.0 (very bearish) to +1.0 (very bullish).
|
|
|
|
When for_currency is "XAU", monetary-policy keywords contribute
|
|
with inverted sign (see GOLD_INVERSE_KEYWORDS).
|
|
|
|
Note: This is intentionally simple. NLP models like FinBERT
|
|
would be more accurate but require large dependencies. This catches
|
|
the most impactful sentiment signals from headlines.
|
|
"""
|
|
headline_lower = headline.lower()
|
|
invert_policy = for_currency == "XAU"
|
|
|
|
bullish_score = 0
|
|
bearish_score = 0
|
|
for kw in self.BULLISH_KEYWORDS:
|
|
if kw in headline_lower:
|
|
if invert_policy and kw in self.GOLD_INVERSE_KEYWORDS:
|
|
bearish_score += 1
|
|
else:
|
|
bullish_score += 1
|
|
for kw in self.BEARISH_KEYWORDS:
|
|
if kw in headline_lower:
|
|
if invert_policy and kw in self.GOLD_INVERSE_KEYWORDS:
|
|
bullish_score += 1
|
|
else:
|
|
bearish_score += 1
|
|
|
|
total = bullish_score + bearish_score
|
|
if total == 0:
|
|
return 0.0
|
|
|
|
return (bullish_score - bearish_score) / total
|
|
|
|
# ── DeepSeek enrichment (display/warning-only, keyword fallback) ──
|
|
|
|
def _llm_headline_scores(
|
|
self, texts: List[str]
|
|
) -> Optional[List[Tuple[float, float]]]:
|
|
"""One batched DeepSeek call → [(overall, xau), ...] aligned with
|
|
`texts`, or None on any failure/mismatch (caller keeps keyword
|
|
scores). Tests stub this method directly."""
|
|
result = get_llm_client().chat_json(
|
|
_LLM_SENTIMENT_SYSTEM, {"headlines": texts})
|
|
if not result:
|
|
return None
|
|
scores = result.get("scores")
|
|
if not isinstance(scores, list) or len(scores) != len(texts):
|
|
return None
|
|
out = []
|
|
for item in scores:
|
|
try:
|
|
s = max(-1.0, min(1.0, float(item[0])))
|
|
x = max(-1.0, min(1.0, float(item[1])))
|
|
except (TypeError, KeyError, ValueError, IndexError):
|
|
return None
|
|
out.append((s, x))
|
|
return out
|
|
|
|
def _apply_llm_sentiment(self, headlines: List[NewsSentiment]) -> None:
|
|
"""Overwrite keyword sentiment on `headlines` with LLM scores.
|
|
|
|
Runs inside _cached_fetch, i.e. UNDER _cache_lock — cache access
|
|
is direct and the (rare) HTTP calls block sibling scans at most
|
|
once per cache window, matching the existing shared-fetch design.
|
|
Only headlines missing from the cache are sent, in batches of
|
|
_LLM_BATCH_CAP; a failed batch changes nothing and ends the
|
|
cycle's sending.
|
|
"""
|
|
if not headlines or not get_llm_client().available:
|
|
return
|
|
cache = NewsEngine._llm_sent_cache
|
|
keyed = [(hashlib.sha1(h.headline.encode()).hexdigest(), h)
|
|
for h in headlines]
|
|
# Every new headline goes out this cycle, one call per batch.
|
|
# Until 2026-09-17 only the first 25 went and the overflow stayed
|
|
# keyword-scored (31 of the first 1,577 archived headlines never
|
|
# got a DeepSeek label that way). A failed call ends the loop: a
|
|
# timeout or the daily cap would fail the next one too, and what
|
|
# is left stays keyword-scored and, still uncached, goes out with
|
|
# the next window.
|
|
new = [(k, h) for k, h in keyed if k not in cache]
|
|
for start in range(0, len(new), self._LLM_BATCH_CAP):
|
|
batch = new[start:start + self._LLM_BATCH_CAP]
|
|
scores = self._llm_headline_scores([h.headline for _, h in batch])
|
|
if scores is None:
|
|
break
|
|
for (k, _), sc in zip(batch, scores):
|
|
cache[k] = sc
|
|
while len(cache) > self._LLM_SENT_CACHE_MAX:
|
|
cache.pop(next(iter(cache)))
|
|
for k, h in keyed:
|
|
sc = cache.get(k)
|
|
if sc is None:
|
|
continue
|
|
overall, xau = sc
|
|
h.sentiment_score = overall
|
|
h.sentiment_by_currency = {
|
|
cur: (xau if cur == "XAU" else overall)
|
|
for cur in h.currencies_mentioned
|
|
}
|
|
h.xau_score = xau
|
|
h.score_source = "llm"
|
|
|
|
def _llm_event_direction(self, event: EconomicEvent) -> Optional[str]:
|
|
"""Gold direction for a released surprise, e.g. "bearish: strong
|
|
jobs lift yields". None → caller keeps the numeric description.
|
|
Cached per release; failures are NOT cached (retry next scan).
|
|
|
|
The actual-vs-forecast comparison is precomputed and sent as
|
|
"vs_forecast": the 2026-08-06 validation found the model reads
|
|
close K/M-suffixed pairs backwards ~11% of the time (e.g. claims
|
|
202K vs 217K called "higher than forecast") when left to compare
|
|
the raw numbers itself."""
|
|
key = (event.event, event.currency, str(event.actual or ""))
|
|
with NewsEngine._cache_lock:
|
|
if key in NewsEngine._llm_event_cache:
|
|
return NewsEngine._llm_event_cache[key] or None
|
|
client = get_llm_client()
|
|
if not client.available:
|
|
return None
|
|
vs_forecast = {
|
|
"positive": "above forecast",
|
|
"negative": "below forecast",
|
|
"inline": "in line with forecast",
|
|
}.get(event.surprise or "", "unknown")
|
|
result = client.chat_json(_LLM_EVENT_SYSTEM, {
|
|
"event": event.event,
|
|
"currency": event.currency,
|
|
"actual": event.actual,
|
|
"forecast": event.forecast,
|
|
"previous": event.previous,
|
|
"vs_forecast": vs_forecast,
|
|
})
|
|
if not result:
|
|
return None
|
|
direction = result.get("gold_direction")
|
|
if direction not in ("bullish", "bearish", "neutral"):
|
|
return None
|
|
reason = str(result.get("reason", "")).strip()[:80]
|
|
text = "" if direction == "neutral" else (
|
|
f"{direction}: {reason}" if reason else direction)
|
|
with NewsEngine._cache_lock:
|
|
NewsEngine._llm_event_cache[key] = text
|
|
while len(NewsEngine._llm_event_cache) > self._LLM_EVENT_CACHE_MAX:
|
|
NewsEngine._llm_event_cache.pop(
|
|
next(iter(NewsEngine._llm_event_cache)))
|
|
return text or None
|
|
|
|
def _llm_market_summary(self, context: FundamentalContext) -> str:
|
|
"""2-3 sentence plain-language news note for alerts; "" when
|
|
unavailable. Headlines go newest-first with the platform's own
|
|
gold direction so the note agrees with the score badges. Cached
|
|
(success OR failure) for the context-cache TTL so at most one
|
|
call is made per window."""
|
|
with NewsEngine._cache_lock:
|
|
ts, cached = NewsEngine._llm_summary_cache
|
|
if time.time() - ts < self.config.context_cache_ttl_seconds:
|
|
return cached
|
|
high_impact = [e for e in context.events_today if e.is_high_impact]
|
|
if not context.news_headlines and not high_impact:
|
|
return ""
|
|
client = get_llm_client()
|
|
if not client.available:
|
|
return ""
|
|
oldest = datetime.min.replace(tzinfo=timezone.utc)
|
|
recent = sorted(
|
|
context.news_headlines,
|
|
key=lambda h: getattr(h, "timestamp", None) or oldest,
|
|
reverse=True,
|
|
)[:15]
|
|
result = client.chat_json(_LLM_SUMMARY_SYSTEM, {
|
|
"headlines": [
|
|
{"headline": h.headline, "gold": _gold_direction_label(h)}
|
|
for h in recent
|
|
],
|
|
"events": [
|
|
{"event": e.event, "currency": e.currency,
|
|
"actual": e.actual, "forecast": e.forecast,
|
|
"released": not e.is_upcoming}
|
|
for e in high_impact[:10]
|
|
],
|
|
})
|
|
summary = ""
|
|
if result:
|
|
summary = str(result.get("summary", "")).strip()[:400]
|
|
with NewsEngine._cache_lock:
|
|
NewsEngine._llm_summary_cache = (time.time(), summary)
|
|
return summary
|
|
|
|
def _extract_currencies(self, headline: str) -> List[str]:
|
|
"""Extract mentioned currencies from a headline."""
|
|
headline_lower = headline.lower()
|
|
return [
|
|
currency
|
|
for currency, pattern in self._CURRENCY_PATTERNS.items()
|
|
if pattern.search(headline_lower)
|
|
]
|
|
|
|
def _extract_keywords(self, headline: str) -> List[str]:
|
|
"""Extract relevant trading keywords from headline."""
|
|
headline_lower = headline.lower()
|
|
return [kw for kw in self.BULLISH_KEYWORDS | self.BEARISH_KEYWORDS if kw in headline_lower]
|
|
|
|
@staticmethod
|
|
def _parse_rss_date(date_str: Optional[str]) -> Optional[datetime]:
|
|
"""Parse various RSS date formats into a timezone-aware datetime."""
|
|
if not date_str:
|
|
return None
|
|
|
|
# RFC 822/2822 forms, with or without seconds, numeric offsets AND
|
|
# named zones (GMT, EST/EDT, PST…). The old strptime "%Z" path
|
|
# accepted ANY abbreviation but attached no offset, so e.g. an EST
|
|
# pubDate was silently stamped UTC — a 5-hour error.
|
|
try:
|
|
dt = parsedate_to_datetime(date_str.strip())
|
|
if dt is not None:
|
|
if dt.tzinfo is None: # unknown zone name — assume UTC
|
|
return dt.replace(tzinfo=timezone.utc)
|
|
return dt.astimezone(timezone.utc)
|
|
except (TypeError, ValueError):
|
|
pass
|
|
|
|
formats = [
|
|
"%Y-%m-%dT%H:%M:%S%z", # ISO 8601 with offset
|
|
"%Y-%m-%d %H:%M:%S", # ISO 8601 naive (assume UTC)
|
|
]
|
|
for fmt in formats:
|
|
try:
|
|
dt = datetime.strptime(date_str.strip(), fmt)
|
|
# If strptime already attached tzinfo (from %z), keep it.
|
|
# Otherwise the string had no timezone info — assume UTC.
|
|
if dt.tzinfo is not None:
|
|
return dt
|
|
return dt.replace(tzinfo=timezone.utc)
|
|
except ValueError:
|
|
continue
|
|
return None
|