"""
Forex Signal Intelligence Platform - News & Sentiment Engine
=============================================================
Monitors economic calendar events and news sentiment to provide
fundamental context for technical signals.

Key principle: High-impact news events override technical signals.
A perfect technical setup can be destroyed by an unexpected NFP print.
"""

import copy
import hashlib
import html
import logging
import math
import re
import threading
import time
import xml.etree.ElementTree as ET
from dataclasses import dataclass, field
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
from typing import Dict, List, Optional, Tuple

import requests

import app_config
import news_archive
import news_model
from config import NewsConfig
from llm_client import get_llm_client

logger = logging.getLogger(__name__)


# ── DeepSeek prompts ─────────────────────────────────────────────────
# STATIC constants on purpose: DeepSeek prices a repeated byte-identical
# prompt prefix at the cache-hit rate (~50x cheaper), so nothing here may
# be interpolated or timestamped. All variable content travels in the
# user payload. LLM output is display/warning-only — it never gates
# signals (see NewsConfig.sentiment_affects_signals).

_LLM_SENTIMENT_SYSTEM = """\
You are a forex/gold news analyst. You receive JSON:
{"headlines": ["...", ...]}
For EACH headline, in the same order, score:
- "s": overall market sentiment of the headline, -1.0 (very bearish)
  to 1.0 (very bullish), 0.0 if irrelevant or neutral.
- "xau": the effect on the GOLD (XAU/USD) price, -1.0 (gold down)
  to 1.0 (gold up), 0.0 if irrelevant.
Gold doctrine: hawkish Fed / rate hikes / rising real yields / strong
dollar are gold-bearish; easing / rate cuts / safe-haven flows /
geopolitical stress / weak dollar are gold-bullish.
Handle negation correctly ("no rate hike" is not hawkish).
Reply with ONLY JSON: {"scores": [[s, xau], ...]} — exactly one
[s, xau] pair per input headline, in the same order, nothing else.

Examples:
"Fed signals faster rate hikes as inflation stays hot" -> [0.4, -0.7]
"Gold surges to record high on safe-haven demand" -> [0.8, 0.9]
"Dollar slips after soft jobs report; Treasury yields fall" -> [-0.4, 0.6]
"""

_LLM_EVENT_SYSTEM = """\
You are a forex/gold analyst. You receive one released economic event as
JSON: {"event", "currency", "actual", "forecast", "previous",
"vs_forecast"}. "vs_forecast" states whether the actual print came in
above or below the forecast; it is computed programmatically — trust it
over your own comparison of the numbers. If it is "unknown", compare
actual and forecast yourself.
Decide the likely effect of this RELEASE on the GOLD (XAU/USD) price.
A stronger-than-forecast USD economy or a hawkish implication is usually
gold-bearish; weaker data or a dovish implication is usually
gold-bullish.
Inverted metrics — for unemployment rate and unemployment/jobless
claims a LOWER number means a STRONGER labor market: below forecast =
strong economy = usually gold-bearish; above forecast = weak economy =
usually gold-bullish.
Example: {"event": "Unemployment Claims", "actual": "202K", "forecast":
"217K", "vs_forecast": "below forecast"} ->
{"gold_direction": "bearish", "reason": "fewer claims show strong labor market"}
Reply with ONLY JSON:
{"gold_direction": "bullish"|"bearish"|"neutral", "reason": "<max 10 words>"}
"""

_LLM_SUMMARY_SYSTEM = """\
You write a short news note about the gold price for someone who has
never traded. You receive JSON with recent "headlines" (newest first;
each carries "gold": "up" or "down", the platform's own reading of how
that story pushes the gold price — trust it) and today's high-impact
"events" (released: true means the number is out).
Say three things, in this order: what is happening to gold right now,
why it matters for the gold price, and the next scheduled report to
watch if one is listed. Use only the headlines and events given; if
they are thin, say so plainly.
Style: plain everyday English, short simple sentences, no acronyms, no
abbreviations, no trader words. Write "the US central bank", not "the
Fed"; "interest rates", not "yields" or "hawkish"; "a report on US
prices", not "CPI" or "PCE"; "gold", never "XAU". No advice, no hedging
boilerplate. 2-3 sentences, max 55 words.
Reply with ONLY JSON: {"summary": "..."}

Examples:
{"summary": "Gold is falling today. The US central bank hinted it may raise interest rates, which makes the dollar stronger, and a stronger dollar usually pushes gold down. The next thing to watch is Thursday's report on US prices."}
{"summary": "Gold is climbing. Fresh fighting in the Middle East has people moving money into gold as a safe place, and a weaker dollar is helping too. No major US reports are due today."}
"""


#: Identity of the labelling prompt, stored on every archived LLM score.
#: If the prompt changes, scores before and after are answers to different
#: questions and must not be pooled into one training set.
_LLM_SENTIMENT_PROMPT_SHA1 = hashlib.sha1(
    _LLM_SENTIMENT_SYSTEM.encode("utf-8")).hexdigest()


_RELEASE_VALUE_RE = re.compile(r"(-?\d+(?:\.\d+)?)\s*([kmbt])?", re.IGNORECASE)
_RELEASE_MULTIPLIERS = {"k": 1e3, "m": 1e6, "b": 1e9, "t": 1e12}


def _parse_release_value(raw: Optional[str]) -> Optional[float]:
    """Parse a calendar release value like '0.3%', '-1.7%', '228K', '1.1M'."""
    if not raw:
        return None
    match = _RELEASE_VALUE_RE.search(raw.replace(",", ""))
    if not match:
        return None
    multiplier = _RELEASE_MULTIPLIERS.get((match.group(2) or "").lower(), 1.0)
    return float(match.group(1)) * multiplier


_DESC_MAX_CHARS = 500
_MAX_LINK_CHARS = 2048


def _clean_html_text(raw: Optional[str],
                     limit: int = _DESC_MAX_CHARS) -> Optional[str]:
    """RSS <description> → plain text: unescape entities, strip tags,
    collapse whitespace, truncate. Unescape and strip both run twice
    because feeds routinely double-encode (&amp;lt;p&amp;gt; wrapping
    real markup) — the second unescape can reveal fresh tags."""
    if not raw:
        return None
    text = html.unescape(raw)
    text = re.sub(r"<[^>]+>", " ", text)
    text = html.unescape(text)
    text = re.sub(r"<[^>]+>", " ", text)
    text = re.sub(r"\s+", " ", text).strip()
    if not text:
        return None
    if len(text) > limit:
        text = text[:limit].rstrip() + "…"
    return text


def _safe_link(raw: Optional[str]) -> Optional[str]:
    """Article URL for display: http(s) only (an RSS <link> is untrusted
    and lands in an href), bounded length (Google News redirect URLs run
    1-2 KB; anything past that is bloat, not a link)."""
    if not raw:
        return None
    link = raw.strip()
    if not link.lower().startswith(("http://", "https://")):
        return None
    if len(link) > _MAX_LINK_CHARS:
        return None
    return link


@dataclass
class EconomicEvent:
    """Single economic calendar event."""
    time: datetime
    currency: str
    impact: str  # "low", "medium", "high"
    event: str
    forecast: Optional[str] = None
    previous: Optional[str] = None
    actual: Optional[str] = None

    @property
    def surprise(self) -> Optional[str]:
        """Actual vs forecast: "positive"/"negative"/"inline", or None when
        either value is missing or non-numeric. Direction is numeric only —
        it does not know whether higher is market-bullish for this event."""
        actual = _parse_release_value(self.actual)
        forecast = _parse_release_value(self.forecast)
        if actual is None or forecast is None:
            return None
        if actual > forecast:
            return "positive"
        if actual < forecast:
            return "negative"
        return "inline"

    @property
    def is_upcoming(self) -> bool:
        return self.time > datetime.now(timezone.utc)

    @property
    def minutes_until(self) -> float:
        delta = self.time - datetime.now(timezone.utc)
        return delta.total_seconds() / 60

    @property
    def is_high_impact(self) -> bool:
        return self.impact == "high"


@dataclass
class NewsSentiment:
    """News sentiment analysis result."""
    headline: str
    source: str
    timestamp: datetime
    currencies_mentioned: List[str]
    sentiment_score: float  # -1.0 (bearish) to +1.0 (bullish)
    keywords: List[str]
    # Per-currency score: monetary-policy keywords flip sign for XAU
    # (hawkish is dollar-bullish but gold-bearish).
    sentiment_by_currency: Dict[str, float] = field(default_factory=dict)
    # Display-only enrichment (dashboard detail dialog). Optional with
    # defaults so existing constructor calls keep working.
    link: Optional[str] = None         # article URL (Google News = redirect)
    description: Optional[str] = None  # plain text, tags stripped, ≤500 chars
    publisher: Optional[str] = None    # Google News <source> text, else None
    # Which scorer produced sentiment_score / sentiment_by_currency:
    # "keyword" (the built-in scorer, ~31% on the 2026-08-06 golden set),
    # "llm" (DeepSeek, 78/78 on the same set) or "model" (the local XAU
    # transformer, news_model.py, 72/78; since 2026-09-27). "model" means
    # the model scored xau_score and XAU's per-currency entry; every
    # other currency's entry, and sentiment_score when the headline names
    # another currency, remain keyword-scored — the model has a gold head
    # only. Without the tag the scorers are indistinguishable in the
    # archive and a model trained on the mix inherits the keyword errors.
    score_source: str = "keyword"
    # The gold-specific score for THIS headline (DeepSeek's, or the local
    # model's raw signed score), whatever currencies it names.
    # sentiment_by_currency["XAU"] carries it only when the headline
    # matched a gold keyword; for a pure Fed / dollar / yields headline it
    # used to be discarded, which left 930 of the first 1,546 archived
    # DeepSeek rows without a gold label (eval 2026-09-17). Nothing
    # displayed or averaged reads it — the archive stores it as training
    # data. None until DeepSeek or the model has scored the headline (the
    # keyword scorer never sets it).
    xau_score: Optional[float] = None


@dataclass
class FundamentalContext:
    """Complete fundamental/news context for signal decisions."""
    events_today: List[EconomicEvent] = field(default_factory=list)
    events_upcoming_1h: List[EconomicEvent] = field(default_factory=list)
    high_impact_warning: bool = False
    warning_message: str = ""
    news_sentiment: Dict[str, float] = field(default_factory=dict)  # currency -> sentiment
    news_headlines: List[NewsSentiment] = field(default_factory=list)
    calendar_available: bool = False  # True if live calendar data was fetched
    llm_summary: str = ""  # DeepSeek market-context blurb (display-only)


# Headlines whose every relevant score is inside ±NEUTRAL_HEADLINE_THRESHOLD
# lean neither way: they carry no direction for display and only dilute the
# per-currency mean. Same threshold agent_panel uses for its macro payload.
NEUTRAL_HEADLINE_THRESHOLD = 0.1


def _is_neutral_headline(h) -> bool:
    """True when no per-currency score on the headline leans either way.

    A headline whose overall score is 0.0 but whose gold-specific score is
    -0.7 is NOT neutral. Missing, empty or non-finite scores count as
    neutral (they cannot be shown as leaning one way)."""
    by_currency = getattr(h, "sentiment_by_currency", None) or {}
    overall = getattr(h, "sentiment_score", None)
    values = []
    for currency in (getattr(h, "currencies_mentioned", None) or []):
        value = by_currency.get(currency, overall)
        if isinstance(value, (int, float)) and math.isfinite(value):
            values.append(abs(float(value)))
    return not values or max(values) < NEUTRAL_HEADLINE_THRESHOLD


def _gold_direction_label(h) -> str:
    """"up" / "down" / "flat" for the market-summary payload — the same
    XAU-first, USD-inverted reading agent_panel uses for its macro persona."""
    by_currency = getattr(h, "sentiment_by_currency", None) or {}
    xau = by_currency.get("XAU")
    usd = by_currency.get("USD")
    if isinstance(xau, (int, float)) and math.isfinite(xau):
        score = float(xau)
    elif isinstance(usd, (int, float)) and math.isfinite(usd):
        score = -float(usd)
    else:
        raw = getattr(h, "sentiment_score", None)
        score = float(raw) if isinstance(raw, (int, float)) and math.isfinite(raw) else 0.0
    if score > 0:
        return "up"
    if score < 0:
        return "down"
    return "flat"


class NewsEngine:
    """
    Economic calendar and news sentiment analyzer.

    Provides fundamental context that modifies technical signal confidence.
    """

    # Keywords that indicate directional sentiment
    BULLISH_KEYWORDS = {
        "rate hike", "hawkish", "strong growth", "beat expectations",
        "above forecast", "above estimate", "above expectation",
        "surplus", "expansion", "employment gains",
        "strong jobs", "gdp growth", "inflation rising", "tightening",
        "positive surprise", "record high", "bullish", "rally",
        "stronger than expected", "upbeat",
        "beats", "surges", "soars", "jumps", "climbs", "rises",
        "higher than expected", "better than expected",
        "strong data", "robust", "resilient", "accelerates",
    }

    BEARISH_KEYWORDS = {
        "rate cut", "dovish", "recession", "slowdown", "contraction",
        "below forecast", "below estimate", "below expectation",
        "deficit", "miss expectations", "weak data",
        "unemployment rise", "negative surprise", "bearish", "selloff",
        "weaker than expected", "downbeat", "easing", "stimulus",
        "misses", "slides", "drops", "falls", "plunges", "tumbles",
        "weakens", "slumps", "declines", "retreats",
        "lower than expected", "worse than expected",
        "disappointing", "soft data",
    }

    # Monetary-policy keywords whose direction inverts for gold: tighter
    # policy lifts the dollar and yields, weighing on non-yielding gold;
    # easing does the opposite. Only unambiguous policy words — economic
    # data words (beats/misses) keep their normal direction.
    GOLD_INVERSE_KEYWORDS = {
        "rate hike", "hawkish", "tightening",
        "rate cut", "dovish", "easing", "stimulus",
    }

    # Currency-specific keywords for headline parsing. Matched with word
    # boundaries — bare "us"/"eu"/"uk" are deliberately excluded because as
    # substrings they false-positive constantly ("versus" → USD).
    CURRENCY_KEYWORDS = {
        "USD": ["dollar", "fed", "federal reserve", "fomc", "treasury", "american", "nonfarm", "nfp"],
        "EUR": ["euro", "ecb", "eurozone", "german", "french"],
        "GBP": ["pound", "sterling", "boe", "bank of england", "british"],
        "JPY": ["yen", "boj", "bank of japan", "japanese"],
        "CHF": ["franc", "snb", "swiss"],
        "AUD": ["aussie", "rba", "australian", "australia"],
        "CAD": ["loonie", "boc", "canadian", "canada"],
        "NZD": ["kiwi", "rbnz", "new zealand"],
        "XAU": ["gold", "precious metal", "safe haven", "bullion"],
    }

    # One compiled alternation per currency (code + keywords). (?<![a-z]) /
    # (?![a-z]) instead of \b so "0.3%fed" still matches but "versus",
    # "audit", "cascade" don't.
    _CURRENCY_PATTERNS = {
        cur: re.compile(
            r"(?<![a-z])(?:"
            + "|".join(re.escape(kw) for kw in [cur.lower()] + kws)
            + r")(?![a-z])"
        )
        for cur, kws in CURRENCY_KEYWORDS.items()
    }

    _calendar_backoff_until: float = 0.0

    # Fetched-data cache, class-level: a fresh NewsEngine is built per scan,
    # so instance state would never be reused. Guarded by _cache_lock since
    # scans run concurrently in a thread pool.
    _cache_lock = threading.Lock()
    _calendar_cache: Tuple[float, List[EconomicEvent]] = (0.0, [])
    _headlines_cache: Tuple[float, List[NewsSentiment]] = (0.0, [])
    _EMPTY_RESULT_TTL = 60.0  # retry sooner after a failed/empty fetch

    # DeepSeek result caches. Headlines and releases are immutable, so a
    # scored one never needs re-scoring; bounded by dropping the oldest
    # insertion. _llm_sent_cache is only touched under _cache_lock (its
    # writer runs inside _cached_fetch); the other two take the lock
    # around cache access but keep HTTP calls outside it.
    _llm_sent_cache: Dict[str, Tuple[float, float]] = {}   # sha1 → (s, xau)
    _llm_event_cache: Dict[Tuple[str, str, str, str, str], str] = {}
    _llm_summary_cache: Tuple[float, str] = (0.0, "")
    _LLM_SENT_CACHE_MAX = 400
    _LLM_EVENT_CACHE_MAX = 100
    #: Headlines per DeepSeek sentiment call — the reply must fit the
    #: client's max_tokens (~10 tokens per headline, 2000 max).
    _LLM_BATCH_CAP = 25

    # primary-mode DeepSeek audit sample (news_model.deepseek_audit_per_day):
    # how many model-scored headlines went to DeepSeek today, archive-only.
    # Written under _cache_lock like _llm_sent_cache.
    _audit_day: str = ""
    _audit_sent_today: int = 0

    # Per-source fetch health for status displays (telegram /status).
    # name → {"ok": ts|None, "err": ts|None, "detail": str, "extra": str}.
    # All writers run under _cache_lock (fetches happen inside
    # _cached_fetch); readers render without the lock — worst case a
    # display line is one cycle stale. A source is "failing" when its
    # last error is newer than its last success.
    _source_health: Dict[str, dict] = {}

    def __init__(self, config: NewsConfig):
        self.config = config
        self._session = requests.Session()
        self._session.headers.update({
            "User-Agent": "Mozilla/5.0 (compatible; ForexIntel/1.0)"
        })
        # Create the local scorer's ONNX session off the scan thread so
        # the first news cycle does not pay 3-5 s under _cache_lock.
        # Idempotent per process (a NewsEngine is built for every scan).
        try:
            if app_config.news_model_mode() != "off":
                news_model.warm_up_in_background()
        except Exception as e:  # never let the warm-up cost a scan
            logger.debug(f"news model warm-up skipped: {e}")

    def get_fundamental_context(
        self, currencies: Optional[List[str]] = None
    ) -> FundamentalContext:
        """
        Get complete fundamental context for signal decisions.

        Combines economic calendar + news sentiment into actionable context.
        """
        if currencies is None:
            currencies = self.config.tracked_currencies

        context = FundamentalContext()

        # 1. Economic calendar events
        try:
            events = self._fetch_economic_calendar(currencies)
            now = datetime.now(timezone.utc)

            # Compare dates in UTC — feed timestamps carry their own offset
            # (FF sends US-Eastern), so a late-evening ET event would
            # otherwise land on the wrong day.
            context.events_today = [
                e for e in events
                if e.time.astimezone(timezone.utc).date() == now.date()
            ]
            context.events_upcoming_1h = [
                e for e in events
                if 0 < e.minutes_until <= 60
            ]
            context.calendar_available = bool(events)

            # Check for high-impact warnings
            for event in events:
                if event.is_high_impact and 0 < event.minutes_until <= self.config.pre_event_buffer_minutes:
                    context.high_impact_warning = True
                    context.warning_message = (
                        f"HIGH IMPACT: {event.currency} {event.event} "
                        f"in {event.minutes_until:.0f} min — signals may be unreliable"
                    )
                    break

        except Exception as e:
            logger.warning(f"Economic calendar fetch failed: {e}")

        # 2. News sentiment
        try:
            headlines = self._fetch_news_sentiment()
            # Drop headlines that lean neither way (2026-09-04). Applied
            # here rather than in the fetch so the class-level cache and
            # the feed-health counts still describe what the feeds sent;
            # every consumer (aggregate score, summary, dashboard, panel,
            # signal log) reads context.news_headlines.
            kept = [h for h in headlines if not _is_neutral_headline(h)]
            if len(kept) != len(headlines):
                logger.debug(
                    f"News: dropped {len(headlines) - len(kept)} neutral "
                    f"headlines, {len(kept)} kept")
            headlines = kept
            context.news_headlines = headlines

            # Aggregate sentiment per currency (per-currency score where
            # available, so gold-inverted policy keywords are respected)
            for currency in currencies:
                scores = [
                    h.sentiment_by_currency.get(currency, h.sentiment_score)
                    for h in headlines
                    if currency in h.currencies_mentioned
                ]
                if scores:
                    context.news_sentiment[currency] = sum(scores) / len(scores)

        except Exception as e:
            logger.warning(f"News sentiment fetch failed: {e}")

        # 3. LLM market-context summary (display-only)
        try:
            context.llm_summary = self._llm_market_summary(context)
        except Exception as e:
            logger.debug(f"LLM market summary failed: {e}")

        return context

    @staticmethod
    def _parse_pair(pair_symbol: str) -> Tuple[str, str]:
        base, sep, quote = pair_symbol.partition("/")
        if not sep:
            raise ValueError(f"pair symbol must be BASE/QUOTE, got {pair_symbol!r}")
        return base, quote

    def get_event_risk_level(self, pair_symbol: str, context: FundamentalContext) -> Tuple[str, str]:
        """
        Get event risk level for a specific pair.
        Returns (risk_level, description).
        risk_level: "clear", "caution", "danger"
        """
        base, quote = self._parse_pair(pair_symbol)

        # Check upcoming high-impact events. Grade from the SOONEST
        # matching event, not the first listed — an unsorted feed could
        # otherwise report "caution" for a far event while a nearer one
        # warrants "danger".
        soonest = None
        for event in context.events_upcoming_1h:
            if event.is_high_impact and event.currency in (base, quote):
                if soonest is None or event.minutes_until < soonest.minutes_until:
                    soonest = event
        if soonest is not None:
            mins = soonest.minutes_until
            if mins <= self.config.pre_event_buffer_minutes:
                return "danger", f"{soonest.currency} {soonest.event} in {mins:.0f}min"
            return "caution", f"{soonest.currency} {soonest.event} in {mins:.0f}min"

        # Check for recent high-impact releases. A surprise print (actual
        # away from forecast) keeps the market unstable longer than an
        # inline release, so it gets a wider post-event window.
        for event in context.events_today:
            if event.is_high_impact and event.currency in (base, quote) and not event.is_upcoming:
                surprise = event.surprise
                surprised = surprise in ("positive", "negative")
                window = (
                    self.config.post_event_surprise_buffer_minutes
                    if surprised
                    else self.config.post_event_buffer_minutes
                )
                if event.minutes_until > -window:
                    if surprised:
                        desc = (
                            f"{event.currency} {event.event} released — {surprise} surprise "
                            f"(actual {event.actual} vs forecast {event.forecast})"
                        )
                        # LLM enrichment touches the DESCRIPTION only —
                        # `surprise` is numeric-direction and does not know
                        # whether a beat is gold-bullish; the LLM does.
                        gold_dir = self._llm_event_direction(event)
                        if gold_dir:
                            desc += f" — gold {gold_dir}"
                        return "caution", desc
                    return "caution", f"{event.currency} {event.event} just released"

        return "clear", "No significant events"

    def get_pair_sentiment(
        self, pair_symbol: str, context: FundamentalContext
    ) -> Tuple[float, str]:
        """
        Get net sentiment for a pair based on news.
        Returns (score, description) where score is -1 to +1.
        Positive = bullish for the pair, negative = bearish.
        """
        base, quote = self._parse_pair(pair_symbol)

        base_sent = context.news_sentiment.get(base, 0.0)
        quote_sent = context.news_sentiment.get(quote, 0.0)

        # Net sentiment: positive base or negative quote = bullish for pair
        net = base_sent - quote_sent

        if abs(net) < 0.1:
            desc = "Neutral news sentiment"
        elif net > 0.3:
            desc = f"Bullish sentiment ({base} positive / {quote} negative)"
        elif net > 0:
            desc = "Slightly bullish sentiment"
        elif net < -0.3:
            desc = f"Bearish sentiment ({base} negative / {quote} positive)"
        else:
            desc = "Slightly bearish sentiment"

        return net, desc

    def get_week_events(
        self, currencies: Optional[List[str]] = None
    ) -> List[EconomicEvent]:
        """Full cached calendar week (past and upcoming), filtered to
        `currencies` (default: tracked_currencies), sorted by time.

        Read-only accessor for display surfaces (the web dashboard) —
        rides the same class-level cache as signal scans, so it adds no
        network I/O. No production signal path calls this.
        """
        if currencies is None:
            currencies = self.config.tracked_currencies
        events = self._cached_fetch("_calendar_cache", self._fetch_calendar_raw)
        return sorted(
            (e for e in events if e.currency in currencies),
            key=lambda e: e.time,
        )

    def peek_week_events(
        self, currencies: Optional[List[str]] = None
    ) -> List[EconomicEvent]:
        """get_week_events WITHOUT the refresh: whatever calendar the
        class-level cache holds right now, however old, filtered and sorted
        the same way. Never fetches and never waits on the cache lock, for
        a caller that must not add a network wait: the agent panel runs
        while an alert is waiting to go out, and a refetch with both
        calendar sources down takes about 40 seconds."""
        if currencies is None:
            currencies = self.config.tracked_currencies
        _stamp, events = NewsEngine._calendar_cache
        return sorted(
            (e for e in events if e.currency in currencies),
            key=lambda e: e.time,
        )

    # ── Private Methods ────────────────────────────────────────────

    # Maps country names in MyFXBook titles to currency codes
    _COUNTRY_TO_CURRENCY = {
        "united states": "USD", "euro area": "EUR", "united kingdom": "GBP",
        "japan": "JPY", "switzerland": "CHF", "australia": "AUD",
        "canada": "CAD", "new zealand": "NZD",
    }

    @classmethod
    def _record_source(cls, name: str, ok: bool, detail: str = "",
                       extra: str = "") -> None:
        """Track one dependency's fetch outcome for status displays.
        Logs state TRANSITIONS so the log file carries the story without
        a line every 10-min cycle: first failure WARNING (with detail),
        repeat failures DEBUG, recovery INFO."""
        h = cls._source_health.setdefault(
            name, {"ok": None, "err": None, "detail": "", "extra": ""})
        was_failing = h["err"] is not None and (
            h["ok"] is None or h["err"] > h["ok"])
        if ok:
            h["ok"] = time.time()
            h["extra"] = extra
            if was_failing:
                logger.info("Source recovered: %s", name)
        else:
            h["err"] = time.time()
            h["detail"] = detail
            if was_failing:
                logger.debug("Source still failing: %s (%s)", name, detail)
            else:
                logger.warning("Source failing: %s (%s)", name, detail)

    @staticmethod
    def source_is_failing(entry: Optional[dict]) -> bool:
        """True when the entry's last error is newer than its last ok."""
        return bool(entry and entry["err"] is not None and
                    (entry["ok"] is None or entry["err"] > entry["ok"]))

    def _cached_fetch(self, cache_attr: str, fetch_fn) -> list:
        """Return cached data for `cache_attr` or refetch when the TTL
        expired. Successful results live for context_cache_ttl_seconds;
        empty results only _EMPTY_RESULT_TTL so outages retry quickly.
        The lock is held across the fetch so concurrent scans share one
        request instead of stampeding the source."""
        ts, data = getattr(NewsEngine, cache_attr)
        ttl = self.config.context_cache_ttl_seconds if data else self._EMPTY_RESULT_TTL
        if time.time() - ts < ttl:
            return data

        with NewsEngine._cache_lock:
            ts, data = getattr(NewsEngine, cache_attr)  # re-check under lock
            ttl = self.config.context_cache_ttl_seconds if data else self._EMPTY_RESULT_TTL
            if time.time() - ts < ttl:
                return data
            data = fetch_fn()
            setattr(NewsEngine, cache_attr, (time.time(), data))
            return data

    def _fetch_economic_calendar(self, currencies: List[str]) -> List[EconomicEvent]:
        """Cached calendar fetch, filtered to the tracked currencies."""
        events = self._cached_fetch("_calendar_cache", self._fetch_calendar_raw)
        return [e for e in events if e.currency in currencies]

    def _fetch_calendar_raw(self) -> List[EconomicEvent]:
        """
        Fetch economic calendar events from a free API.
        Tries Forex Factory first, falls back to MyFXBook RSS.
        Returns an empty list (with a log warning) if unavailable —
        we never generate fake events since that causes constant false alarms.
        """
        sources = (
            (self._try_fetch_calendar_api, "calendar:forexfactory",
             "ForexFactory"),
            (self._try_fetch_myfxbook_rss, "calendar:myfxbook", "MyFXBook"),
        )
        for fetcher, src, label in sources:
            try:
                events = fetcher()
                if events:
                    self._record_source(src, True)
                    self._record_source(
                        "calendar", True,
                        extra=f"{len(events)} events via {label}")
                    return events
                self._record_source(src, False, "no events returned")
            except Exception as e:
                self._record_source(
                    src, False, f"{type(e).__name__}: {e}")
                logger.debug(f"Calendar fetch exception ({fetcher.__name__}): {e}")

        self._record_source("calendar", False, "all sources failed")
        logger.info(
            "Economic calendar data unavailable. "
            "Event risk checks will be skipped for this cycle."
        )
        return []

    def _try_fetch_calendar_api(self) -> List[EconomicEvent]:
        """Attempt to fetch from the Forex Factory free JSON calendar."""
        now = time.time()
        if now < NewsEngine._calendar_backoff_until:
            wait = NewsEngine._calendar_backoff_until - now
            logger.debug(f"Calendar API in backoff for {wait:.0f}s more")
            return []

        url = "https://nfs.faireconomy.media/ff_calendar_thisweek.json"

        for attempt in range(3):
            try:
                resp = self._session.get(url, timeout=10)
            except requests.RequestException as e:
                logger.debug(f"Calendar request failed (attempt {attempt + 1}): {e}")
                if attempt < 2:
                    time.sleep(2 ** attempt)
                continue

            if resp.status_code == 429:
                retry_after = int(resp.headers.get("Retry-After", 300))
                retry_after = min(retry_after, 600)
                NewsEngine._calendar_backoff_until = time.time() + retry_after
                logger.warning(
                    f"Calendar API rate-limited (429), backing off {retry_after}s"
                )
                return []

            if resp.status_code != 200:
                logger.debug(f"Calendar API returned {resp.status_code}")
                return []

            data = resp.json()
            events = []
            impact_map = {"High": "high", "Medium": "medium", "Low": "low"}

            for item in data:
                try:
                    impact = impact_map.get(item.get("impact", ""), "low")

                    date_str = item.get("date", "")
                    if not date_str:
                        continue
                    event_time = datetime.fromisoformat(date_str.replace("Z", "+00:00"))

                    events.append(EconomicEvent(
                        time=event_time,
                        currency=item.get("country", ""),
                        impact=impact,
                        event=item.get("title", ""),
                        forecast=item.get("forecast") or None,
                        previous=item.get("previous") or None,
                        actual=item.get("actual") or None,
                    ))
                except Exception:
                    continue

            return events

        return []

    def _try_fetch_myfxbook_rss(self) -> List[EconomicEvent]:
        """Fallback calendar source: MyFXBook economic calendar RSS."""
        url = "https://www.myfxbook.com/rss/forex-economic-calendar-events"
        try:
            resp = self._session.get(url, timeout=10)
        except requests.RequestException as e:
            logger.debug(f"MyFXBook calendar request failed: {e}")
            return []

        if resp.status_code != 200:
            return []

        try:
            root = ET.fromstring(resp.content)
        except ET.ParseError:
            return []

        events = []
        for item in root.iter("item"):
            try:
                title_el = item.find("title")
                pub_el = item.find("pubDate")
                desc_el = item.find("description")
                if title_el is None or title_el.text is None:
                    continue

                title = title_el.text.strip()

                currency = ""
                title_lower = title.lower()
                for country, cur in self._COUNTRY_TO_CURRENCY.items():
                    if title_lower.startswith(country):
                        currency = cur
                        title = title[len(country):].strip()
                        break
                if not currency:
                    continue

                event_time = self._parse_rss_date(
                    pub_el.text if pub_el is not None else None
                )
                if event_time is None:
                    continue

                impact = "low"
                desc_text = desc_el.text if desc_el is not None and desc_el.text else ""
                impact_match = re.search(r"sprite-(high|medium|low)-impact", desc_text)
                if impact_match:
                    impact = impact_match.group(1)

                cells = re.findall(r"<td[^>]*>(.*?)</td>", desc_text, re.DOTALL)
                cells = [re.sub(r"<[^>]+>", "", c).strip() for c in cells]
                previous = cells[2] if len(cells) > 2 and cells[2] else None
                forecast = cells[3] if len(cells) > 3 and cells[3] else None
                actual = cells[4] if len(cells) > 4 and cells[4] else None

                events.append(EconomicEvent(
                    time=event_time,
                    currency=currency,
                    impact=impact,
                    event=title,
                    forecast=forecast,
                    previous=previous,
                    actual=actual,
                ))
            except Exception:
                continue

        logger.debug(f"MyFXBook calendar: parsed {len(events)} events")
        return events

    def _fetch_news_sentiment(self) -> List[NewsSentiment]:
        """Cached headline fetch + sentiment analysis."""
        return self._cached_fetch("_headlines_cache", self._fetch_news_sentiment_raw)

    def _fetch_news_sentiment_raw(self) -> List[NewsSentiment]:
        """Fetch and analyze news headlines for sentiment."""
        headlines = []      # names a tracked currency: what the platform uses
        unmatched = []      # names none: scored and archived, never shown
        seen_titles = set()
        ok_feeds = 0
        undated = 0

        for feed_url in self.config.rss_feeds:
            host = feed_url.split("/")[2] if "//" in feed_url else feed_url
            kept_before = len(headlines)
            try:
                resp = self._session.get(feed_url, timeout=10)
                if resp.status_code != 200:
                    self._record_source(
                        f"rss:{host}", False, f"HTTP {resp.status_code}")
                    continue

                root = ET.fromstring(resp.content)

                for item in root.iter("item"):
                    title_el = item.find("title")
                    pub_date_el = item.find("pubDate")

                    if title_el is None:
                        continue

                    title = title_el.text or ""
                    # A blank title carries nothing to score, show or
                    # archive, and one such item made the local model
                    # reject the whole batch (2026-10-01).
                    if not title.strip():
                        continue
                    pub_date = self._parse_rss_date(
                        pub_date_el.text if pub_date_el is not None else None
                    )

                    # Only recent headlines (configurable: headline_max_age_seconds).
                    # An item without a parsable pubDate cannot be proven
                    # fresh, so it is dropped too — it used to be stamped
                    # "now" and never aged out (2026-09-04).
                    if pub_date is None:
                        undated += 1
                        continue
                    if (datetime.now(timezone.utc) - pub_date).total_seconds() > self.config.headline_max_age_seconds:
                        continue

                    # Dedup within and across feeds so a syndicated story
                    # doesn't count twice in the sentiment average. Google
                    # News appends " - Publisher", so compare without it.
                    title_key = title.lower().rsplit(" - ", 1)[0].strip()
                    if title_key in seen_titles:
                        continue
                    seen_titles.add(title_key)

                    sentiment = self._analyze_headline_sentiment(title)
                    currencies = self._extract_currencies(title)

                    link_el = item.find("link")
                    desc_el = item.find("description")
                    src_el = item.find("source")
                    # itertext() survives CDATA and stray child
                    # elements where .text stops at the first child.
                    desc = _clean_html_text(
                        "".join(desc_el.itertext())
                        if desc_el is not None else None)
                    # Google News descriptions are often just the
                    # headline again inside an anchor — worthless.
                    if desc and desc.strip() == title.strip():
                        desc = None
                    publisher = None
                    if src_el is not None:
                        publisher = (src_el.text or "").strip() or None
                    entry = NewsSentiment(
                        headline=title,
                        source=feed_url,
                        timestamp=pub_date or datetime.now(timezone.utc),
                        currencies_mentioned=currencies,
                        sentiment_score=sentiment,
                        keywords=self._extract_keywords(title),
                        sentiment_by_currency={
                            cur: self._analyze_headline_sentiment(title, cur)
                            for cur in currencies
                        },
                        link=_safe_link(
                            link_el.text if link_el is not None else None),
                        description=desc,
                        publisher=publisher,
                    )
                    # A headline naming no tracked currency stays invisible
                    # to the platform (no per-currency mean, no display,
                    # no panel) but is scored and archived since
                    # 2026-09-17: it was 23% of a day's feed, and DeepSeek
                    # gave 10 of those 24 a gold direction — the Iran /
                    # Hormuz stories were the strongest gold headlines of
                    # the day. Training data only.
                    if currencies:
                        headlines.append(entry)
                    else:
                        unmatched.append(entry)

                ok_feeds += 1
                self._record_source(
                    f"rss:{host}", True,
                    extra=f"{len(headlines) - kept_before} kept")
            except Exception as e:
                self._record_source(
                    f"rss:{host}", False, f"{type(e).__name__}: {e}")
                logger.debug(f"Failed to fetch RSS from {feed_url}: {e}")

        if undated:
            logger.debug(f"RSS: dropped {undated} headlines without a publish time")
        if unmatched:
            logger.debug(f"RSS: {len(unmatched)} headlines name no tracked "
                         "currency (scored and archived, not shown)")
        self._record_source(
            "headlines", ok_feeds > 0,
            detail="all feeds failed" if ok_feeds == 0 else "",
            extra=(f"{len(headlines)} headlines from "
                   f"{ok_feeds}/{len(self.config.rss_feeds)} feeds"))

        # Score and archive EVERYTHING fetched; hand back only the
        # currency-matched headlines so display, per-currency means, the
        # panel and the summary see exactly what they saw before.
        scored = headlines + unmatched
        mode = app_config.news_model_mode()
        audit_rows: List[NewsSentiment] = []
        try:
            audit_rows = self._enrich_headline_scores(scored, mode)
        except Exception as e:
            logger.debug(f"headline sentiment enrichment failed, keyword "
                         f"scores stand: {e}")

        # Archive BEFORE any downstream filtering. get_fundamental_context
        # drops every headline inside ±NEUTRAL_HEADLINE_THRESHOLD, so a
        # corpus built from what the context carries holds almost no
        # genuinely neutral headlines and would teach a trained scorer
        # that "neutral" barely exists. Never raises.
        # Guarded at the call site as well as inside archive_headlines:
        # arguments are evaluated BEFORE that function's own try/except, so
        # a raising get_llm_client() would escape it. Archiving is a side
        # effect — it must never cost us the headlines we just fetched.
        #
        # In "primary" mode the local model is the scorer and its output
        # is not training data, so the cycle's rows are NOT archived
        # (owner decision 2026-09-27: stop writing newsdata.db once the
        # model scores). Only the DeepSeek audit copies, the one source of
        # fresh labels, are stored; with deepseek_audit_per_day 0 the
        # archive is frozen. "off" and "fallback" archive as before.
        try:
            teacher = getattr(get_llm_client(), "_model", None)
            if mode == "primary":
                if audit_rows:
                    news_archive.archive_headlines(
                        audit_rows, teacher_model=teacher,
                        teacher_prompt_sha1=_LLM_SENTIMENT_PROMPT_SHA1)
                logger.debug(
                    f"news archive: primary mode, {len(audit_rows)} DeepSeek "
                    f"audit rows stored, {len(scored)} model/keyword rows "
                    f"not archived")
            else:
                news_archive.archive_headlines(
                    scored, teacher_model=teacher,
                    teacher_prompt_sha1=_LLM_SENTIMENT_PROMPT_SHA1,
                    model_teacher=news_model.teacher_tag())
        except Exception as e:
            logger.debug(f"news archive skipped, headlines unaffected: {e}")

        return headlines

    def _enrich_headline_scores(
        self, scored: List[NewsSentiment], mode: str
    ) -> List[NewsSentiment]:
        """Upgrade keyword scores on `scored` per the news_model mode.
        Returns the DeepSeek audit copies to archive (primary mode only,
        usually empty). Runs under _cache_lock like the scorers it calls.

        | mode     | DeepSeek for headlines | local model                |
        |----------|------------------------|----------------------------|
        | off      | every new headline     | never                      |
        | fallback | first, as before       | rows DeepSeek left keyword |
        | primary  | audit sample only      | every row                  |

        primary with the model unavailable in this process (missing
        files, onnxruntime not installed) degrades to the DeepSeek path
        for the cycle, never straight to keywords; a model failure mid-
        batch leaves keyword scores standing (see _apply_model_sentiment).
        """
        if mode == "primary":
            n = self._apply_model_sentiment(scored)
            if n == 0 and news_model.get_news_model() is None:
                self._apply_llm_sentiment(scored)
                return []
            return self._audit_with_deepseek(scored)
        if mode == "fallback":
            self._apply_llm_sentiment(scored)
            self._apply_model_sentiment(scored)
            return []
        self._apply_llm_sentiment(scored)
        return []

    def _analyze_headline_sentiment(
        self, headline: str, for_currency: Optional[str] = None
    ) -> float:
        """
        Simple keyword-based sentiment analysis.

        Returns -1.0 (very bearish) to +1.0 (very bullish).

        When for_currency is "XAU", monetary-policy keywords contribute
        with inverted sign (see GOLD_INVERSE_KEYWORDS).

        Note: This is intentionally simple. NLP models like FinBERT
        would be more accurate but require large dependencies. This catches
        the most impactful sentiment signals from headlines.
        """
        headline_lower = headline.lower()
        invert_policy = for_currency == "XAU"

        bullish_score = 0
        bearish_score = 0
        for kw in self.BULLISH_KEYWORDS:
            if kw in headline_lower:
                if invert_policy and kw in self.GOLD_INVERSE_KEYWORDS:
                    bearish_score += 1
                else:
                    bullish_score += 1
        for kw in self.BEARISH_KEYWORDS:
            if kw in headline_lower:
                if invert_policy and kw in self.GOLD_INVERSE_KEYWORDS:
                    bullish_score += 1
                else:
                    bearish_score += 1

        total = bullish_score + bearish_score
        if total == 0:
            return 0.0

        return (bullish_score - bearish_score) / total

    # ── DeepSeek enrichment (display/warning-only, keyword fallback) ──

    def _llm_headline_scores(
        self, texts: List[str]
    ) -> Optional[List[Tuple[float, float]]]:
        """One batched DeepSeek call → [(overall, xau), ...] aligned with
        `texts`, or None on any failure/mismatch (caller keeps keyword
        scores). Tests stub this method directly."""
        result = get_llm_client().chat_json(
            _LLM_SENTIMENT_SYSTEM, {"headlines": texts})
        if not result:
            return None
        scores = result.get("scores")
        if not isinstance(scores, list) or len(scores) != len(texts):
            return None
        out = []
        for item in scores:
            try:
                s = max(-1.0, min(1.0, float(item[0])))
                x = max(-1.0, min(1.0, float(item[1])))
            except (TypeError, KeyError, ValueError, IndexError):
                return None
            out.append((s, x))
        return out

    def _apply_llm_sentiment(self, headlines: List[NewsSentiment]) -> None:
        """Overwrite keyword sentiment on `headlines` with LLM scores.

        Runs inside _cached_fetch, i.e. UNDER _cache_lock — cache access
        is direct and the (rare) HTTP calls block sibling scans at most
        once per cache window, matching the existing shared-fetch design.
        Only headlines missing from the cache are sent, in batches of
        _LLM_BATCH_CAP; a failed batch changes nothing and ends the
        cycle's sending.
        """
        if not headlines or not get_llm_client().available:
            return
        cache = NewsEngine._llm_sent_cache
        keyed = [(hashlib.sha1(h.headline.encode()).hexdigest(), h)
                 for h in headlines]
        # Every new headline goes out this cycle, one call per batch.
        # Until 2026-09-17 only the first 25 went and the overflow stayed
        # keyword-scored (31 of the first 1,577 archived headlines never
        # got a DeepSeek label that way). A failed call ends the loop: a
        # timeout or the daily cap would fail the next one too, and what
        # is left stays keyword-scored and, still uncached, goes out with
        # the next window.
        new = [(k, h) for k, h in keyed if k not in cache]
        for start in range(0, len(new), self._LLM_BATCH_CAP):
            batch = new[start:start + self._LLM_BATCH_CAP]
            scores = self._llm_headline_scores([h.headline for _, h in batch])
            if scores is None:
                break
            for (k, _), sc in zip(batch, scores):
                cache[k] = sc
        while len(cache) > self._LLM_SENT_CACHE_MAX:
            cache.pop(next(iter(cache)))
        for k, h in keyed:
            sc = cache.get(k)
            if sc is None:
                continue
            overall, xau = sc
            h.sentiment_score = overall
            h.sentiment_by_currency = {
                cur: (xau if cur == "XAU" else overall)
                for cur in h.currencies_mentioned
            }
            h.xau_score = xau
            h.score_source = "llm"

    # ── Local XAU transformer (news_model.py, since 2026-09-27) ──────

    def _apply_model_sentiment(self, headlines: List[NewsSentiment]) -> int:
        """Score with the local XAU transformer every headline that is
        still keyword-scored. Returns the number scored. Never raises.

        The model has a gold head only. It writes xau_score always, XAU's
        per-currency score when XAU is mentioned, and the overall score
        only when the headline names no other currency. USD/EUR/...
        entries keep their keyword score: get_pair_sentiment subtracts
        USD from XAU and must not see the same number on both sides.

        Rules (spec 2026-09-27):
        A. never write the model score into a non-XAU currency;
        B. sentiment_score from the model only for [] or ["XAU"] — the
           archive's `score` column must be honest about what the model
           did and did not score;
        C. xau_score is the raw signed score (training data, unfiltered);
        D. displayed values are 0.0 when the model says not relevant — a
           confident direction head can still give |score| ~0.2, which
           NEUTRAL_HEADLINE_THRESHOLD (0.1) would show as "leaning";
        E. NewsEngine._llm_sent_cache is never touched — it decides what
           is re-sent to DeepSeek (fallback mode retries, audit sample);
        F. only score_source == "keyword" rows change; an llm row stands.
        """
        if not headlines:
            return 0
        bundle = news_model.get_news_model()
        if bundle is None:
            self._record_source("news_model", False,
                                "not loaded in this process")
            return 0
        # score_batch cleans every text first and raises on one that is
        # empty afterwards ("&nbsp;" is), which used to cost the whole
        # batch its scores; such a row keeps its keyword score instead.
        todo = [h for h in headlines if h.score_source == "keyword"
                and self._model_can_score(h.headline)]
        if not todo:
            # Nothing for the model this cycle (fallback mode with DeepSeek
            # healthy, or no readable headline). A failure an earlier batch
            # recorded says nothing about now and must not sit on /status
            # as if it were current; the next batch records its own outcome
            # and warns again if it fails.
            if self.source_is_failing(
                    NewsEngine._source_health.get("news_model")):
                NewsEngine._source_health.pop("news_model", None)
            return 0
        try:
            results = news_model.score_batch(bundle, [h.headline for h in todo])
            if len(results) != len(todo):
                raise ValueError(
                    f"{len(results)} results for {len(todo)} headlines")
        except Exception as e:
            # Keyword scores stand for the cycle. Loud once (WARNING on the
            # first failure, INFO on recovery) and shown by /status: in
            # primary mode nothing else scores these headlines.
            self._record_source("news_model", False,
                                f"{type(e).__name__}: {e}")
            return 0
        self._record_source("news_model", True, extra=f"{len(todo)} scored")
        for h, r in zip(todo, results):
            raw = float(r["score"])                       # archive value
            shown = raw if r["relevant"] else 0.0         # display value
            h.xau_score = raw
            if "XAU" in h.currencies_mentioned:
                h.sentiment_by_currency["XAU"] = shown
            if set(h.currencies_mentioned) <= {"XAU"}:    # [] or ["XAU"]
                h.sentiment_score = shown
            h.score_source = "model"
        return len(todo)

    @staticmethod
    def _model_can_score(text: str) -> bool:
        """True when the local model's own cleaner accepts the text."""
        try:
            news_model.clean_headline(text)
            return True
        except Exception:
            return False

    def _audit_with_deepseek(
        self, headlines: List[NewsSentiment]
    ) -> List[NewsSentiment]:
        """primary mode: send up to news_model.deepseek_audit_per_day
        model-scored headlines per UTC day to DeepSeek and return `llm`
        COPIES for the archive. The displayed objects keep the model's
        numbers. One batch per cycle at most; the pairs go into
        _llm_sent_cache so a headline is never audited twice. This is the
        only place DeepSeek is called for headlines in primary mode; with
        the cap at 0 (the shipped value) it returns [] without a call.
        The copies exist only to be archived, so a process that does not
        write the archive (the dashboard daemon, NEWS_ARCHIVE_ENABLED=0)
        makes no call either."""
        cap = app_config.news_model_audit_per_day()
        if cap <= 0 or not headlines or not get_llm_client().available:
            return []
        if not app_config.news_archive_enabled():
            return []
        today = datetime.now(timezone.utc).date().isoformat()
        if NewsEngine._audit_day != today:
            NewsEngine._audit_day, NewsEngine._audit_sent_today = today, 0
        room = cap - NewsEngine._audit_sent_today
        if room <= 0:
            return []
        cache = NewsEngine._llm_sent_cache
        cand = [h for h in headlines if h.score_source == "model"
                and hashlib.sha1(h.headline.encode()).hexdigest() not in cache]
        batch = cand[:min(room, self._LLM_BATCH_CAP)]
        if not batch:
            return []
        scores = self._llm_headline_scores([h.headline for h in batch])
        if scores is None:
            return []
        NewsEngine._audit_sent_today += len(batch)
        copies = []
        for h, (overall, xau) in zip(batch, scores):
            cache[hashlib.sha1(h.headline.encode()).hexdigest()] = (overall, xau)
            c = copy.copy(h)
            c.sentiment_score = overall
            c.sentiment_by_currency = {
                cur: (xau if cur == "XAU" else overall)
                for cur in h.currencies_mentioned
            }
            c.xau_score = xau
            c.score_source = "llm"
            copies.append(c)
        while len(cache) > self._LLM_SENT_CACHE_MAX:
            cache.pop(next(iter(cache)))
        return copies

    def _llm_event_direction(self, event: EconomicEvent) -> Optional[str]:
        """Gold direction for a released surprise, e.g. "bearish: strong
        jobs lift yields". None → caller keeps the numeric description.
        Cached per release; failures are NOT cached (retry next scan).

        The actual-vs-forecast comparison is precomputed and sent as
        "vs_forecast": the 2026-08-06 validation found the model reads
        close K/M-suffixed pairs backwards ~11% of the time (e.g. claims
        202K vs 217K called "higher than forecast") when left to compare
        the raw numbers itself."""
        # Forecast and release time are part of the key: the direction is
        # a reading of actual AGAINST forecast, and next month's release
        # can print the same actual against a different forecast. The time
        # is keyed in UTC so one instant is one key whatever offset the
        # feed printed it with.
        try:
            when = event.time.astimezone(timezone.utc).isoformat()
        except Exception:
            when = str(event.time)
        key = (event.event, event.currency, str(event.actual or ""),
               str(event.forecast or ""), when)
        with NewsEngine._cache_lock:
            if key in NewsEngine._llm_event_cache:
                return NewsEngine._llm_event_cache[key] or None
        client = get_llm_client()
        if not client.available:
            return None
        vs_forecast = {
            "positive": "above forecast",
            "negative": "below forecast",
            "inline": "in line with forecast",
        }.get(event.surprise or "", "unknown")
        result = client.chat_json(_LLM_EVENT_SYSTEM, {
            "event": event.event,
            "currency": event.currency,
            "actual": event.actual,
            "forecast": event.forecast,
            "previous": event.previous,
            "vs_forecast": vs_forecast,
        })
        if not result:
            return None
        direction = result.get("gold_direction")
        if direction not in ("bullish", "bearish", "neutral"):
            return None
        reason = str(result.get("reason", "")).strip()[:80]
        text = "" if direction == "neutral" else (
            f"{direction}: {reason}" if reason else direction)
        with NewsEngine._cache_lock:
            NewsEngine._llm_event_cache[key] = text
            while len(NewsEngine._llm_event_cache) > self._LLM_EVENT_CACHE_MAX:
                NewsEngine._llm_event_cache.pop(
                    next(iter(NewsEngine._llm_event_cache)))
        return text or None

    def _llm_market_summary(self, context: FundamentalContext) -> str:
        """2-3 sentence plain-language news note for alerts; "" when
        unavailable. Headlines go newest-first with the platform's own
        gold direction so the note agrees with the score badges. Cached
        (success OR failure) for the context-cache TTL so at most one
        call is made per window."""
        with NewsEngine._cache_lock:
            ts, cached = NewsEngine._llm_summary_cache
            if time.time() - ts < self.config.context_cache_ttl_seconds:
                return cached
        high_impact = [e for e in context.events_today if e.is_high_impact]
        if not context.news_headlines and not high_impact:
            return ""
        client = get_llm_client()
        if not client.available:
            return ""
        oldest = datetime.min.replace(tzinfo=timezone.utc)
        recent = sorted(
            context.news_headlines,
            key=lambda h: getattr(h, "timestamp", None) or oldest,
            reverse=True,
        )[:15]
        result = client.chat_json(_LLM_SUMMARY_SYSTEM, {
            "headlines": [
                {"headline": h.headline, "gold": _gold_direction_label(h)}
                for h in recent
            ],
            "events": [
                {"event": e.event, "currency": e.currency,
                 "actual": e.actual, "forecast": e.forecast,
                 "released": not e.is_upcoming}
                for e in high_impact[:10]
            ],
        })
        summary = ""
        if result:
            summary = str(result.get("summary", "")).strip()[:400]
        with NewsEngine._cache_lock:
            NewsEngine._llm_summary_cache = (time.time(), summary)
        return summary

    def _extract_currencies(self, headline: str) -> List[str]:
        """Extract mentioned currencies from a headline."""
        headline_lower = headline.lower()
        return [
            currency
            for currency, pattern in self._CURRENCY_PATTERNS.items()
            if pattern.search(headline_lower)
        ]

    def _extract_keywords(self, headline: str) -> List[str]:
        """Extract relevant trading keywords from headline."""
        headline_lower = headline.lower()
        return [kw for kw in self.BULLISH_KEYWORDS | self.BEARISH_KEYWORDS if kw in headline_lower]

    @staticmethod
    def _parse_rss_date(date_str: Optional[str]) -> Optional[datetime]:
        """Parse various RSS date formats into a timezone-aware datetime."""
        if not date_str:
            return None

        # RFC 822/2822 forms, with or without seconds, numeric offsets AND
        # named zones (GMT, EST/EDT, PST…). The old strptime "%Z" path
        # accepted ANY abbreviation but attached no offset, so e.g. an EST
        # pubDate was silently stamped UTC — a 5-hour error.
        try:
            dt = parsedate_to_datetime(date_str.strip())
            if dt is not None:
                if dt.tzinfo is None:   # unknown zone name — assume UTC
                    return dt.replace(tzinfo=timezone.utc)
                return dt.astimezone(timezone.utc)
        except (TypeError, ValueError):
            pass

        formats = [
            "%Y-%m-%dT%H:%M:%S%z",         # ISO 8601 with offset
            "%Y-%m-%d %H:%M:%S",           # ISO 8601 naive (assume UTC)
        ]
        for fmt in formats:
            try:
                dt = datetime.strptime(date_str.strip(), fmt)
                # If strptime already attached tzinfo (from %z), keep it.
                # Otherwise the string had no timezone info — assume UTC.
                if dt.tzinfo is not None:
                    return dt
                return dt.replace(tzinfo=timezone.utc)
            except ValueError:
                continue
        return None