ForexBot2/news_engine.py

"""
Forex Signal Intelligence Platform - News & Sentiment Engine
=============================================================
Monitors economic calendar events and news sentiment to provide
fundamental context for technical signals.

Key principle: High-impact news events override technical signals.
A perfect technical setup can be destroyed by an unexpected NFP print.
"""

import logging
import re
import time
import xml.etree.ElementTree as ET
from dataclasses import dataclass, field
from datetime import datetime, timedelta, timezone
from typing import Dict, List, Optional, Tuple

import requests

from config import NewsConfig

logger = logging.getLogger(__name__)


@dataclass
class EconomicEvent:
    """Single economic calendar event."""
    time: datetime
    currency: str
    impact: str  # "low", "medium", "high"
    event: str
    forecast: Optional[str] = None
    previous: Optional[str] = None
    actual: Optional[str] = None
    surprise: Optional[str] = None  # "positive", "negative", "inline", None

    @property
    def is_upcoming(self) -> bool:
        return self.time > datetime.now(timezone.utc)

    @property
    def minutes_until(self) -> float:
        delta = self.time - datetime.now(timezone.utc)
        return delta.total_seconds() / 60

    @property
    def is_high_impact(self) -> bool:
        return self.impact == "high"


@dataclass
class NewsSentiment:
    """News sentiment analysis result."""
    headline: str
    source: str
    timestamp: datetime
    currencies_mentioned: List[str]
    sentiment_score: float  # -1.0 (bearish) to +1.0 (bullish)
    keywords: List[str]


@dataclass
class FundamentalContext:
    """Complete fundamental/news context for signal decisions."""
    events_today: List[EconomicEvent] = field(default_factory=list)
    events_upcoming_1h: List[EconomicEvent] = field(default_factory=list)
    high_impact_warning: bool = False
    warning_message: str = ""
    news_sentiment: Dict[str, float] = field(default_factory=dict)  # currency -> sentiment
    news_headlines: List[NewsSentiment] = field(default_factory=list)
    event_impact_on_pairs: Dict[str, str] = field(default_factory=dict)
    calendar_available: bool = False  # True if live calendar data was fetched


class NewsEngine:
    """
    Economic calendar and news sentiment analyzer.

    Provides fundamental context that modifies technical signal confidence.
    """

    # Keywords that indicate directional sentiment
    BULLISH_KEYWORDS = {
        "rate hike", "hawkish", "strong growth", "beat expectations",
        "above forecast", "above estimate", "above expectation",
        "surplus", "expansion", "employment gains",
        "strong jobs", "gdp growth", "inflation rising", "tightening",
        "positive surprise", "record high", "bullish", "rally",
        "stronger than expected", "upbeat",
        "beats", "surges", "soars", "jumps", "climbs", "rises",
        "higher than expected", "better than expected",
        "strong data", "robust", "resilient", "accelerates",
    }

    BEARISH_KEYWORDS = {
        "rate cut", "dovish", "recession", "slowdown", "contraction",
        "below forecast", "below estimate", "below expectation",
        "deficit", "miss expectations", "weak data",
        "unemployment rise", "negative surprise", "bearish", "selloff",
        "weaker than expected", "downbeat", "easing", "stimulus",
        "misses", "slides", "drops", "falls", "plunges", "tumbles",
        "weakens", "slumps", "declines", "retreats",
        "lower than expected", "worse than expected",
        "disappointing", "soft data",
    }

    # Currency-specific keywords for headline parsing
    CURRENCY_KEYWORDS = {
        "USD": ["dollar", "fed", "fomc", "treasury", "us ", "american", "nonfarm", "nfp"],
        "EUR": ["euro", "ecb", "eurozone", "eu ", "german", "french"],
        "GBP": ["pound", "sterling", "boe", "bank of england", "uk ", "british"],
        "JPY": ["yen", "boj", "bank of japan", "japanese"],
        "CHF": ["franc", "snb", "swiss"],
        "AUD": ["aussie", "rba", "australian", "australia"],
        "CAD": ["loonie", "boc", "canadian", "canada"],
        "NZD": ["kiwi", "rbnz", "new zealand"],
        "XAU": ["gold", "xau", "precious metal", "safe haven", "bullion"],
    }

    # High-impact event types (these move markets significantly)
    HIGH_IMPACT_EVENTS = {
        "interest rate decision", "nonfarm payrolls", "non-farm",
        "gdp", "cpi", "inflation", "employment change",
        "retail sales", "pmi", "fomc", "ecb press conference",
        "boe rate", "rba rate", "snb rate", "boc rate",
    }

    _calendar_backoff_until: float = 0.0

    def __init__(self, config: NewsConfig):
        self.config = config
        self._session = requests.Session()
        self._session.headers.update({
            "User-Agent": "Mozilla/5.0 (compatible; ForexIntel/1.0)"
        })

    def get_fundamental_context(
        self, currencies: Optional[List[str]] = None
    ) -> FundamentalContext:
        """
        Get complete fundamental context for signal decisions.

        Combines economic calendar + news sentiment into actionable context.
        """
        if currencies is None:
            currencies = self.config.tracked_currencies

        context = FundamentalContext()

        # 1. Economic calendar events
        try:
            events = self._fetch_economic_calendar(currencies)
            now = datetime.now(timezone.utc)

            context.events_today = [e for e in events if e.time.date() == now.date()]
            context.events_upcoming_1h = [
                e for e in events
                if 0 < e.minutes_until <= 60
            ]
            context.calendar_available = bool(events)

            # Check for high-impact warnings
            for event in events:
                if event.is_high_impact and 0 < event.minutes_until <= self.config.pre_event_buffer_minutes:
                    context.high_impact_warning = True
                    context.warning_message = (
                        f"HIGH IMPACT: {event.currency} {event.event} "
                        f"in {event.minutes_until:.0f} min — signals may be unreliable"
                    )
                    break

            # Map upcoming high-impact events to affected pairs
            for event in context.events_upcoming_1h:
                if event.is_high_impact:
                    affected = self._get_affected_pairs(event.currency)
                    for pair in affected:
                        context.event_impact_on_pairs[pair] = (
                            f"{event.currency} {event.event} in {event.minutes_until:.0f}min"
                        )

        except Exception as e:
            logger.warning(f"Economic calendar fetch failed: {e}")

        # 2. News sentiment
        try:
            headlines = self._fetch_news_sentiment()
            context.news_headlines = headlines

            # Aggregate sentiment per currency
            for currency in currencies:
                scores = [
                    h.sentiment_score for h in headlines
                    if currency in h.currencies_mentioned
                ]
                if scores:
                    context.news_sentiment[currency] = sum(scores) / len(scores)

        except Exception as e:
            logger.warning(f"News sentiment fetch failed: {e}")

        return context

    @staticmethod
    def _parse_pair(pair_symbol: str) -> Tuple[str, str]:
        if "/" in pair_symbol:
            parts = pair_symbol.split("/")
            return parts[0], parts[1]
        clean = pair_symbol.replace("=X", "")
        return clean[:3], clean[3:]

    def get_event_risk_level(self, pair_symbol: str, context: FundamentalContext) -> Tuple[str, str]:
        """
        Get event risk level for a specific pair.
        Returns (risk_level, description).
        risk_level: "clear", "caution", "danger"
        """
        base, quote = self._parse_pair(pair_symbol)

        # Check upcoming high-impact events
        for event in context.events_upcoming_1h:
            if event.is_high_impact and event.currency in (base, quote):
                mins = event.minutes_until
                if mins <= self.config.pre_event_buffer_minutes:
                    return "danger", f"{event.currency} {event.event} in {mins:.0f}min"
                else:
                    return "caution", f"{event.currency} {event.event} in {mins:.0f}min"

        # Check for recent high-impact releases (within post-event buffer)
        for event in context.events_today:
            if event.is_high_impact and event.currency in (base, quote):
                if not event.is_upcoming and event.minutes_until > -self.config.post_event_buffer_minutes:
                    return "caution", f"{event.currency} {event.event} just released"

        return "clear", "No significant events"

    def get_pair_sentiment(
        self, pair_symbol: str, context: FundamentalContext
    ) -> Tuple[float, str]:
        """
        Get net sentiment for a pair based on news.
        Returns (score, description) where score is -1 to +1.
        Positive = bullish for the pair, negative = bearish.
        """
        base, quote = self._parse_pair(pair_symbol)

        base_sent = context.news_sentiment.get(base, 0.0)
        quote_sent = context.news_sentiment.get(quote, 0.0)

        # Net sentiment: positive base or negative quote = bullish for pair
        net = base_sent - quote_sent

        if abs(net) < 0.1:
            desc = "Neutral news sentiment"
        elif net > 0.3:
            desc = f"Bullish sentiment ({base} positive / {quote} negative)"
        elif net > 0:
            desc = "Slightly bullish sentiment"
        elif net < -0.3:
            desc = f"Bearish sentiment ({base} negative / {quote} positive)"
        else:
            desc = "Slightly bearish sentiment"

        return net, desc

    # ── Private Methods ────────────────────────────────────────────

    # Maps country names in MyFXBook titles to currency codes
    _COUNTRY_TO_CURRENCY = {
        "united states": "USD", "euro area": "EUR", "united kingdom": "GBP",
        "japan": "JPY", "switzerland": "CHF", "australia": "AUD",
        "canada": "CAD", "new zealand": "NZD",
    }

    def _fetch_economic_calendar(self, currencies: List[str]) -> List[EconomicEvent]:
        """
        Fetch economic calendar events from a free API.
        Tries Forex Factory first, falls back to MyFXBook RSS.
        Returns an empty list (with a log warning) if unavailable —
        we never generate fake events since that causes constant false alarms.
        """
        for fetcher in (self._try_fetch_calendar_api, self._try_fetch_myfxbook_rss):
            try:
                events = fetcher()
                if events:
                    return [e for e in events if e.currency in currencies]
            except Exception as e:
                logger.debug(f"Calendar fetch exception ({fetcher.__name__}): {e}")

        logger.info(
            "Economic calendar data unavailable. "
            "Event risk checks will be skipped for this cycle."
        )
        return []

    def _try_fetch_calendar_api(self) -> List[EconomicEvent]:
        """Attempt to fetch from the Forex Factory free JSON calendar."""
        now = time.time()
        if now < NewsEngine._calendar_backoff_until:
            wait = NewsEngine._calendar_backoff_until - now
            logger.debug(f"Calendar API in backoff for {wait:.0f}s more")
            return []

        url = "https://nfs.faireconomy.media/ff_calendar_thisweek.json"

        for attempt in range(3):
            try:
                resp = self._session.get(url, timeout=10)
            except requests.RequestException as e:
                logger.debug(f"Calendar request failed (attempt {attempt + 1}): {e}")
                if attempt < 2:
                    time.sleep(2 ** attempt)
                continue

            if resp.status_code == 429:
                retry_after = int(resp.headers.get("Retry-After", 300))
                retry_after = min(retry_after, 600)
                NewsEngine._calendar_backoff_until = time.time() + retry_after
                logger.warning(
                    f"Calendar API rate-limited (429), backing off {retry_after}s"
                )
                return []

            if resp.status_code != 200:
                logger.debug(f"Calendar API returned {resp.status_code}")
                return []

            data = resp.json()
            events = []
            impact_map = {"High": "high", "Medium": "medium", "Low": "low"}

            for item in data:
                try:
                    impact = impact_map.get(item.get("impact", ""), "low")

                    date_str = item.get("date", "")
                    if not date_str:
                        continue
                    event_time = datetime.fromisoformat(date_str.replace("Z", "+00:00"))

                    events.append(EconomicEvent(
                        time=event_time,
                        currency=item.get("country", ""),
                        impact=impact,
                        event=item.get("title", ""),
                        forecast=item.get("forecast") or None,
                        previous=item.get("previous") or None,
                        actual=item.get("actual") or None,
                    ))
                except Exception:
                    continue

            return events

        return []

    def _try_fetch_myfxbook_rss(self) -> List[EconomicEvent]:
        """Fallback calendar source: MyFXBook economic calendar RSS."""
        url = "https://www.myfxbook.com/rss/forex-economic-calendar-events"
        try:
            resp = self._session.get(url, timeout=10)
        except requests.RequestException as e:
            logger.debug(f"MyFXBook calendar request failed: {e}")
            return []

        if resp.status_code != 200:
            return []

        try:
            root = ET.fromstring(resp.content)
        except ET.ParseError:
            return []

        events = []
        for item in root.iter("item"):
            try:
                title_el = item.find("title")
                pub_el = item.find("pubDate")
                desc_el = item.find("description")
                if title_el is None or title_el.text is None:
                    continue

                title = title_el.text.strip()

                currency = ""
                title_lower = title.lower()
                for country, cur in self._COUNTRY_TO_CURRENCY.items():
                    if title_lower.startswith(country):
                        currency = cur
                        title = title[len(country):].strip()
                        break
                if not currency:
                    continue

                event_time = self._parse_rss_date(
                    pub_el.text if pub_el is not None else None
                )
                if event_time is None:
                    continue

                impact = "low"
                desc_text = desc_el.text if desc_el is not None and desc_el.text else ""
                impact_match = re.search(r"sprite-(high|medium|low)-impact", desc_text)
                if impact_match:
                    impact = impact_match.group(1)

                cells = re.findall(r"<td[^>]*>(.*?)</td>", desc_text, re.DOTALL)
                cells = [re.sub(r"<[^>]+>", "", c).strip() for c in cells]
                previous = cells[2] if len(cells) > 2 and cells[2] else None
                forecast = cells[3] if len(cells) > 3 and cells[3] else None
                actual = cells[4] if len(cells) > 4 and cells[4] else None

                events.append(EconomicEvent(
                    time=event_time,
                    currency=currency,
                    impact=impact,
                    event=title,
                    forecast=forecast,
                    previous=previous,
                    actual=actual,
                ))
            except Exception:
                continue

        logger.debug(f"MyFXBook calendar: parsed {len(events)} events")
        return events

    def _fetch_news_sentiment(self) -> List[NewsSentiment]:
        """Fetch and analyze news headlines for sentiment."""
        headlines = []

        for feed_url in self.config.rss_feeds:
            try:
                resp = self._session.get(feed_url, timeout=10)
                if resp.status_code != 200:
                    continue

                root = ET.fromstring(resp.content)

                for item in root.iter("item"):
                    title_el = item.find("title")
                    pub_date_el = item.find("pubDate")

                    if title_el is None:
                        continue

                    title = title_el.text or ""
                    pub_date = self._parse_rss_date(
                        pub_date_el.text if pub_date_el is not None else None
                    )

                    # Only recent headlines (last 24h)
                    if pub_date and (datetime.now(timezone.utc) - pub_date).total_seconds() > 86400:
                        continue

                    sentiment = self._analyze_headline_sentiment(title)
                    currencies = self._extract_currencies(title)

                    if currencies:
                        headlines.append(NewsSentiment(
                            headline=title,
                            source=feed_url,
                            timestamp=pub_date or datetime.now(timezone.utc),
                            currencies_mentioned=currencies,
                            sentiment_score=sentiment,
                            keywords=self._extract_keywords(title),
                        ))

            except Exception as e:
                logger.debug(f"Failed to fetch RSS from {feed_url}: {e}")

        return headlines

    def _analyze_headline_sentiment(self, headline: str) -> float:
        """
        Simple keyword-based sentiment analysis.

        Returns -1.0 (very bearish) to +1.0 (very bullish).

        Note: This is intentionally simple. NLP models like FinBERT
        would be more accurate but require large dependencies. This catches
        the most impactful sentiment signals from headlines.
        """
        headline_lower = headline.lower()
        bullish_score = sum(1 for kw in self.BULLISH_KEYWORDS if kw in headline_lower)
        bearish_score = sum(1 for kw in self.BEARISH_KEYWORDS if kw in headline_lower)

        total = bullish_score + bearish_score
        if total == 0:
            return 0.0

        return (bullish_score - bearish_score) / total

    def _extract_currencies(self, headline: str) -> List[str]:
        """Extract mentioned currencies from a headline."""
        headline_lower = headline.lower()
        found = []

        for currency, keywords in self.CURRENCY_KEYWORDS.items():
            if currency.lower() in headline_lower:
                found.append(currency)
                continue
            for kw in keywords:
                if kw in headline_lower:
                    found.append(currency)
                    break

        return list(set(found))

    def _extract_keywords(self, headline: str) -> List[str]:
        """Extract relevant trading keywords from headline."""
        headline_lower = headline.lower()
        return [kw for kw in self.BULLISH_KEYWORDS | self.BEARISH_KEYWORDS if kw in headline_lower]

    @staticmethod
    def _parse_rss_date(date_str: Optional[str]) -> Optional[datetime]:
        """Parse various RSS date formats into a timezone-aware datetime."""
        if not date_str:
            return None

        formats = [
            "%a, %d %b %Y %H:%M:%S %z",   # RFC 822 with numeric tz offset
            "%a, %d %b %Y %H:%M:%S %Z",   # RFC 822 with tz abbreviation (GMT/UTC)
            "%a, %d %b %Y %H:%M %Z",      # RFC 822 without seconds (MyFXBook)
            "%a, %d %b %Y %H:%M %z",      # RFC 822 without seconds, numeric offset
            "%Y-%m-%dT%H:%M:%S%z",         # ISO 8601 with offset
            "%Y-%m-%d %H:%M:%S",           # ISO 8601 naive (assume UTC)
        ]
        for fmt in formats:
            try:
                dt = datetime.strptime(date_str.strip(), fmt)
                # If strptime already attached tzinfo (from %z), keep it.
                # Otherwise the string had no timezone info — assume UTC.
                if dt.tzinfo is not None:
                    return dt
                return dt.replace(tzinfo=timezone.utc)
            except ValueError:
                continue
        return None

    @staticmethod
    def _get_affected_pairs(currency: str) -> List[str]:
        _MAP = {
            "USD": ["EURUSD=X", "USDJPY=X", "GBPUSD=X", "XAU/USD"],
            "EUR": ["EURUSD=X"],
            "JPY": ["USDJPY=X"],
            "GBP": ["GBPUSD=X"],
            "XAU": ["XAU/USD"],
        }
        return _MAP.get(currency, [])