From 0bb52083a30d3620613f4c4b74be5d78c40612da Mon Sep 17 00:00:00 2001 From: Harwinder Date: Tue, 5 May 2026 22:33:43 -0400 Subject: [PATCH] Changed news engine code a little --- CLAUDE.md | 36 ++++++++++++++++++++++++++++++++ ScriptsActivate.ps1 | 0 news_engine.py | 50 ++++++++++++++++++++++++++++++++++++--------- 3 files changed, 76 insertions(+), 10 deletions(-) create mode 100644 ScriptsActivate.ps1 diff --git a/CLAUDE.md b/CLAUDE.md index 1159a8e..076295b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -215,6 +215,42 @@ Telegram: edit `SCAN_TIMEFRAMES`. --- +## Suggested improvements (not yet implemented) + +### Replace keyword sentiment with FinBERT + +The current `news_engine._analyze_headline_sentiment()` uses keyword matching against `BULLISH_KEYWORDS` / `BEARISH_KEYWORDS` sets ([news_engine.py:82-95](news_engine.py#L82-L95)). This misses obvious context — e.g. "RBA signals pause", "Fed patient on rate cuts", "inflation cools" all score `0.00` because the lexicon doesn't include "pause", "patient", or "cools". Around 80% of headlines fetched currently score 0. + +**Recommended fix: FinBERT (HuggingFace pretrained model on financial news).** + +- Drop-in replacement for `_analyze_headline_sentiment()`. +- Pretrained on financial headlines, ~70-75% accuracy on this specific domain. +- One-time ~500MB model download; CPU inference ~50ms/headline (fine for ~30 headlines per fetch). +- No API cost, no external dependency at runtime. + +Implementation sketch (~30 lines): +```python +from transformers import pipeline +_finbert = pipeline("sentiment-analysis", model="ProsusAI/finbert") + +def _analyze_headline_sentiment(self, headline: str) -> float: + result = _finbert(headline[:512])[0] # truncate to model max + label, score = result["label"], result["score"] + if label == "positive": return score + if label == "negative": return -score + return 0.0 +``` + +Add `transformers` and `torch` (or `torch-cpu`) to `requirements.txt`. Cache scored headlines by `hash(headline)` to avoid re-scoring across pairs in the same fetch. Add a try/except fallback to the current keyword logic if model load fails (graceful degradation). + +**Realistic uplift on overall signal quality: 1-3% win-rate.** Sentiment is only 10% of the composite score ([config.py:139](config.py#L139)), so even doubling sentiment accuracy moves the needle modestly. Worth doing for technical correctness, not as a P&L optimization. + +**Higher-tier alternative: Claude/GPT API per headline.** ~80-85% accurate, $0.001/headline (~$0.03 per /scan with caching). Adds external dependency and latency. Only worth it if FinBERT proves insufficient. + +**Cheap alternative without ML: Expand keyword sets.** Add ~50-100 phrases to `BULLISH_KEYWORDS` / `BEARISH_KEYWORDS` ("pause", "patient", "data-dependent", "soft landing", "beat", "revised up" / "miss", "revised down", "warning", "fragile"). Gets ~50% → ~60% accuracy in 30 min, no new deps. + +--- + ## Recent architectural decisions (kept short) - **Removed synthetic data path** — explicit failure beats fake prices. diff --git a/ScriptsActivate.ps1 b/ScriptsActivate.ps1 new file mode 100644 index 0000000..e69de29 diff --git a/news_engine.py b/news_engine.py index 7424c62..0cecd79 100644 --- a/news_engine.py +++ b/news_engine.py @@ -1,6 +1,4 @@ """ -Forex Signal Intelligence Platform - News & Sentiment Engine -============================================================= Monitors economic calendar events and news sentiment to provide fundamental context for technical signals. @@ -16,6 +14,9 @@ from typing import Dict, List, Optional, Tuple import requests +# Import the transformers pipeline for FinBERT +from transformers import pipeline + from config import NewsConfig logger = logging.getLogger(__name__) @@ -78,7 +79,7 @@ class NewsEngine: Provides fundamental context that modifies technical signal confidence. """ - # Keywords that indicate directional sentiment + # Keywords that indicate directional sentiment (kept for fallback and keyword extraction) BULLISH_KEYWORDS = { "rate hike", "hawkish", "strong growth", "beat expectations", "above forecast", "surplus", "expansion", "employment gains", @@ -120,6 +121,16 @@ class NewsEngine: self._session.headers.update({ "User-Agent": "Mozilla/5.0 (compatible; ForexIntel/1.0)" }) + + # Initialize the FinBERT LLM Pipeline + try: + logger.info("Initializing FinBERT NLP model. This may take a moment on startup...") + # We use ProsusAI/finbert as it is specifically fine-tuned on financial tone + self.nlp_pipeline = pipeline("sentiment-analysis", model="ProsusAI/finbert") + logger.info("FinBERT successfully loaded.") + except Exception as e: + logger.error(f"Failed to load FinBERT model: {e}. Defaulting to basic keyword analysis.") + self.nlp_pipeline = None def get_fundamental_context( self, currencies: Optional[List[str]] = None @@ -348,14 +359,33 @@ class NewsEngine: def _analyze_headline_sentiment(self, headline: str) -> float: """ - Simple keyword-based sentiment analysis. - + Analyze headline sentiment using the FinBERT NLP model. Returns -1.0 (very bearish) to +1.0 (very bullish). - - Note: This is intentionally simple. NLP models like FinBERT - would be more accurate but require large dependencies. This catches - the most impactful sentiment signals from headlines. """ + if self.nlp_pipeline is None: + # Fallback to simple keyword logic if the LLM failed to load + return self._fallback_keyword_sentiment(headline) + + try: + # The pipeline returns a list with a dict, e.g.: [{'label': 'positive', 'score': 0.85}] + result = self.nlp_pipeline(headline)[0] + label = result['label'].lower() + confidence = result['score'] + + if label == 'positive': + return confidence # e.g., +0.85 + elif label == 'negative': + return -confidence # e.g., -0.85 + else: + # Neutral label + return 0.0 + + except Exception as e: + logger.debug(f"FinBERT sentiment analysis failed for '{headline}': {e}") + return self._fallback_keyword_sentiment(headline) + + def _fallback_keyword_sentiment(self, headline: str) -> float: + """Original keyword-based fallback if FinBERT is unavailable.""" headline_lower = headline.lower() bullish_score = sum(1 for kw in self.BULLISH_KEYWORDS if kw in headline_lower) bearish_score = sum(1 for kw in self.BEARISH_KEYWORDS if kw in headline_lower) @@ -424,4 +454,4 @@ class NewsEngine: "CAD": ["USDCAD=X", "CADJPY=X"], "NZD": ["NZDUSD=X"], } - return pairs.get(currency, []) + return pairs.get(currency, []) \ No newline at end of file