Initial import: XAUUSD news sentiment models, evals, and data tooling

TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-27 19:22:30 -04:00
commit 1d72e4006a
98 changed files with 1719757 additions and 0 deletions
@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""
+How many feed headlines never reach the archive, and do they matter?
+
+The live fetch (news_engine._fetch_news_sentiment_raw) keeps a headline
+only when it matches a tracked-currency keyword (dollar, Fed, gold,
+euro, ...). Everything else is dropped BEFORE scoring and archiving.
+This probe re-runs the real fetch over the last N hours with that
+filter observed instead of applied, sends the dropped headlines to
+DeepSeek (cached, batches of 25), and reports how many of them carry a
+gold direction. LLM step and archive hook are switched off during the
+fetch, exactly as in eval_live.py.
+
+Usage:
+    venv/bin/python probe_dropped_headlines.py --max-age-hours 24
+"""
+
+from __future__ import annotations
+
+import argparse
+from collections import Counter
+
+from eval_common import (LABEL_THRESHOLD_TRAIN, DeepSeekScorer, fmt_pct, hr,
+                         to_class)
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--max-age-hours", type=float, default=24.0)
+    ap.add_argument("--no-network", action="store_true")
+    ap.add_argument("--timeout", type=float, default=30.0)
+    args = ap.parse_args()
+
+    import news_engine
+    from config import NewsConfig
+    cfg = NewsConfig(headline_max_age_seconds=int(args.max_age_hours * 3600))
+    engine = news_engine.NewsEngine(cfg)
+    real_extract = engine._extract_currencies
+    engine._extract_currencies = lambda h: real_extract(h) or ["_NONE"]
+    engine._apply_llm_sentiment = lambda headlines: None
+    saved = news_engine.news_archive.archive_headlines
+    news_engine.news_archive.archive_headlines = lambda *a, **k: None
+    try:
+        headlines = engine._fetch_news_sentiment_raw()
+    finally:
+        news_engine.news_archive.archive_headlines = saved
+
+    kept = [h for h in headlines if h.currencies_mentioned != ["_NONE"]]
+    dropped = [h for h in headlines if h.currencies_mentioned == ["_NONE"]]
+    hr(f"Feed headlines in the last {args.max_age_hours:g} h")
+    print(f"  total after dedup/age filter: {len(headlines)}")
+    print(f"  kept by the currency filter (scored + archived live): {len(kept)}")
+    print(f"  dropped (never scored, never archived): {len(dropped)} "
+          f"= {100 * len(dropped) / len(headlines):.0f}%")
+    by_feed = Counter(h.source.split('/')[2] for h in dropped)
+    print("  dropped by feed: " + ", ".join(f"{k} {v}" for k, v in by_feed.most_common()))
+
+    if not dropped:
+        return 0
+    ds = DeepSeekScorer(timeout=args.timeout, no_network=args.no_network)
+    res = ds.score_many([h.headline for h in dropped])
+    scored = [(h, d) for h, d in zip(dropped, res) if d is not None]
+    thr = LABEL_THRESHOLD_TRAIN
+    dist = Counter(to_class(d["xau"], thr) for _, d in scored)
+    directional = dist.get(1, 0) + dist.get(-1, 0)
+    strong = sum(1 for _, d in scored if abs(d["xau"]) >= 0.3)
+    hr("DeepSeek's gold reading of the DROPPED headlines")
+    print(f"  scored {len(scored)}: bullish {dist.get(1, 0)}, bearish {dist.get(-1, 0)}, "
+          f"neutral {dist.get(0, 0)}; directional {fmt_pct(directional, len(scored))}, "
+          f"|xau| >= 0.3 on {fmt_pct(strong, len(scored))}")
+    st = ds.stats()
+    print(f"  DeepSeek: {st['calls']} call(s), {st['headlines_sent']} sent, {st['cache_hits']} cached")
+    print("\n  strongest dropped headlines (gold score, headline):")
+    for h, d in sorted(scored, key=lambda t: -abs(t[1]["xau"]))[:15]:
+        print(f"    {d['xau']:+.2f}  {h.headline[:90]}")
+    print("\n  sample of neutral dropped headlines:")
+    for h, d in [t for t in scored if to_class(t[1]["xau"], thr) == 0][:6]:
+        print(f"    {d['xau']:+.2f}  {h.headline[:90]}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())