Harwinder Sekhon
01baef48ff
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked.
84 lines
3.6 KiB
Python
84 lines
3.6 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
How many feed headlines never reach the archive, and do they matter?
|
|
|
|
The live fetch (news_engine._fetch_news_sentiment_raw) keeps a headline
|
|
only when it matches a tracked-currency keyword (dollar, Fed, gold,
|
|
euro, ...). Everything else is dropped BEFORE scoring and archiving.
|
|
This probe re-runs the real fetch over the last N hours with that
|
|
filter observed instead of applied, sends the dropped headlines to
|
|
DeepSeek (cached, batches of 25), and reports how many of them carry a
|
|
gold direction. LLM step and archive hook are switched off during the
|
|
fetch, exactly as in eval_live.py.
|
|
|
|
Usage:
|
|
venv/bin/python probe_dropped_headlines.py --max-age-hours 24
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from collections import Counter
|
|
|
|
from eval_common import (LABEL_THRESHOLD_TRAIN, DeepSeekScorer, fmt_pct, hr,
|
|
to_class)
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--max-age-hours", type=float, default=24.0)
|
|
ap.add_argument("--no-network", action="store_true")
|
|
ap.add_argument("--timeout", type=float, default=30.0)
|
|
args = ap.parse_args()
|
|
|
|
import news_engine
|
|
from config import NewsConfig
|
|
cfg = NewsConfig(headline_max_age_seconds=int(args.max_age_hours * 3600))
|
|
engine = news_engine.NewsEngine(cfg)
|
|
real_extract = engine._extract_currencies
|
|
engine._extract_currencies = lambda h: real_extract(h) or ["_NONE"]
|
|
engine._apply_llm_sentiment = lambda headlines: None
|
|
saved = news_engine.news_archive.archive_headlines
|
|
news_engine.news_archive.archive_headlines = lambda *a, **k: None
|
|
try:
|
|
headlines = engine._fetch_news_sentiment_raw()
|
|
finally:
|
|
news_engine.news_archive.archive_headlines = saved
|
|
|
|
kept = [h for h in headlines if h.currencies_mentioned != ["_NONE"]]
|
|
dropped = [h for h in headlines if h.currencies_mentioned == ["_NONE"]]
|
|
hr(f"Feed headlines in the last {args.max_age_hours:g} h")
|
|
print(f" total after dedup/age filter: {len(headlines)}")
|
|
print(f" kept by the currency filter (scored + archived live): {len(kept)}")
|
|
print(f" dropped (never scored, never archived): {len(dropped)} "
|
|
f"= {100 * len(dropped) / len(headlines):.0f}%")
|
|
by_feed = Counter(h.source.split('/')[2] for h in dropped)
|
|
print(" dropped by feed: " + ", ".join(f"{k} {v}" for k, v in by_feed.most_common()))
|
|
|
|
if not dropped:
|
|
return 0
|
|
ds = DeepSeekScorer(timeout=args.timeout, no_network=args.no_network)
|
|
res = ds.score_many([h.headline for h in dropped])
|
|
scored = [(h, d) for h, d in zip(dropped, res) if d is not None]
|
|
thr = LABEL_THRESHOLD_TRAIN
|
|
dist = Counter(to_class(d["xau"], thr) for _, d in scored)
|
|
directional = dist.get(1, 0) + dist.get(-1, 0)
|
|
strong = sum(1 for _, d in scored if abs(d["xau"]) >= 0.3)
|
|
hr("DeepSeek's gold reading of the DROPPED headlines")
|
|
print(f" scored {len(scored)}: bullish {dist.get(1, 0)}, bearish {dist.get(-1, 0)}, "
|
|
f"neutral {dist.get(0, 0)}; directional {fmt_pct(directional, len(scored))}, "
|
|
f"|xau| >= 0.3 on {fmt_pct(strong, len(scored))}")
|
|
st = ds.stats()
|
|
print(f" DeepSeek: {st['calls']} call(s), {st['headlines_sent']} sent, {st['cache_hits']} cached")
|
|
print("\n strongest dropped headlines (gold score, headline):")
|
|
for h, d in sorted(scored, key=lambda t: -abs(t[1]["xau"]))[:15]:
|
|
print(f" {d['xau']:+.2f} {h.headline[:90]}")
|
|
print("\n sample of neutral dropped headlines:")
|
|
for h, d in [t for t in scored if to_class(t[1]["xau"], thr) == 0][:6]:
|
|
print(f" {d['xau']:+.2f} {h.headline[:90]}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|