#!/usr/bin/env python3
"""
How many feed headlines never reach the archive, and do they matter?

The live fetch (news_engine._fetch_news_sentiment_raw) keeps a headline
only when it matches a tracked-currency keyword (dollar, Fed, gold,
euro, ...). Everything else is dropped BEFORE scoring and archiving.
This probe re-runs the real fetch over the last N hours with that
filter observed instead of applied, sends the dropped headlines to
DeepSeek (cached, batches of 25), and reports how many of them carry a
gold direction. LLM step and archive hook are switched off during the
fetch, exactly as in eval_live.py.

Usage:
    venv/bin/python probe_dropped_headlines.py --max-age-hours 24
"""

from __future__ import annotations

import argparse
from collections import Counter

from eval_common import (LABEL_THRESHOLD_TRAIN, DeepSeekScorer, fmt_pct, hr,
                         to_class)


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--max-age-hours", type=float, default=24.0)
    ap.add_argument("--no-network", action="store_true")
    ap.add_argument("--timeout", type=float, default=30.0)
    args = ap.parse_args()

    import news_engine
    from config import NewsConfig
    cfg = NewsConfig(headline_max_age_seconds=int(args.max_age_hours * 3600))
    engine = news_engine.NewsEngine(cfg)
    real_extract = engine._extract_currencies
    engine._extract_currencies = lambda h: real_extract(h) or ["_NONE"]
    engine._apply_llm_sentiment = lambda headlines: None
    saved = news_engine.news_archive.archive_headlines
    news_engine.news_archive.archive_headlines = lambda *a, **k: None
    try:
        headlines = engine._fetch_news_sentiment_raw()
    finally:
        news_engine.news_archive.archive_headlines = saved

    kept = [h for h in headlines if h.currencies_mentioned != ["_NONE"]]
    dropped = [h for h in headlines if h.currencies_mentioned == ["_NONE"]]
    hr(f"Feed headlines in the last {args.max_age_hours:g} h")
    print(f"  total after dedup/age filter: {len(headlines)}")
    print(f"  kept by the currency filter (scored + archived live): {len(kept)}")
    print(f"  dropped (never scored, never archived): {len(dropped)} "
          f"= {100 * len(dropped) / len(headlines):.0f}%")
    by_feed = Counter(h.source.split('/')[2] for h in dropped)
    print("  dropped by feed: " + ", ".join(f"{k} {v}" for k, v in by_feed.most_common()))

    if not dropped:
        return 0
    ds = DeepSeekScorer(timeout=args.timeout, no_network=args.no_network)
    res = ds.score_many([h.headline for h in dropped])
    scored = [(h, d) for h, d in zip(dropped, res) if d is not None]
    thr = LABEL_THRESHOLD_TRAIN
    dist = Counter(to_class(d["xau"], thr) for _, d in scored)
    directional = dist.get(1, 0) + dist.get(-1, 0)
    strong = sum(1 for _, d in scored if abs(d["xau"]) >= 0.3)
    hr("DeepSeek's gold reading of the DROPPED headlines")
    print(f"  scored {len(scored)}: bullish {dist.get(1, 0)}, bearish {dist.get(-1, 0)}, "
          f"neutral {dist.get(0, 0)}; directional {fmt_pct(directional, len(scored))}, "
          f"|xau| >= 0.3 on {fmt_pct(strong, len(scored))}")
    st = ds.stats()
    print(f"  DeepSeek: {st['calls']} call(s), {st['headlines_sent']} sent, {st['cache_hits']} cached")
    print("\n  strongest dropped headlines (gold score, headline):")
    for h, d in sorted(scored, key=lambda t: -abs(t[1]["xau"]))[:15]:
        print(f"    {d['xau']:+.2f}  {h.headline[:90]}")
    print("\n  sample of neutral dropped headlines:")
    for h, d in [t for t in scored if to_class(t[1]["xau"], thr) == 0][:6]:
        print(f"    {d['xau']:+.2f}  {h.headline[:90]}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())