#!/usr/bin/env python3 """ Archive benchmark: how well does the scikit recipe copy DeepSeek on headlines it has NOT seen, and how noisy is DeepSeek itself? Data: the headlines table of ForexBot2's logs/newsdata.db (the file ML-NEWS trained on is a copy of it). Only rows DeepSeek scored with an explicit XAU value are usable (616 on 2026-09-17, eight days). The DeepSeek XAU score is the label: bullish > +0.05, bearish < -0.05, neutral otherwise — exactly the ML-NEWS trainer's rule. Parts A In-sample sanity: the SHIPPED bundle against its own training labels. Contaminated by construction; it is the ceiling, shown so the out-of-sample numbers can be read against it. B Expanding daily folds: for each day d, retrain the ML-NEWS pipeline (same hyperparameters) on every row published before d and score day d. Two text recipes: headline+description (shipped) and headline only (DeepSeek itself never sees the description, and the 2026-09-08 rule for a local scorer says headline text only). The keyword fallback is scored on the same rows for context. Pooled out-of-fold (OOF) metrics follow the per-day table. C Confidence routing on the pooled OOF predictions: for a grid of confidence / margin thresholds (the WORKFLOW doc starts at 0.70 / 0.20), what share of headlines the model would keep locally, how often those agree with DeepSeek, and the hybrid agreement if the rest go to DeepSeek. D --reprobe N: re-send N archived headlines to DeepSeek today and compare with the score it gave the first time. DeepSeek's self-agreement is the ceiling any student can reach; on the same rows the OOF model is scored against both answers. Usage: venv/bin/python eval_archive.py # offline venv/bin/python eval_archive.py --reprobe 100 # + 4 DeepSeek calls """ from __future__ import annotations import argparse import random import statistics import time from collections import Counter, defaultdict from pathlib import Path from typing import Any, Dict, List, Optional from eval_common import (CLASS_NAMES, DEFAULT_DB, LABEL_THRESHOLD_TRAIN, RESULTS_DIR, DeepSeekScorer, KeywordScorer, make_scorer, NAME_TO_CLASS, agreement_block, fmt_pct, hr, load_archive_rows, print_agreement, to_class, ts_tag, write_csv, write_json) def usable_rows(db: Path, extra_labels: Optional[Path] = None) -> List[Dict[str, Any]]: rows = [r for r in load_archive_rows(db, extra_labels) if r["score_source"] == "llm" and r["xau"] is not None and r["headline"] and r["day"]] rows.sort(key=lambda r: (r["published_at"] or r["first_seen"] or "", r["hash"])) return rows def fit_and_score(train: List[Dict[str, Any]], test: List[Dict[str, Any]], with_description: bool, threshold: float, seed: int): """Train the ML-NEWS pipeline on `train`, return per-row predictions for `test` (class, score, confidence, margin) or None if the fold cannot be trained (a class with < 2 examples).""" from news_sentiment import article_text, build_pipeline texts = [article_text(r["headline"], r["description"] if with_description else None) for r in train] labels = [CLASS_NAMES[to_class(r["xau"], threshold)] for r in train] dist = Counter(labels) if len(dist) < 3 or min(dist.values()) < 2: return None pipe = build_pipeline(random_state=seed, min_df=2, max_features=50_000) t0 = time.perf_counter() pipe.fit(texts, labels) fit_s = time.perf_counter() - t0 classes = [str(c) for c in pipe.classes_] test_texts = [article_text(r["headline"], r["description"] if with_description else None) for r in test] probs = pipe.predict_proba(test_texts) preds = [] for p in probs: by = {c: float(v) for c, v in zip(classes, p)} for c in ("bearish", "neutral", "bullish"): by.setdefault(c, 0.0) label = max(by, key=by.get) ordered = sorted(by.values(), reverse=True) preds.append({"cls": NAME_TO_CLASS[label], "label": label, "score": by["bullish"] - by["bearish"], "confidence": by[label], "margin": ordered[0] - ordered[1]}) return preds, fit_s def routing_table(oof: List[Dict[str, Any]]) -> List[Dict[str, Any]]: out = [] n = len(oof) for conf_t in (0.0, 0.5, 0.6, 0.7, 0.8, 0.9): for margin_t in (0.0, 0.1, 0.2, 0.3, 0.4): kept = [r for r in oof if r["confidence"] >= conf_t and r["margin"] >= margin_t] routed = n - len(kept) hits = sum(1 for r in kept if r["cls"] == r["label_cls"]) acc_kept = hits / len(kept) if kept else None hybrid = (hits + routed) / n if n else None # routed rows = DeepSeek's own answer out.append({"confidence_min": conf_t, "margin_min": margin_t, "kept": len(kept), "kept_share": len(kept) / n if n else None, "kept_accuracy": acc_kept, "hybrid_agreement": hybrid, "llm_calls_saved_share": len(kept) / n if n else None}) return out def main() -> int: ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) ap.add_argument("--db", type=Path, default=DEFAULT_DB) ap.add_argument("--threshold", type=float, default=LABEL_THRESHOLD_TRAIN, help="DeepSeek xau -> class threshold (trainer uses 0.05)") ap.add_argument("--min-train", type=int, default=40, help="skip folds with fewer training rows") ap.add_argument("--seed", type=int, default=42) ap.add_argument("--reprobe", type=int, default=0, help="re-score N archived headlines with DeepSeek today") ap.add_argument("--timeout", type=float, default=30.0) ap.add_argument("--no-network", action="store_true") ap.add_argument("--scorer", choices=("tfidf", "transformer"), default="tfidf", help="model for Part A; transformer runs Part A only (B-D retrain " "the TF-IDF recipe)") ap.add_argument("--model", type=Path, default=None, help="model for Part A (default: the shipped one of --scorer)") ap.add_argument("--extra-labels", type=Path, help="fallback labels JSONL keyed by hash (results/backfill_labels.jsonl)") ap.add_argument("--train-scope", choices=("all", "gold"), default="all", help="Part B trains on every labelled row, or only on rows with " "a gold XAU key; the test day is always every labelled row") args = ap.parse_args() thr = args.threshold rows = usable_rows(args.db, args.extra_labels) if not rows: raise SystemExit("no llm-scored XAU rows in the archive") days = sorted({r["day"] for r in rows}) per_day = Counter(r["day"] for r in rows) label_dist = Counter(CLASS_NAMES[to_class(r["xau"], thr)] for r in rows) print(f"archive {args.db}: {len(rows)} DeepSeek-scored XAU rows over " f"{len(days)} days ({days[0]} .. {days[-1]}); labels at ±{thr}: " f"{dict(sorted(label_dist.items()))}") print(" rows per day: " + ", ".join(f"{d[5:]}={per_day[d]}" for d in days)) print(f" label sources: {dict(Counter(r['xau_src'] for r in rows))}; " f"gold-key rows {sum(r['gold'] for r in rows)}, macro rows " f"{sum(not r['gold'] for r in rows)}") # ── Part A: shipped bundle, in-sample ────────────────────────────── hr("A. Shipped bundle on its own training rows (IN-SAMPLE, ceiling only)") model = make_scorer(args.scorer, args.model) print(f" {model.meta['model_type']} trained {model.meta['trained_at']} on " f"{model.meta['training_examples']} examples") exp = [to_class(r["xau"], thr) for r in rows] a_full = [model.score(r["headline"], r["description"]) for r in rows] a_head = [model.score(r["headline"], None) for r in rows] blk_a_full = agreement_block(exp, [m["cls"] for m in a_full], [r["xau"] for r in rows], [m["score"] for m in a_full]) blk_a_head = agreement_block(exp, [m["cls"] for m in a_head], [r["xau"] for r in rows], [m["score"] for m in a_head]) print_agreement("shipped, headline+description (as trained)", blk_a_full) print_agreement("shipped, headline only (description dropped)", blk_a_head) print(f" model latency: mean {model.mean_ms:.2f} ms per headline") # Rows published from oos_from on were never trained on (TF-IDF: its # trained_at; non-final transformer: its val_from). oos_from = str(model.meta["oos_from"] or "") after = [i for i, r in enumerate(rows) if str(r["published_at"] or "") >= oos_from] blk_a_after: Dict[str, Any] = {} for subset, name in (("all", "all"), ("gold", "gold-key"), ("macro", "macro")): idx = [i for i in after if subset == "all" or rows[i]["gold"] == (subset == "gold")] if idx: blk = agreement_block([exp[i] for i in idx], [a_full[i]["cls"] for i in idx]) blk_a_after[subset] = blk print_agreement(f"published from {oos_from[:10]} ({name}), OUT-OF-SAMPLE", blk) relevance: Dict[str, Any] = {} if after and "p_relevant" in a_full[after[0]]: # relevance vs DeepSeek |xau| (>= 0.25 relevant, <= 0.10 not, grey skipped) pairs = [(abs(rows[i]["xau"]) >= 0.25, a_full[i]["relevant"]) for i in after if abs(rows[i]["xau"]) >= 0.25 or abs(rows[i]["xau"]) <= 0.10] tp = sum(1 for t, p_ in pairs if t and p_) fp = sum(1 for t, p_ in pairs if not t and p_) fn = sum(1 for t, p_ in pairs if t and not p_) prec = tp / (tp + fp) if tp + fp else 0.0 rec = tp / (tp + fn) if tp + fn else 0.0 dir_rows = [i for i in after if abs(rows[i]["xau"]) >= 0.25] dir_hits = sum(1 for i in dir_rows if (a_full[i]["p_bullish"] >= 0.5) == (rows[i]["xau"] > 0)) relevance = {"n": len(pairs), "precision": prec, "recall": rec, "f1": 2 * prec * rec / (prec + rec) if prec + rec else 0.0, "direction_n": len(dir_rows), "direction_accuracy": dir_hits / len(dir_rows) if dir_rows else None} print(f" relevance on OUT-OF-SAMPLE rows (|xau|>=0.25 vs <=0.10, n={len(pairs)}): " f"precision {prec:.3f} recall {rec:.3f} F1 {relevance['f1']:.3f}") print(f" direction on relevant OUT-OF-SAMPLE rows: " f"{fmt_pct(dir_hits, len(dir_rows))}") if args.scorer == "transformer": tag = ts_tag() + "_transformer" + (f"_{args.model.name}" if args.model else "") write_json(RESULTS_DIR / f"archive_{tag}.json", { "generated_at": tag, "db": str(args.db), "threshold": thr, "rows": len(rows), "model": model.meta, "in_sample_or_mixed": {"headline": blk_a_full}, "out_of_sample": blk_a_after, "relevance_out_of_sample": relevance}) print(f"\nwrote results/archive_{tag}.json") return 0 # ── Part B: expanding daily folds ────────────────────────────────── hr("B. Expanding daily folds — retrain on days < d, score day d (OUT-OF-SAMPLE)") kw = KeywordScorer() oof: Dict[str, List[Dict[str, Any]]] = {"full": [], "head": []} fold_table = [] print(f" {'day':<10} {'train':>5} {'test':>4} | {'full acc':>8} {'full F1':>7} " f"| {'head acc':>8} {'head F1':>7} | {'kw acc':>6} {'kw F1':>5} | fit s") for d in days: train = [r for r in rows if r["day"] < d and (args.train_scope == "all" or r["gold"])] test = [r for r in rows if r["day"] == d] if len(train) < args.min_train or not test: continue exp_d = [to_class(r["xau"], thr) for r in test] res_full = fit_and_score(train, test, True, thr, args.seed) res_head = fit_and_score(train, test, False, thr, args.seed) if res_full is None or res_head is None: print(f" {d:<10} {len(train):>5} {len(test):>4} | skipped (a class has < 2 rows)") continue preds_full, fit_full = res_full preds_head, fit_head = res_head kw_cls = [to_class(kw.score(r["headline"]), thr) for r in test] b_full = agreement_block(exp_d, [p["cls"] for p in preds_full]) b_head = agreement_block(exp_d, [p["cls"] for p in preds_head]) b_kw = agreement_block(exp_d, kw_cls) fold_table.append({"day": d, "n_train": len(train), "n_test": len(test), "full": b_full, "head": b_head, "keyword": b_kw, "fit_seconds_full": fit_full, "fit_seconds_head": fit_head}) print(f" {d:<10} {len(train):>5} {len(test):>4} | " f"{100 * b_full['accuracy']:7.1f}% {b_full['macro_f1']:7.3f} | " f"{100 * b_head['accuracy']:7.1f}% {b_head['macro_f1']:7.3f} | " f"{100 * b_kw['accuracy']:5.1f}% {b_kw['macro_f1']:5.3f} | {fit_full:.1f}") for variant, preds in (("full", preds_full), ("head", preds_head)): for r, p, e, k in zip(test, preds, exp_d, kw_cls): oof[variant].append({ "hash": r["hash"], "day": d, "n_train": len(train), "gold": r["gold"], "headline": r["headline"], "label_xau": r["xau"], "label_cls": e, "cls": p["cls"], "label": p["label"], "score": round(p["score"], 4), "confidence": round(p["confidence"], 4), "margin": round(p["margin"], 4), "kw_cls": k, }) pooled = {} if oof["full"]: print("\n pooled out-of-fold:") for variant, name in (("full", "headline+description (shipped recipe)"), ("head", "headline only")): o = oof[variant] blk = agreement_block([r["label_cls"] for r in o], [r["cls"] for r in o], [r["label_xau"] for r in o], [r["score"] for r in o]) pooled[variant] = blk print_agreement(f"OOF model, {name}", blk) o = oof["full"] blk = agreement_block([r["label_cls"] for r in o], [r["kw_cls"] for r in o]) pooled["keyword"] = blk print_agreement("keyword fallback on the same rows", blk) for subset in ("gold", "macro"): o = [r for r in oof["full"] if r["gold"] == (subset == "gold")] if o: blk = agreement_block([r["label_cls"] for r in o], [r["cls"] for r in o], [r["label_xau"] for r in o], [r["score"] for r in o]) pooled[f"full_{subset}"] = blk print_agreement(f"OOF model, shipped recipe, {subset}-key rows", blk) for since in ("2026-09-18",): for subset in ("all", "gold", "macro"): o = [r for r in oof["full"] if r["day"] >= since and (subset == "all" or r["gold"] == (subset == "gold"))] if o: blk = agreement_block([r["label_cls"] for r in o], [r["cls"] for r in o]) pooled[f"full_since_{since}_{subset}"] = blk print_agreement(f"OOF, days >= {since}, {subset}", blk) big = [r for r in oof["full"] if r["n_train"] >= 300] if big: blk = agreement_block([r["label_cls"] for r in big], [r["cls"] for r in big], [r["label_xau"] for r in big], [r["score"] for r in big]) pooled["full_train_ge_300"] = blk print_agreement("OOF model, shipped recipe, folds with ≥300 train rows", blk) for variant in ("full", "head"): o = oof[variant] print(f"\n OOF confusion, {variant} (rows = DeepSeek label, cols = model):") cm = pooled[variant]["confusion"] print(" " + "".join(f"{CLASS_NAMES[c]:>9}" for c in (-1, 0, 1))) for c, row in zip((-1, 0, 1), cm): print(f" {CLASS_NAMES[c]:<9} " + "".join(f"{v:>9}" for v in row)) else: print(" no fold could be trained (too few rows); Parts C and D need OOF rows") # ── Part C: routing ──────────────────────────────────────────────── routing: List[Dict[str, Any]] = [] if oof["full"]: hr("C. Confidence routing on pooled OOF (shipped recipe): keep locally vs send to DeepSeek") routing = routing_table(oof["full"]) print(f" {'conf≥':>5} {'margin≥':>7} | {'kept':>5} {'share':>6} | " f"{'kept agree':>10} | {'hybrid agree':>12}") for r in routing: if r["margin_min"] in (0.0, 0.2, 0.4) or r["confidence_min"] in (0.7,): ka = "n/a" if r["kept_accuracy"] is None else f"{100 * r['kept_accuracy']:.1f}%" print(f" {r['confidence_min']:>5.2f} {r['margin_min']:>7.2f} | " f"{r['kept']:>5} {100 * r['kept_share']:5.1f}% | {ka:>10} | " f"{100 * r['hybrid_agreement']:11.1f}%") print(" (hybrid agree = kept rows judged by the model + routed rows answered " "by DeepSeek itself; kept share = DeepSeek calls saved)") # ── Part D: DeepSeek self-agreement ──────────────────────────────── reprobe: Dict[str, Any] = {} if args.reprobe > 0: hr(f"D. DeepSeek re-scored today on {args.reprobe} archived headlines (teacher noise)") rng = random.Random(args.seed) sample = rng.sample(rows, min(args.reprobe, len(rows))) ds = DeepSeekScorer(timeout=args.timeout, no_network=args.no_network) live = ds.score_many([r["headline"] for r in sample]) pairs = [(r, l) for r, l in zip(sample, live) if l is not None] if not pairs: print(" no live scores (network off or every call failed)") else: old_cls = [to_class(r["xau"], thr) for r, _ in pairs] new_cls = [to_class(l["xau"], thr) for _, l in pairs] exact = sum(1 for r, l in pairs if abs(r["xau"] - l["xau"]) < 1e-9) blk = agreement_block(old_cls, new_cls, [r["xau"] for r, _ in pairs], [l["xau"] for _, l in pairs]) print_agreement("DeepSeek today vs DeepSeek archived", blk) print(f" identical xau value: {fmt_pct(exact, len(pairs))}; " f"|diff| mean {blk['mae']:.3f}") same_prompt = sum(1 for r, _ in pairs if r["teacher_prompt_sha1"] == ds.prompt_sha1) print(f" archived rows scored with today's prompt: {same_prompt}/{len(pairs)}" f" (a different prompt sha1 means a different question was asked)") oof_by_hash = {r["hash"]: r for r in oof["full"]} joined = [(r, l, oof_by_hash[r["hash"]]) for r, l in pairs if r["hash"] in oof_by_hash] d_out: Dict[str, Any] = {"n_live": len(pairs), "identical_xau": exact, "self_agreement": blk, "same_prompt": same_prompt, "deepseek": ds.stats()} if joined: b_old = agreement_block([to_class(r["xau"], thr) for r, _, _ in joined], [o["cls"] for _, _, o in joined]) b_new = agreement_block([to_class(l["xau"], thr) for _, l, _ in joined], [o["cls"] for _, _, o in joined]) print_agreement(f"OOF model vs archived DeepSeek ({len(joined)} joined)", b_old) print_agreement("OOF model vs today's DeepSeek (same rows)", b_new) unstable = [(r, l, o) for r, l, o in joined if to_class(r["xau"], thr) != to_class(l["xau"], thr)] if unstable: m_hit = sum(1 for r, l, o in unstable if o["cls"] in (to_class(r["xau"], thr), to_class(l["xau"], thr))) print(f" rows where DeepSeek changed class: {len(unstable)}; model " f"matched one of the two answers on {m_hit}") d_out["oof_vs_archived"] = b_old d_out["oof_vs_today"] = b_new reprobe = d_out write_csv(RESULTS_DIR / f"archive_reprobe_{ts_tag()}.csv", [ {"hash": r["hash"], "headline": r["headline"], "day": r["day"], "archived_xau": r["xau"], "today_xau": l["xau"], "today_s": l["s"], "archived_cls": to_class(r["xau"], thr), "today_cls": to_class(l["xau"], thr), "oof_model_cls": oof_by_hash.get(r["hash"], {}).get("cls"), "oof_model_score": oof_by_hash.get(r["hash"], {}).get("score")} for r, l in pairs]) tag = ts_tag() + f"_{args.train_scope}" + (f"_{args.model.stem}" if args.model else "") summary = { "generated_at": tag, "db": str(args.db), "threshold": thr, "rows": len(rows), "days": days, "rows_per_day": dict(per_day), "label_distribution": dict(label_dist), "model": model.meta, "train_scope": args.train_scope, "extra_labels": str(args.extra_labels) if args.extra_labels else None, "in_sample": {"headline_description": blk_a_full, "headline_only": blk_a_head}, "after_training": blk_a_after, "relevance_after_training": relevance, "folds": fold_table, "pooled_oof": pooled, "routing": routing, "reprobe": reprobe, } write_json(RESULTS_DIR / f"archive_{tag}.json", summary) if oof["full"]: merged = [] head_by_hash = {r["hash"]: r for r in oof["head"]} for r in oof["full"]: h = head_by_hash.get(r["hash"], {}) merged.append({**r, "head_cls": h.get("cls"), "head_score": h.get("score"), "head_confidence": h.get("confidence")}) write_csv(RESULTS_DIR / f"archive_oof_{tag}.csv", merged) print(f"\nwrote results/archive_{tag}.json" + (f" and results/archive_oof_{tag}.csv" if oof["full"] else "")) return 0 if __name__ == "__main__": raise SystemExit(main())