#!/usr/bin/env python3 """ Backfill the missing gold labels with DeepSeek, retrain, re-measure. The archive holds ~930 DeepSeek-scored headlines whose gold score was discarded because they carry no gold keyword (plus ~31 headlines that were never DeepSeek-scored at all). This script sends every one of them through the live DeepSeek headline path (batches of 25, cached), keeps the labels in results/backfill_labels.jsonl, and then asks: 1. Daily folds, same as eval_archive.py, but with the backfilled macro headlines added to each fold's training set. Test rows are split into gold-keyword headlines (the old test set, so the number is comparable to the 64.4% baseline) and macro headlines (new). 2. A model trained on everything, graded on the 78-headline golden set per category — the independent test. Baseline 39/78. The retrained bundle is written to results/, never to models/. The archive is opened read-only. Usage: venv/bin/python backfill_labels.py # ~40 DeepSeek calls once venv/bin/python backfill_labels.py --no-network """ from __future__ import annotations import argparse import json from collections import Counter from typing import Any, Dict, List from eval_common import (CLASS_NAMES, DEFAULT_DB, LABEL_THRESHOLD_TRAIN, RESULTS_DIR, DeepSeekScorer, NAME_TO_CLASS, agreement_block, fmt_pct, hr, load_archive_rows, load_golden, print_agreement, to_class, ts_tag, utc_now_iso, write_json) LABELS_FILE = RESULTS_DIR / "backfill_labels.jsonl" def load_labels() -> Dict[str, Dict[str, Any]]: out: Dict[str, Dict[str, Any]] = {} if LABELS_FILE.is_file(): with LABELS_FILE.open(encoding="utf-8") as fh: for line in fh: if line.strip(): r = json.loads(line) out[r["hash"]] = r return out def train(rows: List[Dict[str, Any]], thr: float, seed: int = 42): from news_sentiment import article_text, build_pipeline texts = [article_text(r["headline"], r["description"]) for r in rows] labels = [CLASS_NAMES[to_class(r["label_xau"], thr)] for r in rows] dist = Counter(labels) if len(dist) < 3 or min(dist.values()) < 2: return None pipe = build_pipeline(random_state=seed, min_df=2, max_features=50_000) pipe.fit(texts, labels) return pipe def predict(pipe, rows: List[Dict[str, Any]]) -> List[int]: from news_sentiment import article_text classes = [str(c) for c in pipe.classes_] out = [] for p in pipe.predict_proba([article_text(r["headline"], r["description"]) for r in rows]): by = {c: float(v) for c, v in zip(classes, p)} out.append(NAME_TO_CLASS[max(by, key=by.get)]) return out def main() -> int: ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) ap.add_argument("--threshold", type=float, default=LABEL_THRESHOLD_TRAIN) ap.add_argument("--timeout", type=float, default=30.0) ap.add_argument("--no-network", action="store_true") ap.add_argument("--min-train", type=int, default=40) args = ap.parse_args() thr = args.threshold archive = load_archive_rows(DEFAULT_DB) gold_rows = [r for r in archive if r["score_source"] == "llm" and r["xau"] is not None] missing = [r for r in archive if not (r["score_source"] == "llm" and r["xau"] is not None)] print(f"archive: {len(archive)} rows; {len(gold_rows)} with a DeepSeek gold label, " f"{len(missing)} without ({sum(1 for r in missing if r['score_source'] == 'llm')} " f"scored by DeepSeek without a gold key, " f"{sum(1 for r in missing if r['score_source'] != 'llm')} never DeepSeek-scored)") # ── 1. label the missing rows ────────────────────────────────────── labels = load_labels() todo = [r for r in missing if r["hash"] not in labels] if todo: ds = DeepSeekScorer(timeout=args.timeout, no_network=args.no_network) res = ds.score_many([r["headline"] for r in todo]) with LABELS_FILE.open("a", encoding="utf-8") as fh: for r, d in zip(todo, res): if d is None: continue rec = {"hash": r["hash"], "headline": r["headline"], "day": r["day"], "s": d["s"], "xau": d["xau"], "labelled_at": utc_now_iso(), "model": ds.model, "prompt_sha1": ds.prompt_sha1, "archived_overall": r["score"], "score_source_before": r["score_source"]} fh.write(json.dumps(rec, ensure_ascii=False) + "\n") labels[r["hash"]] = rec st = ds.stats() print(f" labelled {sum(1 for d in res if d)} of {len(todo)}: {st['calls']} call(s), " f"{st['headlines_sent']} sent, {st['cache_hits']} cache hits, {st['failed_calls']} failed") macro_rows = [] for r in missing: lab = labels.get(r["hash"]) if lab is not None: macro_rows.append({**r, "label_xau": float(lab["xau"]), "kind": "macro"}) for r in gold_rows: r["label_xau"] = r["xau"] r["kind"] = "gold" print(f" usable: {len(gold_rows)} gold-keyword rows + {len(macro_rows)} macro rows = " f"{len(gold_rows) + len(macro_rows)}") dist_m = Counter(CLASS_NAMES[to_class(r["label_xau"], thr)] for r in macro_rows) print(f" macro labels at ±{thr}: {dict(sorted(dist_m.items()))}") # how the live platform's -overall reading compares with DeepSeek's own xau usd_only = [r for r in macro_rows if r["score_source"] == "llm" and r["score"] is not None] if usd_only: b = agreement_block([to_class(r["label_xau"], thr) for r in usd_only], [to_class(-float(r["score"]), thr) for r in usd_only]) print(f" side note: on {b['n']} macro rows the live gold reading (-overall) matches " f"DeepSeek's own gold score {fmt_pct(b['hits'], b['n'])} " f"(sign {fmt_pct(b['sign_hits'], b['sign_n'])})") # ── 2. daily folds: gold-only training vs gold+macro training ───── all_rows = gold_rows + macro_rows days = sorted({r["day"] for r in all_rows if r["day"]}) hr("Daily folds — train on days < d; test on day d (OOF agreement with DeepSeek)") print(f" {'day':<10} {'train g':>7} {'train g+m':>9} | {'gold test':>9} {'g-only':>7} {'g+m':>6} " f"| {'macro test':>10} {'g-only':>7} {'g+m':>6}") oof: Dict[str, Dict[str, List[int]]] = {k: {"exp": [], "pred": []} for k in ("gold/g-only", "gold/g+m", "macro/g-only", "macro/g+m")} for d in days: tr_g = [r for r in gold_rows if r["day"] < d] tr_gm = [r for r in all_rows if r["day"] < d] te_g = [r for r in gold_rows if r["day"] == d] te_m = [r for r in macro_rows if r["day"] == d] if len(tr_g) < args.min_train or not (te_g or te_m): continue p_g = train(tr_g, thr) p_gm = train(tr_gm, thr) if p_g is None or p_gm is None: continue cells = [] for name, te in (("gold", te_g), ("macro", te_m)): exp = [to_class(r["label_xau"], thr) for r in te] for tag, pipe in (("g-only", p_g), ("g+m", p_gm)): pred = predict(pipe, te) if te else [] oof[f"{name}/{tag}"]["exp"].extend(exp) oof[f"{name}/{tag}"]["pred"].extend(pred) hits = sum(1 for e, p in zip(exp, pred) if e == p) cells.append(f"{100 * hits / len(te):5.1f}%" if te else " n/a") print(f" {d:<10} {len(tr_g):>7} {len(tr_gm):>9} | {len(te_g):>9} {cells[0]:>7} {cells[1]:>6} " f"| {len(te_m):>10} {cells[2]:>7} {cells[3]:>6}") pooled = {} print("\n pooled out-of-fold:") for k, v in oof.items(): blk = agreement_block(v["exp"], v["pred"]) pooled[k] = blk print_agreement(f"{k:<14} test rows, training = {k.split('/')[1]}", blk) # ── 3. final models on everything -> golden set ──────────────────── hr("Golden set (human truth, 78 headlines): gold-only training vs gold+macro training") golden = load_golden() g_rows = [{"headline": g["headline"], "description": None} for g in golden] g_exp = [g["expected_xau_sign"] for g in golden] golden_out = {} for tag, rows in (("gold-only (shipped data)", gold_rows), ("gold+macro (backfilled)", all_rows)): pipe = train(rows, thr) pred = predict(pipe, g_rows) blk = agreement_block(g_exp, pred) golden_out[tag] = blk by_cat: Dict[str, List[int]] = {} for g, p in zip(golden, pred): by_cat.setdefault(g["category"], []).append(int(p == g["expected_xau_sign"])) print(f"\n {tag}: {fmt_pct(blk['hits'], blk['n'])}, directional " f"{fmt_pct(blk['directional_hits'], blk['directional_n'])}") print(" " + ", ".join(f"{c} {sum(h)}/{len(h)}" for c, h in sorted(by_cat.items()))) if tag.startswith("gold+macro"): import joblib out_path = RESULTS_DIR / "xauusd_sentiment_backfilled.joblib" joblib.dump({"model": pipe, "trained_on": len(rows), "threshold": thr, "note": "research retrain on gold+macro rows; not the shipped bundle"}, out_path) print(f" retrained pipeline saved to {out_path.name} (research copy)") tag = ts_tag() write_json(RESULTS_DIR / f"backfill_{tag}.json", { "generated_at": tag, "threshold": thr, "gold_rows": len(gold_rows), "macro_rows": len(macro_rows), "macro_label_distribution": dict(dist_m), "pooled_oof": pooled, "golden": golden_out}) print(f"\nwrote results/backfill_{tag}.json; labels in results/{LABELS_FILE.name}") return 0 if __name__ == "__main__": raise SystemExit(main())