Harwinder Sekhon
1d72e4006a
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
206 lines
9.9 KiB
Python
206 lines
9.9 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Backfill the missing gold labels with DeepSeek, retrain, re-measure.
|
|
|
|
The archive holds ~930 DeepSeek-scored headlines whose gold score was
|
|
discarded because they carry no gold keyword (plus ~31 headlines that
|
|
were never DeepSeek-scored at all). This script sends every one of
|
|
them through the live DeepSeek headline path (batches of 25, cached),
|
|
keeps the labels in results/backfill_labels.jsonl, and then asks:
|
|
|
|
1. Daily folds, same as eval_archive.py, but with the backfilled
|
|
macro headlines added to each fold's training set. Test rows are
|
|
split into gold-keyword headlines (the old test set, so the
|
|
number is comparable to the 64.4% baseline) and macro headlines
|
|
(new).
|
|
2. A model trained on everything, graded on the 78-headline golden
|
|
set per category — the independent test. Baseline 39/78.
|
|
|
|
The retrained bundle is written to results/, never to models/. The
|
|
archive is opened read-only.
|
|
|
|
Usage:
|
|
venv/bin/python backfill_labels.py # ~40 DeepSeek calls once
|
|
venv/bin/python backfill_labels.py --no-network
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from collections import Counter
|
|
from typing import Any, Dict, List
|
|
|
|
from eval_common import (CLASS_NAMES, DEFAULT_DB, LABEL_THRESHOLD_TRAIN,
|
|
RESULTS_DIR, DeepSeekScorer, NAME_TO_CLASS,
|
|
agreement_block, fmt_pct, hr, load_archive_rows,
|
|
load_golden, print_agreement, to_class, ts_tag,
|
|
utc_now_iso, write_json)
|
|
|
|
LABELS_FILE = RESULTS_DIR / "backfill_labels.jsonl"
|
|
|
|
|
|
def load_labels() -> Dict[str, Dict[str, Any]]:
|
|
out: Dict[str, Dict[str, Any]] = {}
|
|
if LABELS_FILE.is_file():
|
|
with LABELS_FILE.open(encoding="utf-8") as fh:
|
|
for line in fh:
|
|
if line.strip():
|
|
r = json.loads(line)
|
|
out[r["hash"]] = r
|
|
return out
|
|
|
|
|
|
def train(rows: List[Dict[str, Any]], thr: float, seed: int = 42):
|
|
from news_sentiment import article_text, build_pipeline
|
|
texts = [article_text(r["headline"], r["description"]) for r in rows]
|
|
labels = [CLASS_NAMES[to_class(r["label_xau"], thr)] for r in rows]
|
|
dist = Counter(labels)
|
|
if len(dist) < 3 or min(dist.values()) < 2:
|
|
return None
|
|
pipe = build_pipeline(random_state=seed, min_df=2, max_features=50_000)
|
|
pipe.fit(texts, labels)
|
|
return pipe
|
|
|
|
|
|
def predict(pipe, rows: List[Dict[str, Any]]) -> List[int]:
|
|
from news_sentiment import article_text
|
|
classes = [str(c) for c in pipe.classes_]
|
|
out = []
|
|
for p in pipe.predict_proba([article_text(r["headline"], r["description"]) for r in rows]):
|
|
by = {c: float(v) for c, v in zip(classes, p)}
|
|
out.append(NAME_TO_CLASS[max(by, key=by.get)])
|
|
return out
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
ap.add_argument("--threshold", type=float, default=LABEL_THRESHOLD_TRAIN)
|
|
ap.add_argument("--timeout", type=float, default=30.0)
|
|
ap.add_argument("--no-network", action="store_true")
|
|
ap.add_argument("--min-train", type=int, default=40)
|
|
args = ap.parse_args()
|
|
thr = args.threshold
|
|
|
|
archive = load_archive_rows(DEFAULT_DB)
|
|
gold_rows = [r for r in archive if r["score_source"] == "llm" and r["xau"] is not None]
|
|
missing = [r for r in archive if not (r["score_source"] == "llm" and r["xau"] is not None)]
|
|
print(f"archive: {len(archive)} rows; {len(gold_rows)} with a DeepSeek gold label, "
|
|
f"{len(missing)} without ({sum(1 for r in missing if r['score_source'] == 'llm')} "
|
|
f"scored by DeepSeek without a gold key, "
|
|
f"{sum(1 for r in missing if r['score_source'] != 'llm')} never DeepSeek-scored)")
|
|
|
|
# ── 1. label the missing rows ──────────────────────────────────────
|
|
labels = load_labels()
|
|
todo = [r for r in missing if r["hash"] not in labels]
|
|
if todo:
|
|
ds = DeepSeekScorer(timeout=args.timeout, no_network=args.no_network)
|
|
res = ds.score_many([r["headline"] for r in todo])
|
|
with LABELS_FILE.open("a", encoding="utf-8") as fh:
|
|
for r, d in zip(todo, res):
|
|
if d is None:
|
|
continue
|
|
rec = {"hash": r["hash"], "headline": r["headline"], "day": r["day"],
|
|
"s": d["s"], "xau": d["xau"], "labelled_at": utc_now_iso(),
|
|
"model": ds.model, "prompt_sha1": ds.prompt_sha1,
|
|
"archived_overall": r["score"], "score_source_before": r["score_source"]}
|
|
fh.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
|
labels[r["hash"]] = rec
|
|
st = ds.stats()
|
|
print(f" labelled {sum(1 for d in res if d)} of {len(todo)}: {st['calls']} call(s), "
|
|
f"{st['headlines_sent']} sent, {st['cache_hits']} cache hits, {st['failed_calls']} failed")
|
|
macro_rows = []
|
|
for r in missing:
|
|
lab = labels.get(r["hash"])
|
|
if lab is not None:
|
|
macro_rows.append({**r, "label_xau": float(lab["xau"]), "kind": "macro"})
|
|
for r in gold_rows:
|
|
r["label_xau"] = r["xau"]
|
|
r["kind"] = "gold"
|
|
print(f" usable: {len(gold_rows)} gold-keyword rows + {len(macro_rows)} macro rows = "
|
|
f"{len(gold_rows) + len(macro_rows)}")
|
|
dist_m = Counter(CLASS_NAMES[to_class(r["label_xau"], thr)] for r in macro_rows)
|
|
print(f" macro labels at ±{thr}: {dict(sorted(dist_m.items()))}")
|
|
# how the live platform's -overall reading compares with DeepSeek's own xau
|
|
usd_only = [r for r in macro_rows if r["score_source"] == "llm" and r["score"] is not None]
|
|
if usd_only:
|
|
b = agreement_block([to_class(r["label_xau"], thr) for r in usd_only],
|
|
[to_class(-float(r["score"]), thr) for r in usd_only])
|
|
print(f" side note: on {b['n']} macro rows the live gold reading (-overall) matches "
|
|
f"DeepSeek's own gold score {fmt_pct(b['hits'], b['n'])} "
|
|
f"(sign {fmt_pct(b['sign_hits'], b['sign_n'])})")
|
|
|
|
# ── 2. daily folds: gold-only training vs gold+macro training ─────
|
|
all_rows = gold_rows + macro_rows
|
|
days = sorted({r["day"] for r in all_rows if r["day"]})
|
|
hr("Daily folds — train on days < d; test on day d (OOF agreement with DeepSeek)")
|
|
print(f" {'day':<10} {'train g':>7} {'train g+m':>9} | {'gold test':>9} {'g-only':>7} {'g+m':>6} "
|
|
f"| {'macro test':>10} {'g-only':>7} {'g+m':>6}")
|
|
oof: Dict[str, Dict[str, List[int]]] = {k: {"exp": [], "pred": []} for k in
|
|
("gold/g-only", "gold/g+m", "macro/g-only", "macro/g+m")}
|
|
for d in days:
|
|
tr_g = [r for r in gold_rows if r["day"] < d]
|
|
tr_gm = [r for r in all_rows if r["day"] < d]
|
|
te_g = [r for r in gold_rows if r["day"] == d]
|
|
te_m = [r for r in macro_rows if r["day"] == d]
|
|
if len(tr_g) < args.min_train or not (te_g or te_m):
|
|
continue
|
|
p_g = train(tr_g, thr)
|
|
p_gm = train(tr_gm, thr)
|
|
if p_g is None or p_gm is None:
|
|
continue
|
|
cells = []
|
|
for name, te in (("gold", te_g), ("macro", te_m)):
|
|
exp = [to_class(r["label_xau"], thr) for r in te]
|
|
for tag, pipe in (("g-only", p_g), ("g+m", p_gm)):
|
|
pred = predict(pipe, te) if te else []
|
|
oof[f"{name}/{tag}"]["exp"].extend(exp)
|
|
oof[f"{name}/{tag}"]["pred"].extend(pred)
|
|
hits = sum(1 for e, p in zip(exp, pred) if e == p)
|
|
cells.append(f"{100 * hits / len(te):5.1f}%" if te else " n/a")
|
|
print(f" {d:<10} {len(tr_g):>7} {len(tr_gm):>9} | {len(te_g):>9} {cells[0]:>7} {cells[1]:>6} "
|
|
f"| {len(te_m):>10} {cells[2]:>7} {cells[3]:>6}")
|
|
pooled = {}
|
|
print("\n pooled out-of-fold:")
|
|
for k, v in oof.items():
|
|
blk = agreement_block(v["exp"], v["pred"])
|
|
pooled[k] = blk
|
|
print_agreement(f"{k:<14} test rows, training = {k.split('/')[1]}", blk)
|
|
|
|
# ── 3. final models on everything -> golden set ────────────────────
|
|
hr("Golden set (human truth, 78 headlines): gold-only training vs gold+macro training")
|
|
golden = load_golden()
|
|
g_rows = [{"headline": g["headline"], "description": None} for g in golden]
|
|
g_exp = [g["expected_xau_sign"] for g in golden]
|
|
golden_out = {}
|
|
for tag, rows in (("gold-only (shipped data)", gold_rows), ("gold+macro (backfilled)", all_rows)):
|
|
pipe = train(rows, thr)
|
|
pred = predict(pipe, g_rows)
|
|
blk = agreement_block(g_exp, pred)
|
|
golden_out[tag] = blk
|
|
by_cat: Dict[str, List[int]] = {}
|
|
for g, p in zip(golden, pred):
|
|
by_cat.setdefault(g["category"], []).append(int(p == g["expected_xau_sign"]))
|
|
print(f"\n {tag}: {fmt_pct(blk['hits'], blk['n'])}, directional "
|
|
f"{fmt_pct(blk['directional_hits'], blk['directional_n'])}")
|
|
print(" " + ", ".join(f"{c} {sum(h)}/{len(h)}" for c, h in sorted(by_cat.items())))
|
|
if tag.startswith("gold+macro"):
|
|
import joblib
|
|
out_path = RESULTS_DIR / "xauusd_sentiment_backfilled.joblib"
|
|
joblib.dump({"model": pipe, "trained_on": len(rows), "threshold": thr,
|
|
"note": "research retrain on gold+macro rows; not the shipped bundle"},
|
|
out_path)
|
|
print(f" retrained pipeline saved to {out_path.name} (research copy)")
|
|
|
|
tag = ts_tag()
|
|
write_json(RESULTS_DIR / f"backfill_{tag}.json", {
|
|
"generated_at": tag, "threshold": thr, "gold_rows": len(gold_rows),
|
|
"macro_rows": len(macro_rows), "macro_label_distribution": dict(dist_m),
|
|
"pooled_oof": pooled, "golden": golden_out})
|
|
print(f"\nwrote results/backfill_{tag}.json; labels in results/{LABELS_FILE.name}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|