harry/News-ML-Decoder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial import: XAUUSD news sentiment models, evals, and data tooling

Browse Source 
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the
two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py)
with its Google News crawler and DeepSeek labeller (data_collect/), the
golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27
results and progress notes.

Transformer weights (model.pt, model.onnx) and the crawl database are not
tracked; rebuild with transformer/train.py and transformer/export_onnx.py.
The DeepSeek label cache in data/ is tracked.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Harwinder Sekhon
2026-09-27 19:22:30 -04:00
commit 1d72e4006a
98 changed files with 1719757 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
eval/backfill_labels.py
+205
View File
@@ -0,0 +1,205 @@
#!/usr/bin/env python3
"""
Backfill the missing gold labels with DeepSeek, retrain, re-measure.
The archive holds ~930 DeepSeek-scored headlines whose gold score was
discarded because they carry no gold keyword (plus ~31 headlines that
were never DeepSeek-scored at all). This script sends every one of
them through the live DeepSeek headline path (batches of 25, cached),
keeps the labels in results/backfill_labels.jsonl, and then asks:
1. Daily folds, same as eval_archive.py, but with the backfilled
macro headlines added to each fold's training set. Test rows are
split into gold-keyword headlines (the old test set, so the
number is comparable to the 64.4% baseline) and macro headlines
(new).
2. A model trained on everything, graded on the 78-headline golden
set per category — the independent test. Baseline 39/78.
The retrained bundle is written to results/, never to models/. The
archive is opened read-only.
Usage:
venv/bin/python backfill_labels.py # ~40 DeepSeek calls once
venv/bin/python backfill_labels.py --no-network
"""
from __future__ import annotations
import argparse
import json
from collections import Counter
from typing import Any, Dict, List
from eval_common import (CLASS_NAMES, DEFAULT_DB, LABEL_THRESHOLD_TRAIN,
RESULTS_DIR, DeepSeekScorer, NAME_TO_CLASS,
agreement_block, fmt_pct, hr, load_archive_rows,
load_golden, print_agreement, to_class, ts_tag,
utc_now_iso, write_json)
LABELS_FILE = RESULTS_DIR / "backfill_labels.jsonl"
def load_labels() -> Dict[str, Dict[str, Any]]:
out: Dict[str, Dict[str, Any]] = {}
if LABELS_FILE.is_file():
with LABELS_FILE.open(encoding="utf-8") as fh:
for line in fh:
if line.strip():
r = json.loads(line)
out[r["hash"]] = r
return out
def train(rows: List[Dict[str, Any]], thr: float, seed: int = 42):
from news_sentiment import article_text, build_pipeline
texts = [article_text(r["headline"], r["description"]) for r in rows]
labels = [CLASS_NAMES[to_class(r["label_xau"], thr)] for r in rows]
dist = Counter(labels)
if len(dist) < 3 or min(dist.values()) < 2:
return None
pipe = build_pipeline(random_state=seed, min_df=2, max_features=50_000)
pipe.fit(texts, labels)
return pipe
def predict(pipe, rows: List[Dict[str, Any]]) -> List[int]:
from news_sentiment import article_text
classes = [str(c) for c in pipe.classes_]
out = []
for p in pipe.predict_proba([article_text(r["headline"], r["description"]) for r in rows]):
by = {c: float(v) for c, v in zip(classes, p)}
out.append(NAME_TO_CLASS[max(by, key=by.get)])
return out
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
ap.add_argument("--threshold", type=float, default=LABEL_THRESHOLD_TRAIN)
ap.add_argument("--timeout", type=float, default=30.0)
ap.add_argument("--no-network", action="store_true")
ap.add_argument("--min-train", type=int, default=40)
args = ap.parse_args()
thr = args.threshold
archive = load_archive_rows(DEFAULT_DB)
gold_rows = [r for r in archive if r["score_source"] == "llm" and r["xau"] is not None]
missing = [r for r in archive if not (r["score_source"] == "llm" and r["xau"] is not None)]
print(f"archive: {len(archive)} rows; {len(gold_rows)} with a DeepSeek gold label, "
f"{len(missing)} without ({sum(1 for r in missing if r['score_source'] == 'llm')} "
f"scored by DeepSeek without a gold key, "
f"{sum(1 for r in missing if r['score_source'] != 'llm')} never DeepSeek-scored)")
# ── 1. label the missing rows ──────────────────────────────────────
labels = load_labels()
todo = [r for r in missing if r["hash"] not in labels]
if todo:
ds = DeepSeekScorer(timeout=args.timeout, no_network=args.no_network)
res = ds.score_many([r["headline"] for r in todo])
with LABELS_FILE.open("a", encoding="utf-8") as fh:
for r, d in zip(todo, res):
if d is None:
continue
rec = {"hash": r["hash"], "headline": r["headline"], "day": r["day"],
"s": d["s"], "xau": d["xau"], "labelled_at": utc_now_iso(),
"model": ds.model, "prompt_sha1": ds.prompt_sha1,
"archived_overall": r["score"], "score_source_before": r["score_source"]}
fh.write(json.dumps(rec, ensure_ascii=False) + "\n")
labels[r["hash"]] = rec
st = ds.stats()
print(f" labelled {sum(1 for d in res if d)} of {len(todo)}: {st['calls']} call(s), "
f"{st['headlines_sent']} sent, {st['cache_hits']} cache hits, {st['failed_calls']} failed")
macro_rows = []
for r in missing:
lab = labels.get(r["hash"])
if lab is not None:
macro_rows.append({**r, "label_xau": float(lab["xau"]), "kind": "macro"})
for r in gold_rows:
r["label_xau"] = r["xau"]
r["kind"] = "gold"
print(f" usable: {len(gold_rows)} gold-keyword rows + {len(macro_rows)} macro rows = "
f"{len(gold_rows) + len(macro_rows)}")
dist_m = Counter(CLASS_NAMES[to_class(r["label_xau"], thr)] for r in macro_rows)
print(f" macro labels at ±{thr}: {dict(sorted(dist_m.items()))}")
# how the live platform's -overall reading compares with DeepSeek's own xau
usd_only = [r for r in macro_rows if r["score_source"] == "llm" and r["score"] is not None]
if usd_only:
b = agreement_block([to_class(r["label_xau"], thr) for r in usd_only],
[to_class(-float(r["score"]), thr) for r in usd_only])
print(f" side note: on {b['n']} macro rows the live gold reading (-overall) matches "
f"DeepSeek's own gold score {fmt_pct(b['hits'], b['n'])} "
f"(sign {fmt_pct(b['sign_hits'], b['sign_n'])})")
# ── 2. daily folds: gold-only training vs gold+macro training ─────
all_rows = gold_rows + macro_rows
days = sorted({r["day"] for r in all_rows if r["day"]})
hr("Daily folds — train on days < d; test on day d (OOF agreement with DeepSeek)")
print(f" {'day':<10} {'train g':>7} {'train g+m':>9} | {'gold test':>9} {'g-only':>7} {'g+m':>6} "
f"| {'macro test':>10} {'g-only':>7} {'g+m':>6}")
oof: Dict[str, Dict[str, List[int]]] = {k: {"exp": [], "pred": []} for k in
("gold/g-only", "gold/g+m", "macro/g-only", "macro/g+m")}
for d in days:
tr_g = [r for r in gold_rows if r["day"] < d]
tr_gm = [r for r in all_rows if r["day"] < d]
te_g = [r for r in gold_rows if r["day"] == d]
te_m = [r for r in macro_rows if r["day"] == d]
if len(tr_g) < args.min_train or not (te_g or te_m):
continue
p_g = train(tr_g, thr)
p_gm = train(tr_gm, thr)
if p_g is None or p_gm is None:
continue
cells = []
for name, te in (("gold", te_g), ("macro", te_m)):
exp = [to_class(r["label_xau"], thr) for r in te]
for tag, pipe in (("g-only", p_g), ("g+m", p_gm)):
pred = predict(pipe, te) if te else []
oof[f"{name}/{tag}"]["exp"].extend(exp)
oof[f"{name}/{tag}"]["pred"].extend(pred)
hits = sum(1 for e, p in zip(exp, pred) if e == p)
cells.append(f"{100 * hits / len(te):5.1f}%" if te else " n/a")
print(f" {d:<10} {len(tr_g):>7} {len(tr_gm):>9} | {len(te_g):>9} {cells[0]:>7} {cells[1]:>6} "
f"| {len(te_m):>10} {cells[2]:>7} {cells[3]:>6}")
pooled = {}
print("\n pooled out-of-fold:")
for k, v in oof.items():
blk = agreement_block(v["exp"], v["pred"])
pooled[k] = blk
print_agreement(f"{k:<14} test rows, training = {k.split('/')[1]}", blk)
# ── 3. final models on everything -> golden set ────────────────────
hr("Golden set (human truth, 78 headlines): gold-only training vs gold+macro training")
golden = load_golden()
g_rows = [{"headline": g["headline"], "description": None} for g in golden]
g_exp = [g["expected_xau_sign"] for g in golden]
golden_out = {}
for tag, rows in (("gold-only (shipped data)", gold_rows), ("gold+macro (backfilled)", all_rows)):
pipe = train(rows, thr)
pred = predict(pipe, g_rows)
blk = agreement_block(g_exp, pred)
golden_out[tag] = blk
by_cat: Dict[str, List[int]] = {}
for g, p in zip(golden, pred):
by_cat.setdefault(g["category"], []).append(int(p == g["expected_xau_sign"]))
print(f"\n {tag}: {fmt_pct(blk['hits'], blk['n'])}, directional "
f"{fmt_pct(blk['directional_hits'], blk['directional_n'])}")
print(" " + ", ".join(f"{c} {sum(h)}/{len(h)}" for c, h in sorted(by_cat.items())))
if tag.startswith("gold+macro"):
import joblib
out_path = RESULTS_DIR / "xauusd_sentiment_backfilled.joblib"
joblib.dump({"model": pipe, "trained_on": len(rows), "threshold": thr,
"note": "research retrain on gold+macro rows; not the shipped bundle"},
out_path)
print(f" retrained pipeline saved to {out_path.name} (research copy)")
tag = ts_tag()
write_json(RESULTS_DIR / f"backfill_{tag}.json", {
"generated_at": tag, "threshold": thr, "gold_rows": len(gold_rows),
"macro_rows": len(macro_rows), "macro_label_distribution": dict(dist_m),
"pooled_oof": pooled, "golden": golden_out})
print(f"\nwrote results/backfill_{tag}.json; labels in results/{LABELS_FILE.name}")
return 0
if __name__ == "__main__":
raise SystemExit(main())