Harwinder Sekhon
01baef48ff
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked.
214 lines
9.6 KiB
Python
214 lines
9.6 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Golden-set benchmark: scikit model vs DeepSeek vs keyword scorer on the
|
|
78 human-labelled headlines from the 2026-08-06 DeepSeek validation.
|
|
|
|
This is the only INDEPENDENT truth available: the model was trained on
|
|
DeepSeek's own labels, so agreement with DeepSeek on archived headlines
|
|
says how well the student copies the teacher, not whether either is
|
|
right. The golden set was labelled by a reviewer, has 13 categories
|
|
(hawkish/dovish Fed, strong/weak dollar, real yields, safe haven,
|
|
risk-on, direct gold news, negation traps, inverted metrics and 8
|
|
irrelevant distractors) and DeepSeek scored 78/78 on it in August.
|
|
|
|
Grading rule (same as the August harness): a continuous XAU score is
|
|
bullish at >= +0.15, bearish at <= -0.15, neutral in between. The
|
|
scikit model is graded two ways: its native argmax label (what the
|
|
WORKFLOW doc says to use) and its P(bull)-P(bear) score through the
|
|
same band; a band sweep shows how sensitive that is.
|
|
|
|
Usage:
|
|
venv/bin/python eval_golden.py # 4 DeepSeek calls, cached
|
|
venv/bin/python eval_golden.py --no-network # report from cache only
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import statistics
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
|
|
from eval_common import (CLASS_NAMES, NEUTRAL_BAND_GOLDEN, RESULTS_DIR,
|
|
DeepSeekScorer, KeywordScorer, make_scorer,
|
|
agreement_block, fmt_pct, hr, load_golden, mcnemar,
|
|
print_agreement, print_confusion, spearman, to_class,
|
|
ts_tag, write_csv, write_json)
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
ap.add_argument("--band", type=float, default=NEUTRAL_BAND_GOLDEN,
|
|
help="neutral band for continuous scores (default 0.15)")
|
|
ap.add_argument("--timeout", type=float, default=30.0)
|
|
ap.add_argument("--no-network", action="store_true",
|
|
help="never call DeepSeek; use the disk cache only")
|
|
ap.add_argument("--scorer", choices=("tfidf", "transformer"), default="tfidf")
|
|
ap.add_argument("--model", type=Path, default=None,
|
|
help="model to grade (default: the shipped bundle of --scorer)")
|
|
ap.add_argument("--force", action="store_true",
|
|
help="re-send every headline to DeepSeek even if cached")
|
|
args = ap.parse_args()
|
|
band = args.band
|
|
|
|
golden = load_golden()
|
|
heads = [g["headline"] for g in golden]
|
|
print(f"golden set: {len(golden)} headlines, "
|
|
f"{len({g['category'] for g in golden})} categories")
|
|
|
|
model = make_scorer(args.scorer, args.model)
|
|
kw = KeywordScorer()
|
|
ds = DeepSeekScorer(timeout=args.timeout, no_network=args.no_network)
|
|
print(f"model: trained {model.meta['trained_at']} on "
|
|
f"{model.meta['training_examples']} examples; "
|
|
f"DeepSeek: {ds.model}, prompt {ds.prompt_sha1[:8]}")
|
|
|
|
m_res = [model.score(h) for h in heads] # headline only: golden has none
|
|
k_res = [kw.score(h) for h in heads]
|
|
d_res = ds.score_many(heads, force=args.force)
|
|
|
|
rows = []
|
|
for g, m, k, d in zip(golden, m_res, k_res, d_res):
|
|
rows.append({
|
|
"headline": g["headline"], "category": g["category"],
|
|
"exp": g["expected_xau_sign"], "rationale": g["rationale"],
|
|
"model_label": m["label"], "model_cls": m["cls"],
|
|
"model_score": round(m["score"], 4),
|
|
"model_conf": round(m["confidence"], 4),
|
|
"model_margin": round(m["margin"], 4),
|
|
"model_cls_band": to_class(m["score"], band),
|
|
"model_ms": round(m["ms"], 2),
|
|
"kw_xau": round(k, 3), "kw_cls": to_class(k, band),
|
|
"ds_s": None if d is None else d["s"],
|
|
"ds_xau": None if d is None else d["xau"],
|
|
"ds_cls": None if d is None else to_class(d["xau"], band),
|
|
"ds_cached": None if d is None else d["cached"],
|
|
})
|
|
|
|
scored = [r for r in rows if r["ds_cls"] is not None]
|
|
missing = len(rows) - len(scored)
|
|
if missing:
|
|
print(f"WARNING: {missing} headlines have no DeepSeek score "
|
|
"(network off or the call failed); DeepSeek rows use the "
|
|
f"{len(scored)} that do")
|
|
|
|
exp_all = [r["exp"] for r in rows]
|
|
exp_ds = [r["exp"] for r in scored]
|
|
|
|
hr("GOLDEN SET — 3-class accuracy against the reviewer's labels")
|
|
blocks = {
|
|
"model (native argmax label)": agreement_block(
|
|
exp_all, [r["model_cls"] for r in rows]),
|
|
f"model (score through band ±{band:.2f})": agreement_block(
|
|
exp_all, [r["model_cls_band"] for r in rows]),
|
|
f"DeepSeek live (xau through band ±{band:.2f})": agreement_block(
|
|
exp_ds, [r["ds_cls"] for r in scored]),
|
|
f"keyword fallback (band ±{band:.2f})": agreement_block(
|
|
exp_all, [r["kw_cls"] for r in rows]),
|
|
}
|
|
for label, blk in blocks.items():
|
|
print_agreement(label, blk)
|
|
print(" (dir = expected non-neutral rows predicted with the right sign; "
|
|
"sign = same sign when both lean)")
|
|
|
|
hr("Model band sweep (its score is P(bullish) - P(bearish))")
|
|
sweep = {}
|
|
for b in (0.05, 0.10, 0.15, 0.20, 0.30, 0.50):
|
|
cls = [to_class(r["model_score"], b) for r in rows]
|
|
hits = sum(1 for e, c in zip(exp_all, cls) if e == c)
|
|
neutral_pred = sum(1 for c in cls if c == 0)
|
|
sweep[str(b)] = {"hits": hits, "n": len(rows), "neutral_predicted": neutral_pred}
|
|
print(f" band ±{b:.2f}: {fmt_pct(hits, len(rows)):<18} "
|
|
f"neutral predictions {neutral_pred}")
|
|
|
|
hr("Per category (hits / n): model | DeepSeek | keyword")
|
|
by_cat = defaultdict(list)
|
|
for r in rows:
|
|
by_cat[r["category"]].append(r)
|
|
cat_table = {}
|
|
for cat in sorted(by_cat):
|
|
grp = by_cat[cat]
|
|
grp_ds = [r for r in grp if r["ds_cls"] is not None]
|
|
mh = sum(1 for r in grp if r["model_cls"] == r["exp"])
|
|
dh = sum(1 for r in grp_ds if r["ds_cls"] == r["exp"])
|
|
kh = sum(1 for r in grp if r["kw_cls"] == r["exp"])
|
|
cat_table[cat] = {"n": len(grp), "model": mh, "deepseek": dh,
|
|
"deepseek_n": len(grp_ds), "keyword": kh}
|
|
print(f" {cat:<18} {mh:>2}/{len(grp):<2} | {dh:>2}/{len(grp_ds):<2} "
|
|
f"| {kh:>2}/{len(grp)}")
|
|
|
|
print_confusion(blocks["model (native argmax label)"]["confusion"],
|
|
"Model confusion")
|
|
if scored:
|
|
print_confusion(
|
|
blocks[f"DeepSeek live (xau through band ±{band:.2f})"]["confusion"],
|
|
"DeepSeek confusion")
|
|
|
|
hr("Paired McNemar (exact, two-sided) on the same headlines")
|
|
tests = {}
|
|
if scored:
|
|
ha = [r["model_cls"] == r["exp"] for r in scored]
|
|
hb = [r["ds_cls"] == r["exp"] for r in scored]
|
|
b, c, p = mcnemar(ha, hb)
|
|
tests["model_vs_deepseek"] = {"model_only_right": b, "deepseek_only_right": c, "p": p}
|
|
print(f" model vs DeepSeek: model-only-right {b}, DeepSeek-only-right {c}, p={p:.4f}")
|
|
ha = [r["model_cls"] == r["exp"] for r in rows]
|
|
hb = [r["kw_cls"] == r["exp"] for r in rows]
|
|
b, c, p = mcnemar(ha, hb)
|
|
tests["model_vs_keyword"] = {"model_only_right": b, "keyword_only_right": c, "p": p}
|
|
print(f" model vs keyword: model-only-right {b}, keyword-only-right {c}, p={p:.4f}")
|
|
|
|
if scored:
|
|
rho = spearman([r["model_score"] for r in scored], [r["ds_xau"] for r in scored])
|
|
print(f"\n Spearman(model score, DeepSeek xau) over {len(scored)}: {rho:+.3f}")
|
|
else:
|
|
rho = float("nan")
|
|
|
|
hr("Model misclassifications (expected -> model label, score, confidence)")
|
|
for r in rows:
|
|
if r["model_cls"] != r["exp"]:
|
|
ds_txt = "" if r["ds_xau"] is None else f" DS {r['ds_xau']:+.2f}"
|
|
print(f" exp {r['exp']:+d} -> {r['model_label']:<7} "
|
|
f"{r['model_score']:+.2f} c={r['model_conf']:.2f}{ds_txt} "
|
|
f"[{r['category']}] {r['headline'][:70]}")
|
|
if scored:
|
|
hr("DeepSeek misclassifications")
|
|
wrong = [r for r in scored if r["ds_cls"] != r["exp"]]
|
|
if not wrong:
|
|
print(" none")
|
|
for r in wrong:
|
|
print(f" exp {r['exp']:+d} -> {r['ds_xau']:+.2f} "
|
|
f"[{r['category']}] {r['headline'][:70]}")
|
|
|
|
hr("Latency")
|
|
m_ms = [r["model_ms"] for r in rows]
|
|
st = ds.stats()
|
|
print(f" model: mean {statistics.fmean(m_ms):.2f} ms, median "
|
|
f"{statistics.median(m_ms):.2f} ms, max {max(m_ms):.2f} ms per headline (in-process)")
|
|
if st["headlines_sent"]:
|
|
print(f" DeepSeek: {st['calls']} call(s), {st['headlines_sent']} headlines sent, "
|
|
f"mean batch {st['batch_ms_mean']:.0f} ms, max {st['batch_ms_max']:.0f} ms, "
|
|
f"≈ {st['per_headline_ms']:.0f} ms per headline; {st['cache_hits']} cache hits")
|
|
else:
|
|
print(f" DeepSeek: 0 headlines sent this run ({st['cache_hits']} cache hits)")
|
|
|
|
tag = ts_tag() + ("" if args.scorer == "tfidf" else f"_{args.scorer}") + (
|
|
f"_{args.model.stem}" if args.model else "")
|
|
summary = {
|
|
"generated_at": tag, "band": band, "n": len(rows),
|
|
"deepseek_scored": len(scored), "model": model.meta,
|
|
"deepseek": st, "blocks": blocks, "band_sweep": sweep,
|
|
"per_category": cat_table, "mcnemar": tests,
|
|
"spearman_model_vs_deepseek": rho,
|
|
"model_latency_ms": {"mean": statistics.fmean(m_ms),
|
|
"median": statistics.median(m_ms), "max": max(m_ms)},
|
|
}
|
|
write_json(RESULTS_DIR / f"golden_{tag}.json", summary)
|
|
write_csv(RESULTS_DIR / f"golden_{tag}.csv", rows)
|
|
print(f"\nwrote results/golden_{tag}.json and .csv")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|