Initial import: XAUUSD news sentiment models, evals, and data tooling
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked.
This commit is contained in:
Harwinder Sekhon
@@ -0,0 +1,391 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Archive benchmark: how well does the scikit recipe copy DeepSeek on
|
||||
headlines it has NOT seen, and how noisy is DeepSeek itself?
|
||||
|
||||
Data: the headlines table of ForexBot2's logs/newsdata.db (the file
|
||||
ML-NEWS trained on is a copy of it). Only rows DeepSeek scored with an
|
||||
explicit XAU value are usable (616 on 2026-09-17, eight days). The
|
||||
DeepSeek XAU score is the label: bullish > +0.05, bearish < -0.05,
|
||||
neutral otherwise — exactly the ML-NEWS trainer's rule.
|
||||
|
||||
Parts
|
||||
A In-sample sanity: the SHIPPED bundle against its own training
|
||||
labels. Contaminated by construction; it is the ceiling, shown so
|
||||
the out-of-sample numbers can be read against it.
|
||||
B Expanding daily folds: for each day d, retrain the ML-NEWS
|
||||
pipeline (same hyperparameters) on every row published before d
|
||||
and score day d. Two text recipes: headline+description (shipped)
|
||||
and headline only (DeepSeek itself never sees the description,
|
||||
and the 2026-09-08 rule for a local scorer says headline text
|
||||
only). The keyword fallback is scored on the same rows for
|
||||
context. Pooled out-of-fold (OOF) metrics follow the per-day table.
|
||||
C Confidence routing on the pooled OOF predictions: for a grid of
|
||||
confidence / margin thresholds (the WORKFLOW doc starts at 0.70 /
|
||||
0.20), what share of headlines the model would keep locally, how
|
||||
often those agree with DeepSeek, and the hybrid agreement if the
|
||||
rest go to DeepSeek.
|
||||
D --reprobe N: re-send N archived headlines to DeepSeek today and
|
||||
compare with the score it gave the first time. DeepSeek's
|
||||
self-agreement is the ceiling any student can reach; on the same
|
||||
rows the OOF model is scored against both answers.
|
||||
|
||||
Usage:
|
||||
venv/bin/python eval_archive.py # offline
|
||||
venv/bin/python eval_archive.py --reprobe 100 # + 4 DeepSeek calls
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import random
|
||||
import statistics
|
||||
import time
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from eval_common import (CLASS_NAMES, DEFAULT_DB, LABEL_THRESHOLD_TRAIN,
|
||||
RESULTS_DIR, DeepSeekScorer, KeywordScorer,
|
||||
make_scorer, NAME_TO_CLASS, agreement_block, fmt_pct,
|
||||
hr, load_archive_rows, print_agreement, to_class,
|
||||
ts_tag, write_csv, write_json)
|
||||
|
||||
|
||||
def usable_rows(db: Path, extra_labels: Optional[Path] = None) -> List[Dict[str, Any]]:
|
||||
rows = [r for r in load_archive_rows(db, extra_labels)
|
||||
if r["score_source"] == "llm" and r["xau"] is not None
|
||||
and r["headline"] and r["day"]]
|
||||
rows.sort(key=lambda r: (r["published_at"] or r["first_seen"] or "", r["hash"]))
|
||||
return rows
|
||||
|
||||
|
||||
def fit_and_score(train: List[Dict[str, Any]], test: List[Dict[str, Any]],
|
||||
with_description: bool, threshold: float, seed: int):
|
||||
"""Train the ML-NEWS pipeline on `train`, return per-row predictions
|
||||
for `test` (class, score, confidence, margin) or None if the fold
|
||||
cannot be trained (a class with < 2 examples)."""
|
||||
from news_sentiment import article_text, build_pipeline
|
||||
texts = [article_text(r["headline"], r["description"] if with_description else None)
|
||||
for r in train]
|
||||
labels = [CLASS_NAMES[to_class(r["xau"], threshold)] for r in train]
|
||||
dist = Counter(labels)
|
||||
if len(dist) < 3 or min(dist.values()) < 2:
|
||||
return None
|
||||
pipe = build_pipeline(random_state=seed, min_df=2, max_features=50_000)
|
||||
t0 = time.perf_counter()
|
||||
pipe.fit(texts, labels)
|
||||
fit_s = time.perf_counter() - t0
|
||||
classes = [str(c) for c in pipe.classes_]
|
||||
test_texts = [article_text(r["headline"], r["description"] if with_description else None)
|
||||
for r in test]
|
||||
probs = pipe.predict_proba(test_texts)
|
||||
preds = []
|
||||
for p in probs:
|
||||
by = {c: float(v) for c, v in zip(classes, p)}
|
||||
for c in ("bearish", "neutral", "bullish"):
|
||||
by.setdefault(c, 0.0)
|
||||
label = max(by, key=by.get)
|
||||
ordered = sorted(by.values(), reverse=True)
|
||||
preds.append({"cls": NAME_TO_CLASS[label], "label": label,
|
||||
"score": by["bullish"] - by["bearish"],
|
||||
"confidence": by[label], "margin": ordered[0] - ordered[1]})
|
||||
return preds, fit_s
|
||||
|
||||
|
||||
def routing_table(oof: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
out = []
|
||||
n = len(oof)
|
||||
for conf_t in (0.0, 0.5, 0.6, 0.7, 0.8, 0.9):
|
||||
for margin_t in (0.0, 0.1, 0.2, 0.3, 0.4):
|
||||
kept = [r for r in oof if r["confidence"] >= conf_t and r["margin"] >= margin_t]
|
||||
routed = n - len(kept)
|
||||
hits = sum(1 for r in kept if r["cls"] == r["label_cls"])
|
||||
acc_kept = hits / len(kept) if kept else None
|
||||
hybrid = (hits + routed) / n if n else None # routed rows = DeepSeek's own answer
|
||||
out.append({"confidence_min": conf_t, "margin_min": margin_t,
|
||||
"kept": len(kept), "kept_share": len(kept) / n if n else None,
|
||||
"kept_accuracy": acc_kept, "hybrid_agreement": hybrid,
|
||||
"llm_calls_saved_share": len(kept) / n if n else None})
|
||||
return out
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
ap.add_argument("--db", type=Path, default=DEFAULT_DB)
|
||||
ap.add_argument("--threshold", type=float, default=LABEL_THRESHOLD_TRAIN,
|
||||
help="DeepSeek xau -> class threshold (trainer uses 0.05)")
|
||||
ap.add_argument("--min-train", type=int, default=40,
|
||||
help="skip folds with fewer training rows")
|
||||
ap.add_argument("--seed", type=int, default=42)
|
||||
ap.add_argument("--reprobe", type=int, default=0,
|
||||
help="re-score N archived headlines with DeepSeek today")
|
||||
ap.add_argument("--timeout", type=float, default=30.0)
|
||||
ap.add_argument("--no-network", action="store_true")
|
||||
ap.add_argument("--scorer", choices=("tfidf", "transformer"), default="tfidf",
|
||||
help="model for Part A; transformer runs Part A only (B-D retrain "
|
||||
"the TF-IDF recipe)")
|
||||
ap.add_argument("--model", type=Path, default=None,
|
||||
help="model for Part A (default: the shipped one of --scorer)")
|
||||
ap.add_argument("--extra-labels", type=Path,
|
||||
help="fallback labels JSONL keyed by hash (results/backfill_labels.jsonl)")
|
||||
ap.add_argument("--train-scope", choices=("all", "gold"), default="all",
|
||||
help="Part B trains on every labelled row, or only on rows with "
|
||||
"a gold XAU key; the test day is always every labelled row")
|
||||
args = ap.parse_args()
|
||||
thr = args.threshold
|
||||
|
||||
rows = usable_rows(args.db, args.extra_labels)
|
||||
if not rows:
|
||||
raise SystemExit("no llm-scored XAU rows in the archive")
|
||||
days = sorted({r["day"] for r in rows})
|
||||
per_day = Counter(r["day"] for r in rows)
|
||||
label_dist = Counter(CLASS_NAMES[to_class(r["xau"], thr)] for r in rows)
|
||||
print(f"archive {args.db}: {len(rows)} DeepSeek-scored XAU rows over "
|
||||
f"{len(days)} days ({days[0]} .. {days[-1]}); labels at ±{thr}: "
|
||||
f"{dict(sorted(label_dist.items()))}")
|
||||
print(" rows per day: " + ", ".join(f"{d[5:]}={per_day[d]}" for d in days))
|
||||
print(f" label sources: {dict(Counter(r['xau_src'] for r in rows))}; "
|
||||
f"gold-key rows {sum(r['gold'] for r in rows)}, macro rows "
|
||||
f"{sum(not r['gold'] for r in rows)}")
|
||||
|
||||
# ── Part A: shipped bundle, in-sample ──────────────────────────────
|
||||
hr("A. Shipped bundle on its own training rows (IN-SAMPLE, ceiling only)")
|
||||
model = make_scorer(args.scorer, args.model)
|
||||
print(f" {model.meta['model_type']} trained {model.meta['trained_at']} on "
|
||||
f"{model.meta['training_examples']} examples")
|
||||
exp = [to_class(r["xau"], thr) for r in rows]
|
||||
a_full = [model.score(r["headline"], r["description"]) for r in rows]
|
||||
a_head = [model.score(r["headline"], None) for r in rows]
|
||||
blk_a_full = agreement_block(exp, [m["cls"] for m in a_full],
|
||||
[r["xau"] for r in rows], [m["score"] for m in a_full])
|
||||
blk_a_head = agreement_block(exp, [m["cls"] for m in a_head],
|
||||
[r["xau"] for r in rows], [m["score"] for m in a_head])
|
||||
print_agreement("shipped, headline+description (as trained)", blk_a_full)
|
||||
print_agreement("shipped, headline only (description dropped)", blk_a_head)
|
||||
print(f" model latency: mean {model.mean_ms:.2f} ms per headline")
|
||||
# Rows published from oos_from on were never trained on (TF-IDF: its
|
||||
# trained_at; non-final transformer: its val_from).
|
||||
oos_from = str(model.meta["oos_from"] or "")
|
||||
after = [i for i, r in enumerate(rows) if str(r["published_at"] or "") >= oos_from]
|
||||
blk_a_after: Dict[str, Any] = {}
|
||||
for subset, name in (("all", "all"), ("gold", "gold-key"), ("macro", "macro")):
|
||||
idx = [i for i in after if subset == "all" or rows[i]["gold"] == (subset == "gold")]
|
||||
if idx:
|
||||
blk = agreement_block([exp[i] for i in idx], [a_full[i]["cls"] for i in idx])
|
||||
blk_a_after[subset] = blk
|
||||
print_agreement(f"published from {oos_from[:10]} ({name}), OUT-OF-SAMPLE", blk)
|
||||
relevance: Dict[str, Any] = {}
|
||||
if after and "p_relevant" in a_full[after[0]]:
|
||||
# relevance vs DeepSeek |xau| (>= 0.25 relevant, <= 0.10 not, grey skipped)
|
||||
pairs = [(abs(rows[i]["xau"]) >= 0.25, a_full[i]["relevant"]) for i in after
|
||||
if abs(rows[i]["xau"]) >= 0.25 or abs(rows[i]["xau"]) <= 0.10]
|
||||
tp = sum(1 for t, p_ in pairs if t and p_)
|
||||
fp = sum(1 for t, p_ in pairs if not t and p_)
|
||||
fn = sum(1 for t, p_ in pairs if t and not p_)
|
||||
prec = tp / (tp + fp) if tp + fp else 0.0
|
||||
rec = tp / (tp + fn) if tp + fn else 0.0
|
||||
dir_rows = [i for i in after if abs(rows[i]["xau"]) >= 0.25]
|
||||
dir_hits = sum(1 for i in dir_rows
|
||||
if (a_full[i]["p_bullish"] >= 0.5) == (rows[i]["xau"] > 0))
|
||||
relevance = {"n": len(pairs), "precision": prec, "recall": rec,
|
||||
"f1": 2 * prec * rec / (prec + rec) if prec + rec else 0.0,
|
||||
"direction_n": len(dir_rows),
|
||||
"direction_accuracy": dir_hits / len(dir_rows) if dir_rows else None}
|
||||
print(f" relevance on OUT-OF-SAMPLE rows (|xau|>=0.25 vs <=0.10, n={len(pairs)}): "
|
||||
f"precision {prec:.3f} recall {rec:.3f} F1 {relevance['f1']:.3f}")
|
||||
print(f" direction on relevant OUT-OF-SAMPLE rows: "
|
||||
f"{fmt_pct(dir_hits, len(dir_rows))}")
|
||||
if args.scorer == "transformer":
|
||||
tag = ts_tag() + "_transformer" + (f"_{args.model.name}" if args.model else "")
|
||||
write_json(RESULTS_DIR / f"archive_{tag}.json", {
|
||||
"generated_at": tag, "db": str(args.db), "threshold": thr, "rows": len(rows),
|
||||
"model": model.meta, "in_sample_or_mixed": {"headline": blk_a_full},
|
||||
"out_of_sample": blk_a_after, "relevance_out_of_sample": relevance})
|
||||
print(f"\nwrote results/archive_{tag}.json")
|
||||
return 0
|
||||
|
||||
# ── Part B: expanding daily folds ──────────────────────────────────
|
||||
hr("B. Expanding daily folds — retrain on days < d, score day d (OUT-OF-SAMPLE)")
|
||||
kw = KeywordScorer()
|
||||
oof: Dict[str, List[Dict[str, Any]]] = {"full": [], "head": []}
|
||||
fold_table = []
|
||||
print(f" {'day':<10} {'train':>5} {'test':>4} | {'full acc':>8} {'full F1':>7} "
|
||||
f"| {'head acc':>8} {'head F1':>7} | {'kw acc':>6} {'kw F1':>5} | fit s")
|
||||
for d in days:
|
||||
train = [r for r in rows if r["day"] < d
|
||||
and (args.train_scope == "all" or r["gold"])]
|
||||
test = [r for r in rows if r["day"] == d]
|
||||
if len(train) < args.min_train or not test:
|
||||
continue
|
||||
exp_d = [to_class(r["xau"], thr) for r in test]
|
||||
res_full = fit_and_score(train, test, True, thr, args.seed)
|
||||
res_head = fit_and_score(train, test, False, thr, args.seed)
|
||||
if res_full is None or res_head is None:
|
||||
print(f" {d:<10} {len(train):>5} {len(test):>4} | skipped (a class has < 2 rows)")
|
||||
continue
|
||||
preds_full, fit_full = res_full
|
||||
preds_head, fit_head = res_head
|
||||
kw_cls = [to_class(kw.score(r["headline"]), thr) for r in test]
|
||||
b_full = agreement_block(exp_d, [p["cls"] for p in preds_full])
|
||||
b_head = agreement_block(exp_d, [p["cls"] for p in preds_head])
|
||||
b_kw = agreement_block(exp_d, kw_cls)
|
||||
fold_table.append({"day": d, "n_train": len(train), "n_test": len(test),
|
||||
"full": b_full, "head": b_head, "keyword": b_kw,
|
||||
"fit_seconds_full": fit_full, "fit_seconds_head": fit_head})
|
||||
print(f" {d:<10} {len(train):>5} {len(test):>4} | "
|
||||
f"{100 * b_full['accuracy']:7.1f}% {b_full['macro_f1']:7.3f} | "
|
||||
f"{100 * b_head['accuracy']:7.1f}% {b_head['macro_f1']:7.3f} | "
|
||||
f"{100 * b_kw['accuracy']:5.1f}% {b_kw['macro_f1']:5.3f} | {fit_full:.1f}")
|
||||
for variant, preds in (("full", preds_full), ("head", preds_head)):
|
||||
for r, p, e, k in zip(test, preds, exp_d, kw_cls):
|
||||
oof[variant].append({
|
||||
"hash": r["hash"], "day": d, "n_train": len(train), "gold": r["gold"],
|
||||
"headline": r["headline"], "label_xau": r["xau"],
|
||||
"label_cls": e, "cls": p["cls"], "label": p["label"],
|
||||
"score": round(p["score"], 4), "confidence": round(p["confidence"], 4),
|
||||
"margin": round(p["margin"], 4), "kw_cls": k,
|
||||
})
|
||||
|
||||
pooled = {}
|
||||
if oof["full"]:
|
||||
print("\n pooled out-of-fold:")
|
||||
for variant, name in (("full", "headline+description (shipped recipe)"),
|
||||
("head", "headline only")):
|
||||
o = oof[variant]
|
||||
blk = agreement_block([r["label_cls"] for r in o], [r["cls"] for r in o],
|
||||
[r["label_xau"] for r in o], [r["score"] for r in o])
|
||||
pooled[variant] = blk
|
||||
print_agreement(f"OOF model, {name}", blk)
|
||||
o = oof["full"]
|
||||
blk = agreement_block([r["label_cls"] for r in o], [r["kw_cls"] for r in o])
|
||||
pooled["keyword"] = blk
|
||||
print_agreement("keyword fallback on the same rows", blk)
|
||||
for subset in ("gold", "macro"):
|
||||
o = [r for r in oof["full"] if r["gold"] == (subset == "gold")]
|
||||
if o:
|
||||
blk = agreement_block([r["label_cls"] for r in o], [r["cls"] for r in o],
|
||||
[r["label_xau"] for r in o], [r["score"] for r in o])
|
||||
pooled[f"full_{subset}"] = blk
|
||||
print_agreement(f"OOF model, shipped recipe, {subset}-key rows", blk)
|
||||
for since in ("2026-09-18",):
|
||||
for subset in ("all", "gold", "macro"):
|
||||
o = [r for r in oof["full"] if r["day"] >= since
|
||||
and (subset == "all" or r["gold"] == (subset == "gold"))]
|
||||
if o:
|
||||
blk = agreement_block([r["label_cls"] for r in o], [r["cls"] for r in o])
|
||||
pooled[f"full_since_{since}_{subset}"] = blk
|
||||
print_agreement(f"OOF, days >= {since}, {subset}", blk)
|
||||
big = [r for r in oof["full"] if r["n_train"] >= 300]
|
||||
if big:
|
||||
blk = agreement_block([r["label_cls"] for r in big], [r["cls"] for r in big],
|
||||
[r["label_xau"] for r in big], [r["score"] for r in big])
|
||||
pooled["full_train_ge_300"] = blk
|
||||
print_agreement("OOF model, shipped recipe, folds with ≥300 train rows", blk)
|
||||
for variant in ("full", "head"):
|
||||
o = oof[variant]
|
||||
print(f"\n OOF confusion, {variant} (rows = DeepSeek label, cols = model):")
|
||||
cm = pooled[variant]["confusion"]
|
||||
print(" " + "".join(f"{CLASS_NAMES[c]:>9}" for c in (-1, 0, 1)))
|
||||
for c, row in zip((-1, 0, 1), cm):
|
||||
print(f" {CLASS_NAMES[c]:<9} " + "".join(f"{v:>9}" for v in row))
|
||||
else:
|
||||
print(" no fold could be trained (too few rows); Parts C and D need OOF rows")
|
||||
|
||||
# ── Part C: routing ────────────────────────────────────────────────
|
||||
routing: List[Dict[str, Any]] = []
|
||||
if oof["full"]:
|
||||
hr("C. Confidence routing on pooled OOF (shipped recipe): keep locally vs send to DeepSeek")
|
||||
routing = routing_table(oof["full"])
|
||||
print(f" {'conf≥':>5} {'margin≥':>7} | {'kept':>5} {'share':>6} | "
|
||||
f"{'kept agree':>10} | {'hybrid agree':>12}")
|
||||
for r in routing:
|
||||
if r["margin_min"] in (0.0, 0.2, 0.4) or r["confidence_min"] in (0.7,):
|
||||
ka = "n/a" if r["kept_accuracy"] is None else f"{100 * r['kept_accuracy']:.1f}%"
|
||||
print(f" {r['confidence_min']:>5.2f} {r['margin_min']:>7.2f} | "
|
||||
f"{r['kept']:>5} {100 * r['kept_share']:5.1f}% | {ka:>10} | "
|
||||
f"{100 * r['hybrid_agreement']:11.1f}%")
|
||||
print(" (hybrid agree = kept rows judged by the model + routed rows answered "
|
||||
"by DeepSeek itself; kept share = DeepSeek calls saved)")
|
||||
|
||||
# ── Part D: DeepSeek self-agreement ────────────────────────────────
|
||||
reprobe: Dict[str, Any] = {}
|
||||
if args.reprobe > 0:
|
||||
hr(f"D. DeepSeek re-scored today on {args.reprobe} archived headlines (teacher noise)")
|
||||
rng = random.Random(args.seed)
|
||||
sample = rng.sample(rows, min(args.reprobe, len(rows)))
|
||||
ds = DeepSeekScorer(timeout=args.timeout, no_network=args.no_network)
|
||||
live = ds.score_many([r["headline"] for r in sample])
|
||||
pairs = [(r, l) for r, l in zip(sample, live) if l is not None]
|
||||
if not pairs:
|
||||
print(" no live scores (network off or every call failed)")
|
||||
else:
|
||||
old_cls = [to_class(r["xau"], thr) for r, _ in pairs]
|
||||
new_cls = [to_class(l["xau"], thr) for _, l in pairs]
|
||||
exact = sum(1 for r, l in pairs if abs(r["xau"] - l["xau"]) < 1e-9)
|
||||
blk = agreement_block(old_cls, new_cls, [r["xau"] for r, _ in pairs],
|
||||
[l["xau"] for _, l in pairs])
|
||||
print_agreement("DeepSeek today vs DeepSeek archived", blk)
|
||||
print(f" identical xau value: {fmt_pct(exact, len(pairs))}; "
|
||||
f"|diff| mean {blk['mae']:.3f}")
|
||||
same_prompt = sum(1 for r, _ in pairs if r["teacher_prompt_sha1"] == ds.prompt_sha1)
|
||||
print(f" archived rows scored with today's prompt: {same_prompt}/{len(pairs)}"
|
||||
f" (a different prompt sha1 means a different question was asked)")
|
||||
oof_by_hash = {r["hash"]: r for r in oof["full"]}
|
||||
joined = [(r, l, oof_by_hash[r["hash"]]) for r, l in pairs if r["hash"] in oof_by_hash]
|
||||
d_out: Dict[str, Any] = {"n_live": len(pairs), "identical_xau": exact,
|
||||
"self_agreement": blk, "same_prompt": same_prompt,
|
||||
"deepseek": ds.stats()}
|
||||
if joined:
|
||||
b_old = agreement_block([to_class(r["xau"], thr) for r, _, _ in joined],
|
||||
[o["cls"] for _, _, o in joined])
|
||||
b_new = agreement_block([to_class(l["xau"], thr) for _, l, _ in joined],
|
||||
[o["cls"] for _, _, o in joined])
|
||||
print_agreement(f"OOF model vs archived DeepSeek ({len(joined)} joined)", b_old)
|
||||
print_agreement("OOF model vs today's DeepSeek (same rows)", b_new)
|
||||
unstable = [(r, l, o) for r, l, o in joined
|
||||
if to_class(r["xau"], thr) != to_class(l["xau"], thr)]
|
||||
if unstable:
|
||||
m_hit = sum(1 for r, l, o in unstable
|
||||
if o["cls"] in (to_class(r["xau"], thr), to_class(l["xau"], thr)))
|
||||
print(f" rows where DeepSeek changed class: {len(unstable)}; model "
|
||||
f"matched one of the two answers on {m_hit}")
|
||||
d_out["oof_vs_archived"] = b_old
|
||||
d_out["oof_vs_today"] = b_new
|
||||
reprobe = d_out
|
||||
write_csv(RESULTS_DIR / f"archive_reprobe_{ts_tag()}.csv", [
|
||||
{"hash": r["hash"], "headline": r["headline"], "day": r["day"],
|
||||
"archived_xau": r["xau"], "today_xau": l["xau"], "today_s": l["s"],
|
||||
"archived_cls": to_class(r["xau"], thr), "today_cls": to_class(l["xau"], thr),
|
||||
"oof_model_cls": oof_by_hash.get(r["hash"], {}).get("cls"),
|
||||
"oof_model_score": oof_by_hash.get(r["hash"], {}).get("score")}
|
||||
for r, l in pairs])
|
||||
|
||||
tag = ts_tag() + f"_{args.train_scope}" + (f"_{args.model.stem}" if args.model else "")
|
||||
summary = {
|
||||
"generated_at": tag, "db": str(args.db), "threshold": thr,
|
||||
"rows": len(rows), "days": days, "rows_per_day": dict(per_day),
|
||||
"label_distribution": dict(label_dist), "model": model.meta,
|
||||
"train_scope": args.train_scope,
|
||||
"extra_labels": str(args.extra_labels) if args.extra_labels else None,
|
||||
"in_sample": {"headline_description": blk_a_full, "headline_only": blk_a_head},
|
||||
"after_training": blk_a_after, "relevance_after_training": relevance,
|
||||
"folds": fold_table, "pooled_oof": pooled, "routing": routing,
|
||||
"reprobe": reprobe,
|
||||
}
|
||||
write_json(RESULTS_DIR / f"archive_{tag}.json", summary)
|
||||
if oof["full"]:
|
||||
merged = []
|
||||
head_by_hash = {r["hash"]: r for r in oof["head"]}
|
||||
for r in oof["full"]:
|
||||
h = head_by_hash.get(r["hash"], {})
|
||||
merged.append({**r, "head_cls": h.get("cls"), "head_score": h.get("score"),
|
||||
"head_confidence": h.get("confidence")})
|
||||
write_csv(RESULTS_DIR / f"archive_oof_{tag}.csv", merged)
|
||||
print(f"\nwrote results/archive_{tag}.json"
|
||||
+ (f" and results/archive_oof_{tag}.csv" if oof["full"] else ""))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user