news corpus for a local scorer + agent panel v2 + dashboard read-only (2026-09-17); books byte-identical
Agent panel v2: personas return p_win anchored to the book base rates with a code-computed decision context; verdict = equal-weight mean (verdict_source "mean"); only the macro persona runs live (agent_panel.personas ["macro"]); prompt version and exact payloads logged; offline replay harness scripts/panel_replay.py; base-rate table agent_panel_base_rates.json (scripts/build_panel_base_rates.py). Backtester `strategy` column now carries the generator's own stamp (label-only; five reference books re-pinned, all 62 other columns identical). News corpus (after evaluating the owner's ML-NEWS scikit model against DeepSeek: golden set 39/78 vs 78/78, out-of-sample agreement 64% against a 90% self-agreement ceiling; verdict: fallback only, not a replacement): - NewsSentiment.xau_score + archive column xau_score keep DeepSeek's gold score for every headline (930 of the first 1,546 DeepSeek rows had none); an existing newsdata.db gains the column in place on first open - headlines naming no tracked currency are scored and archived (currencies_json []) but never returned to the platform - _apply_llm_sentiment sends every new headline in its cycle, 25 per call; a failed call ends the cycle's sending Dashboard publisher never writes a database: main() calls app_config.set_db_writes_for_process(False), honoured by signal_log_enabled() and news_archive_enabled() (owner decision: the bot is the single writer). Test suite: the hermetic app config now points both writers at a suite-private temp dir — the RSS fixture tests were writing fixture headlines into the real logs/newsdata.db — with a Module 32 test pinning it. 529 tests. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Harwinder Sekhon
@@ -794,6 +794,145 @@ def persona_scorecard(events: Sequence[SignalEvent], db_path: str) -> List[dict]
|
||||
for (p, s), v in sorted(agg.items())]
|
||||
|
||||
|
||||
# ── Probability scorecard (agent panel v2, 2026-09-17) ───────────────
|
||||
#
|
||||
# Scores the panel as a PREDICTOR of the trade closing profitably, per
|
||||
# prompt version: Brier score and log-loss per persona and for the mean,
|
||||
# calibration by probability band, discrimination (win rate / mean R by
|
||||
# band), the constant-voter rate, rank correlation of p with R, and the
|
||||
# counterfactual books. v1 rows (stance + confidence) are mapped onto the
|
||||
# same 0-100 probability scale with agent_panel.pseudo_probability so the
|
||||
# two eras are comparable; nothing here is a decision (owner rule: no
|
||||
# persona-weight tuning before ~300 graded outcomes).
|
||||
|
||||
def _p_bands(pairs: Sequence[Tuple[float, float]]) -> Dict[str, dict]:
|
||||
from agent_panel import P_OPPOSE, P_SUPPORT
|
||||
out = {}
|
||||
for label, pick in (("<=40", lambda p: p <= P_OPPOSE),
|
||||
("40-60", lambda p: P_OPPOSE < p < P_SUPPORT),
|
||||
(">=60", lambda p: p >= P_SUPPORT)):
|
||||
sel = [(p, r) for p, r in pairs if pick(p)]
|
||||
out[label] = ({"n": len(sel),
|
||||
"mean_p": sum(p for p, _ in sel) / len(sel),
|
||||
"win_rate": sum(1 for _, r in sel if r > 0) / len(sel),
|
||||
"mean_r": sum(r for _, r in sel) / len(sel),
|
||||
"sum_r": sum(r for _, r in sel)}
|
||||
if sel else {"n": 0})
|
||||
return out
|
||||
|
||||
|
||||
def _score_pairs(pairs: Sequence[Tuple[float, float]]) -> Optional[dict]:
|
||||
from agent_panel import P_OPPOSE, P_SUPPORT, derive_stance
|
||||
if not pairs:
|
||||
return None
|
||||
ys = [1.0 if r > 0 else 0.0 for _, r in pairs]
|
||||
ps = [p / 100.0 for p, _ in pairs]
|
||||
eps = 1e-3
|
||||
brier = sum((p - y) ** 2 for p, y in zip(ps, ys)) / len(ps)
|
||||
ll = -sum(y * math.log(min(1 - eps, max(eps, p)))
|
||||
+ (1 - y) * math.log(min(1 - eps, max(eps, 1 - p)))
|
||||
for p, y in zip(ps, ys)) / len(ps)
|
||||
stances = [derive_stance(p) for p, _ in pairs]
|
||||
modal = max(set(stances), key=stances.count)
|
||||
rho = None
|
||||
if len(pairs) >= 5:
|
||||
a = pd.Series([p for p, _ in pairs]).rank()
|
||||
b = pd.Series([r for _, r in pairs]).rank()
|
||||
c = a.corr(b)
|
||||
rho = None if c != c else float(c)
|
||||
return {"n": len(pairs), "brier": brier, "logloss": ll,
|
||||
"mean_p": sum(p for p, _ in pairs) / len(pairs),
|
||||
"win_rate": sum(ys) / len(ys),
|
||||
"modal": modal, "constant_rate": stances.count(modal) / len(stances),
|
||||
"rho_p_r": rho, "bands": _p_bands(pairs),
|
||||
"sum_all": sum(r for _, r in pairs),
|
||||
"sum_skip_oppose": sum(r for p, r in pairs if p > P_OPPOSE),
|
||||
"sum_support_only": sum(r for p, r in pairs if p >= P_SUPPORT)}
|
||||
|
||||
|
||||
def probability_scorecard(events: Sequence[SignalEvent], db_path: str
|
||||
) -> Dict[str, Dict[str, Optional[dict]]]:
|
||||
"""{prompt_version: {"mean": stats, "<persona>": stats, "base_rate": stats}}
|
||||
over resolved events (one vote per event, first row with a panel)."""
|
||||
from agent_panel import pseudo_probability
|
||||
by_row = _row_index(events)
|
||||
con = open_readonly(db_path)
|
||||
have = {r[1] for r in con.execute("PRAGMA table_info(signals)")}
|
||||
cols = ["id", "panel_takes_json"]
|
||||
for c in ("panel_probs_json", "panel_prompt_version", "panel_probability",
|
||||
"panel_context_json"):
|
||||
cols.append(c if c in have else f"NULL AS {c}")
|
||||
per_version: Dict[str, Dict[str, List[Tuple[float, float]]]] = {}
|
||||
seen: set = set()
|
||||
try:
|
||||
for rid, takes_json, probs_json, version, p_mean, ctx_json in con.execute(
|
||||
f"SELECT {', '.join(cols)} FROM signals"
|
||||
" WHERE panel_takes_json IS NOT NULL ORDER BY id"):
|
||||
ev = by_row.get(rid)
|
||||
if ev is None or ev.r_headline is None:
|
||||
continue
|
||||
ekey = ev.rows[0].id if ev.rows else rid
|
||||
if ekey in seen:
|
||||
continue
|
||||
seen.add(ekey)
|
||||
r = ev.r_headline
|
||||
version = version or "v1"
|
||||
bucket = per_version.setdefault(version, {})
|
||||
probs: Dict[str, float] = {}
|
||||
try:
|
||||
source = json.loads(probs_json) if probs_json else json.loads(takes_json)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
for t in source or []:
|
||||
p = t.get("p_win")
|
||||
if p is None and t.get("stance") in ("support", "oppose", "neutral"):
|
||||
p = pseudo_probability(t["stance"], float(t.get("confidence") or 0.0))
|
||||
if p is not None:
|
||||
probs[str(t.get("persona"))] = float(p)
|
||||
bucket.setdefault(str(t.get("persona")), []).append((float(p), r))
|
||||
if p_mean is not None:
|
||||
bucket.setdefault("mean", []).append((float(p_mean), r))
|
||||
elif len(probs) >= 3:
|
||||
bucket.setdefault("mean", []).append(
|
||||
(sum(probs.values()) / len(probs), r))
|
||||
try:
|
||||
base = (json.loads(ctx_json) if ctx_json else {}).get("base_rate") or {}
|
||||
if base.get("win_rate") is not None:
|
||||
bucket.setdefault("base_rate", []).append((float(base["win_rate"]), r))
|
||||
except (TypeError, ValueError, AttributeError):
|
||||
pass
|
||||
finally:
|
||||
con.close()
|
||||
return {v: {k: _score_pairs(pairs) for k, pairs in sorted(b.items())}
|
||||
for v, b in sorted(per_version.items())}
|
||||
|
||||
|
||||
def print_probability_scorecard(card: Dict[str, Dict[str, Optional[dict]]]) -> None:
|
||||
print("\n── Probability scorecard (Brier: lower is better; 0.25 = coin flip) " + "─" * 6)
|
||||
if not card:
|
||||
print(" no resolved events with a panel yet")
|
||||
return
|
||||
for version, preds in card.items():
|
||||
print(f" prompt version {version}")
|
||||
print(f" {'predictor':12s} {'n':>3s} {'Brier':>6s} {'logloss':>7s} {'mean p':>6s} "
|
||||
f"{'win%':>5s} {'modal':>8s} {'const%':>6s} {'rho(p,R)':>8s} "
|
||||
f"{'R all':>7s} {'skip<=40':>8s} {'>=60only':>8s}")
|
||||
for name, st in preds.items():
|
||||
if not st:
|
||||
continue
|
||||
rho = f"{st['rho_p_r']:+8.2f}" if st["rho_p_r"] is not None else " -"
|
||||
print(f" {name:12s} {st['n']:3d} {st['brier']:6.3f} {st['logloss']:7.3f} "
|
||||
f"{st['mean_p']:6.1f} {100 * st['win_rate']:5.0f} {st['modal']:>8s} "
|
||||
f"{100 * st['constant_rate']:6.0f} {rho} {st['sum_all']:+7.2f} "
|
||||
f"{st['sum_skip_oppose']:+8.2f} {st['sum_support_only']:+8.2f}")
|
||||
mean = preds.get("mean")
|
||||
if mean:
|
||||
cells = " ".join(
|
||||
f"{k}: n={v['n']} p={v['mean_p']:.0f} win={100 * v['win_rate']:.0f}% R={v['mean_r']:+.2f}"
|
||||
if v["n"] else f"{k}: n=0" for k, v in mean["bands"].items())
|
||||
print(f" calibration of the mean: {cells}")
|
||||
|
||||
|
||||
# ── News sentiment lead/lag ───────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -1309,6 +1448,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
|
||||
for p in persona_scorecard(events, args.db):
|
||||
print(f" {p['persona']:11} {p['stance']:8} n={p['n']:3d} "
|
||||
f"mean {p['mean_r']:+.3f}R win {100 * p['win_rate']:.0f}%")
|
||||
print_probability_scorecard(probability_scorecard(events, args.db))
|
||||
if args.news:
|
||||
print_news(news_lead_lag(rows, streams),
|
||||
news_extremes(rows, streams),
|
||||
|
||||
Reference in New Issue
Block a user