#!/usr/bin/env python3 """Offline replay of the agent panel on logged setups, scored against the realised outcome — so a prompt or payload version is judged on the SAME trades as its predecessor before it is enabled live. venv/bin/python scripts/panel_replay.py --db logs/signals.db --dry-run venv/bin/python scripts/panel_replay.py --db logs/signals.db --max-calls 160 --out replay.csv What it does ------------ 1. Loads every resolved setup event (analyze_signal_log's dedup: one event per identical setup, first occurrence) whose advisory outcome is RESOLVED; the outcome is `advisory_r_multiple > 0` (net R at $0.65). 2. Rebuilds what each persona would see. Rows logged by agent_panel v2 carry the exact payloads (panel_payloads_json); older rows are reconstructed from the stored indicators_json / setup_json / mtf_json / penalty_tags_json / reasons_json / warnings_json, the interned headlines and events (signals_with_news), and candles from the kept Oanda cache windows so the price path is what the scan saw. Calendar events are evaluated AS OF the signal bar (released / minutes_until), not as of today. 3. Runs the CURRENT prompts (agent_panel.PANEL_PROMPT_VERSION) through the real DeepSeek client on each payload, validates the replies with the live code path, aggregates with agent_panel.mean_verdict. 4. Scores three predictors on the same events: the new run, the logged v1 takes (stance x confidence -> pseudo-probability, when present) and the base-rate-only predictor (the class win rate for every trade): Brier score, log-loss, calibration by probability band, discrimination (mean R and win rate by band), constant-voter rate per persona, and the counterfactual books (all / skip p <= 40 / only p >= 60). Spend: 4 calls per event against the shared DeepSeek daily cap (500); `--max-calls` is a hard stop. `--dry-run` builds and writes the payloads without a single call. Read-only on the database. Caveat printed with every summary: a few dozen outcomes from one regime can reject a pathological prompt (constant voting, narrative anchoring); they cannot prove accuracy. The live scorecard at 100 and 300 outcomes (analyze_signal_log.py --stored-candles) is the real test. """ from __future__ import annotations import argparse import glob import json import math import os import sys from datetime import datetime, timedelta, timezone from types import SimpleNamespace from typing import Any, Dict, List, Optional, Sequence, Tuple import pandas as pd ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if ROOT not in sys.path: sys.path.insert(0, ROOT) import agent_panel as ap # noqa: E402 from analyze_signal_log import (cluster_setup_rows, load_rows, # noqa: E402 open_readonly) from config import SignalStrength, Timeframe # noqa: E402 from news_engine import (EconomicEvent, FundamentalContext, # noqa: E402 NewsSentiment) from signal_aggregator import SignalReport, TradeSetup # noqa: E402 from signal_log import read_news # noqa: E402 PERSONAS = ap.ALL_PERSONAS # research default: score every persona CACHE_DIR = os.path.join(ROOT, "backtest", "cache") _TF = {tf.value: tf for tf in Timeframe} # ── Reconstruction ──────────────────────────────────────────────────── class _AsOfEvent(EconomicEvent): """EconomicEvent whose `is_upcoming` / `minutes_until` are evaluated relative to the signal bar, not the wall clock.""" def __init__(self, as_of: datetime, **kw): super().__init__(**kw) self._as_of = as_of @property def is_upcoming(self) -> bool: # type: ignore[override] return self.time > self._as_of @property def minutes_until(self) -> float: # type: ignore[override] return (self.time - self._as_of).total_seconds() / 60.0 def _dt(s: Optional[str]) -> Optional[datetime]: if not s: return None try: d = datetime.fromisoformat(str(s).replace("Z", "+00:00")) except ValueError: return None return d if d.tzinfo else d.replace(tzinfo=timezone.utc) def _loads(s: Any, default: Any) -> Any: if s in (None, "", "null"): return default try: return json.loads(s) except (TypeError, ValueError): return default def _analysis(tf: str, blob: Dict[str, Any]) -> SimpleNamespace: inds = {} for name, d in (blob.get("indicators") or {}).items(): try: sig = SignalStrength[d.get("signal", "NEUTRAL")] except KeyError: sig = SignalStrength.NEUTRAL inds[name] = SimpleNamespace( signal=sig, confidence=float(d.get("confidence") or 0.0), weight=float(d.get("weight") or 0.0), value=d.get("value"), description=d.get("description") or "") return SimpleNamespace(timeframe=tf, timestamp=blob.get("timestamp"), key_levels=dict(blob.get("key_levels") or {}), indicators=inds) def _context_from_row(events_json: Optional[str], headlines_json: Optional[str], as_of: datetime) -> FundamentalContext: ctx = FundamentalContext() ev = _loads(events_json, {}) or {} for key in ("events_today", "events_upcoming_1h"): out = [] for e in ev.get(key) or []: t = _dt(e.get("time")) if t is None: continue out.append(_AsOfEvent(as_of, time=t, currency=e.get("currency") or "", impact=e.get("impact") or "", event=e.get("event") or "", forecast=e.get("forecast"), previous=e.get("previous"), actual=e.get("actual"))) setattr(ctx, key, out) ctx.news_sentiment = dict(ev.get("sentiment_by_currency") or {}) ctx.calendar_available = bool(ev.get("calendar_available", False)) for h in _loads(headlines_json, []) or []: t = _dt(h.get("timestamp")) or as_of ctx.news_headlines.append(NewsSentiment( headline=h.get("headline") or "", source=h.get("source") or "", timestamp=t, currencies_mentioned=list(h.get("currencies") or []), sentiment_score=float(h.get("score") or 0.0), keywords=list(h.get("keywords") or []), sentiment_by_currency={k: float(v) for k, v in (h.get("score_by_currency") or {}).items() if v is not None}, score_source=h.get("score_source") or "keyword")) return ctx _frame_cache: Dict[str, pd.DataFrame] = {} _have_cols: set = set() def _col(name: str) -> str: """Column reference that tolerates a database logged before the column existed (a read-only copy is never migrated).""" return name if name in _have_cols else f"NULL AS {name}" def _cached_frame(tf: str) -> Optional[pd.DataFrame]: """Newest kept Oanda cache window for `tf` (raw bars, UTC index).""" if tf in _frame_cache: return _frame_cache[tf] files = sorted(glob.glob(os.path.join(CACHE_DIR, f"oanda_XAU_USD_{tf}_*.parquet")), key=lambda p: p.rsplit("_", 1)[-1]) df = None if files: try: df = pd.read_parquet(files[-1]) if df.index.tz is None: df.index = df.index.tz_localize("UTC") except Exception: df = None _frame_cache[tf] = df return df def _frames_as_of(tfs: Sequence[str], bar_ts: datetime) -> Dict[Timeframe, pd.DataFrame]: out: Dict[Timeframe, pd.DataFrame] = {} for tf in tfs: df = _cached_frame(tf) tfe = _TF.get(tf) if df is None or tfe is None: continue sl = df[df.index <= pd.Timestamp(bar_ts)] if len(sl) >= 3: out[tfe] = sl return out def build_report(con, rowid: int) -> Tuple[Optional[SignalReport], Optional[FundamentalContext], Dict[Timeframe, pd.DataFrame], Optional[Dict[str, Any]]]: """(report, context, frames, stored_payloads) for one logged row.""" row = con.execute( "SELECT symbol, timeframe, " + _col("bar_ts") + ", logged_at, signal, score," " confidence, quality, price, market_regime, volatility_regime, mtf_json," " setup_json, reasons_json, warnings_json, event_risk, event_description," " news_sentiment, indicators_json, " + _col("penalty_tags_json") + ", " + _col("exhaustion_count") + ", " + _col("penalty_total") + ", " + _col("momentum_fading") + ", " + _col("panel_payloads_json") + " FROM signals WHERE id=?", (rowid,)).fetchone() if row is None: return None, None, {}, None (symbol, tf, bar_ts, logged_at, signal, score, conf, quality, price, regime, vol, mtf_json, setup_json, reasons_json, warnings_json, event_risk, event_desc, news_sent, ind_json, pen_json, exh, pen_total, mom, payloads_json) = row inds = _loads(ind_json, {}) or {} bar_label = bar_ts or (inds.get(tf) or {}).get("timestamp") as_of_bar = _dt(bar_label) or _dt(logged_at) if as_of_bar is None: return None, None, {}, None secs = ap._TF_SECONDS.get(tf, 900) as_of = as_of_bar + timedelta(seconds=secs) # the bar closed report = SignalReport(symbol=symbol, display_name="XAU/USD", timestamp=str(bar_label or logged_at)) try: report.final_signal = SignalStrength[signal] except KeyError: report.final_signal = SignalStrength.NEUTRAL report.final_score = float(score or 0.0) report.final_confidence = float(conf or 0.0) report.signal_quality = quality or "" report.current_price = float(price or 0.0) report.market_regime = regime or "" report.volatility_regime = vol or "" mtf = _loads(mtf_json, {}) or {} report.mtf_agreement = bool(mtf.get("agreement", False)) for k, v in (mtf.get("signals") or {}).items(): try: report.mtf_signals[k] = SignalStrength[v] except KeyError: report.mtf_signals[k] = SignalStrength.NEUTRAL report.mtf_scores = {k: float(v or 0.0) for k, v in (mtf.get("scores") or {}).items()} reasons = _loads(reasons_json, {}) or {} report.bullish_reasons = list(reasons.get("bullish") or []) report.bearish_reasons = list(reasons.get("bearish") or []) report.warnings = list(_loads(warnings_json, []) or []) report.event_risk = event_risk or "clear" report.event_description = event_desc or "" report.news_sentiment = float(news_sent or 0.0) report.exhaustion_count = int(exh or 0) report.penalty_tags = {str(k): float(v or 0.0) for k, v in (_loads(pen_json, {}) or {}).items()} report.penalty_total = float(pen_total or 0.0) report.momentum_fading = bool(mom) setup = _loads(setup_json, None) if setup: report.trade_setup = TradeSetup( pair=symbol, display_name="XAU/USD", direction=setup.get("direction"), entry_price=float(setup.get("entry") or 0.0), stop_loss=float(setup.get("stop_loss") or 0.0), take_profit_1=float(setup.get("take_profit_1") or 0.0), take_profit_2=float(setup.get("take_profit_2") or 0.0), risk_reward=float(setup.get("risk_reward") or 0.0), risk_pips=float(setup.get("risk_pips") or 0.0), reward_pips=float(setup.get("reward_pips") or 0.0), pip_value=0.10, strategy=setup.get("strategy") or "GENERIC", setup_pattern=setup.get("setup_pattern"), round_trip_cost=setup.get("round_trip_cost"), trail_arm_r=setup.get("trail_arm_r"), trail_dist_r=setup.get("trail_dist_r")) if tf in inds: report.primary_analysis = _analysis(tf, inds[tf]) report.confirmation_analyses = [_analysis(k, v) for k, v in inds.items() if k != tf] events_json, headlines_json = read_news(con, rowid) context = _context_from_row(events_json, headlines_json, as_of) frames = _frames_as_of([tf] + [k for k in inds if k != tf], as_of_bar) return report, context, frames, _loads(payloads_json, None) def stored_personas(stored: Any) -> List[str]: """The requested personas whose exact payload the row logged.""" if not isinstance(stored, dict): return [] return [p for p in PERSONAS if isinstance(stored.get(p), dict)] def build_jobs(report: SignalReport, context: FundamentalContext, frames: Dict[Timeframe, pd.DataFrame], stored: Optional[Dict[str, Any]], settings: Dict[str, Any]) -> Tuple[Tuple[str, str, Dict[str, Any]], ...]: """Persona jobs on the CURRENT prompts. A persona whose exact payload the row logged replays that payload; every other persona is rebuilt through the live payload builders. Per persona, not all-or-nothing: the live panel stores only the personas it ran (macro alone since 2026-09-17), so "all four or none" never used a stored payload.""" systems = {"macro": ap._MACRO_SYSTEM, "technical": ap._TECH_SYSTEM, "risk": ap._RISK_SYSTEM, "contrarian": ap._CONTRARIAN_SYSTEM} kept = {p: stored[p] for p in stored_personas(stored)} if len(kept) == len(PERSONAS): return tuple((p, systems[p], kept[p]) for p in PERSONAS) candles: Dict[str, Any] = {} analyses = [report.primary_analysis] + list(report.confirmation_analyses) counts = [int(settings.get("candle_bars_primary", ap.DEFAULT_CANDLE_BARS_PRIMARY))] + \ [int(settings.get("candle_bars_confirm", ap.DEFAULT_CANDLE_BARS_CONFIRM))] * \ len(report.confirmation_analyses) for analysis, n in zip(analyses, counts): if analysis is None: continue atr = ap._finite(getattr((analysis.indicators or {}).get("atr"), "value", None)) df = frames.get(_TF.get(analysis.timeframe)) if atr and atr > 0 and df is not None and len(df) >= 2: candles[analysis.timeframe] = ap._normalize_candles(df, atr, n) ctx = ap._price_context(report, frames, context) rebuilt = ap._panel_jobs(report, context, candles or None, ctx, PERSONAS) return tuple((p, system, kept.get(p, payload)) for p, system, payload in rebuilt) # ── Scoring ─────────────────────────────────────────────────────────── def _brier(ps: Sequence[float], ys: Sequence[int]) -> float: return sum((p / 100.0 - y) ** 2 for p, y in zip(ps, ys)) / len(ps) def _logloss(ps: Sequence[float], ys: Sequence[int]) -> float: eps = 1e-3 tot = 0.0 for p, y in zip(ps, ys): q = min(1 - eps, max(eps, p / 100.0)) tot += -(y * math.log(q) + (1 - y) * math.log(1 - q)) return tot / len(ps) def score_predictor(name: str, ps: Sequence[Optional[float]], rs: Sequence[float]) -> Dict[str, Any]: pairs = [(p, r) for p, r in zip(ps, rs) if p is not None] if not pairs: return {"predictor": name, "n": 0} P = [p for p, _ in pairs]; R = [r for _, r in pairs]; Y = [1 if r > 0 else 0 for r in R] bands = {} for label, lo, hi in (("<=40", -1, ap.P_OPPOSE), ("40-60", ap.P_OPPOSE, ap.P_SUPPORT), (">=60", ap.P_SUPPORT, 101)): sel = [(p, r) for p, r in pairs if (lo < p <= hi if label == "<=40" else lo < p < hi if label == "40-60" else p >= lo)] if sel: bands[label] = {"n": len(sel), "mean_p": round(sum(p for p, _ in sel) / len(sel), 1), "win_rate": round(100 * sum(1 for _, r in sel if r > 0) / len(sel), 1), "mean_r": round(sum(r for _, r in sel) / len(sel), 3), "sum_r": round(sum(r for _, r in sel), 2)} else: bands[label] = {"n": 0} stances = [ap.derive_stance(p) for p in P] modal = max(set(stances), key=stances.count) return { "predictor": name, "n": len(P), "brier": round(_brier(P, Y), 4), "logloss": round(_logloss(P, Y), 4), "mean_p": round(sum(P) / len(P), 1), "win_rate": round(100 * sum(Y) / len(Y), 1), "bands": bands, "modal_stance": modal, "constant_voter_rate": round(100 * stances.count(modal) / len(stances), 1), "sum_r_all": round(sum(R), 2), "sum_r_skip_oppose": round(sum(r for p, r in pairs if p > ap.P_OPPOSE), 2), "sum_r_support_only": round(sum(r for p, r in pairs if p >= ap.P_SUPPORT), 2), } def print_summary(results: List[Dict[str, Any]]) -> None: print("\n== Predictor scorecard (outcome = advisory R > 0 at $0.65) ==") print(f"{'predictor':22s} {'n':>3s} {'Brier':>6s} {'logloss':>7s} {'mean p':>6s} {'win%':>5s} " f"{'modal':>8s} {'const%':>6s} {'R all':>7s} {'R skip<=40':>10s} {'R >=60 only':>11s}") for r in results: if r.get("n", 0) == 0: print(f"{r['predictor']:22s} 0"); continue print(f"{r['predictor']:22s} {r['n']:3d} {r['brier']:6.3f} {r['logloss']:7.3f} {r['mean_p']:6.1f} " f"{r['win_rate']:5.1f} {r['modal_stance']:>8s} {r['constant_voter_rate']:6.1f} " f"{r['sum_r_all']:+7.2f} {r['sum_r_skip_oppose']:+10.2f} {r['sum_r_support_only']:+11.2f}") print("\n calibration bands (n / mean p / observed win% / mean R):") for r in results: if r.get("n", 0) == 0: continue cells = " ".join(f"{k}: {v['n']}/{v.get('mean_p', '-')}/{v.get('win_rate', '-')}/{v.get('mean_r', '-')}" if v["n"] else f"{k}: 0" for k, v in r["bands"].items()) print(f" {r['predictor']:22s} {cells}") print("\nCaveat: a few dozen outcomes from one regime can reject a pathological prompt " "(constant voting, narrative anchoring); they cannot prove accuracy.") # ── Main ────────────────────────────────────────────────────────────── def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--db", default=os.path.join(ROOT, "logs", "signals.db")) parser.add_argument("--limit", type=int, default=0, help="max events (0 = all)") parser.add_argument("--max-calls", type=int, default=200, help="hard stop on LLM requests (4 per event)") parser.add_argument("--dry-run", action="store_true", help="build payloads only; no LLM call") parser.add_argument("--out", default=None, help="per-event CSV (and .payloads.json on --dry-run)") parser.add_argument("--include-pending", action="store_true", help="also build payloads for unresolved setups (dry-run only)") parser.add_argument("--personas", default=",".join(ap.ALL_PERSONAS), help="comma-separated personas to run (default: all four, so every " "voice is scored; the live default is agent_panel.personas = macro)") args = parser.parse_args(argv) global PERSONAS PERSONAS = ap.normalize_personas([p for p in args.personas.split(",") if p.strip()]) rows = load_rows(args.db) events = cluster_setup_rows(rows) con = open_readonly(args.db) global _have_cols _have_cols = {r[1] for r in con.execute("PRAGMA table_info(signals)")} settings = {"candle_bars_primary": ap.DEFAULT_CANDLE_BARS_PRIMARY, "candle_bars_confirm": ap.DEFAULT_CANDLE_BARS_CONFIRM} picked = [] for ev in events: res = next((r for r in ev.rows if r.advisory_status == "RESOLVED" and r.advisory_r is not None), None) if res is None and not (args.dry_run and args.include_pending): continue picked.append((ev, res)) if args.limit: picked = picked[:args.limit] print(f"events: {len(events)} total, {len(picked)} selected " f"({'dry run' if args.dry_run else 'live calls, max ' + str(args.max_calls)}), " f"prompt version {ap.PANEL_PROMPT_VERSION}, personas {','.join(PERSONAS)}") client = None if args.dry_run else ap.get_llm_client() calls = 0 per_event: List[Dict[str, Any]] = [] payload_dump: Dict[str, Any] = {} for ev, res in picked: rep = ev.representative # Prefer a member row that already carries the v2 payloads. rowid = rep.id for r in ev.rows: has = (con.execute("SELECT panel_payloads_json IS NOT NULL FROM signals" " WHERE id=?", (r.id,)).fetchone() if "panel_payloads_json" in _have_cols else None) if has and has[0]: rowid = r.id break report, context, frames, stored = build_report(con, rowid) if report is None or report.trade_setup is None: continue jobs = build_jobs(report, context, frames, stored, settings) ctx = next((payload.get("context") for _, _, payload in jobs if payload.get("context")), {}) or {} base_p = (ctx.get("base_rate") or {}).get("win_rate") # logged v1 takes (any member row) -> pseudo-probabilities v1 = {} for r in ev.rows: takes = con.execute("SELECT panel_takes_json, " + _col("panel_prompt_version") + " FROM signals WHERE id=?", (r.id,)).fetchone() if takes and takes[0]: for t in _loads(takes[0], []) or []: p = t.get("p_win") if p is None and t.get("stance") in ap._VALID_STANCES: p = ap.pseudo_probability(t["stance"], float(t.get("confidence") or 0)) if p is not None: v1[t.get("persona")] = float(p) v1["_version"] = takes[1] or "v1-logged" break entry: Dict[str, Any] = { "event_row": rep.id, "bar": report.timestamp, "tf": rep.timeframe, "direction": report.trade_setup.direction, "grade": rep.quality, "strategy": report.trade_setup.strategy, # which personas replayed the exact logged payload ("" = all rebuilt) "R": res.advisory_r if res else None, "stored_payloads": ",".join(stored_personas(stored)), "base_p": base_p, "prior_move_4h_atr": ctx.get("prior_move_4h_atr"), "move_from_start_atr": ctx.get("move_from_24h_start_atr"), "big_candle_last3": ctx.get("big_candle_last3"), } for p in PERSONAS: entry[f"v1_{p}"] = v1.get(p) v1_ps = [v1[p] for p in PERSONAS if p in v1] entry["v1_mean"] = round(sum(v1_ps) / len(v1_ps), 1) if len(v1_ps) >= min(ap.MEAN_QUORUM, len(v1_ps) or 1) and v1_ps else None if args.dry_run: payload_dump[str(rep.id)] = {name: payload for name, _, payload in jobs} else: takes = [] for name, system, payload in jobs: if calls >= args.max_calls: break calls += 1 reply = client.chat_json(system, payload) take = ap._validate_take(name, reply) if take is not None: takes.append(take) entry[f"new_{name}"] = take.p_win entry[f"new_{name}_for"] = take.reason_for entry[f"new_{name}_against"] = take.reason_against v = ap.mean_verdict(takes, quorum=min(ap.MEAN_QUORUM, len(jobs))) entry["new_mean"] = v.probability if v else None entry["new_verdict"] = v.verdict if v else None print(f" #{rep.id:<5d} {report.timestamp[:16]} {rep.timeframe:5s} " f"{report.trade_setup.direction:4s} {rep.quality:2s} R={entry['R']:+.2f} " f"base={base_p} new_mean={entry['new_mean']} " f"({', '.join(f'{t.persona[:4]} {t.p_win:.0f}' for t in takes)}) " f"v1_mean={entry['v1_mean']}") per_event.append(entry) if not args.dry_run and calls >= args.max_calls: print(f" stopped at --max-calls {args.max_calls}") break con.close() if args.dry_run: print(f"built payloads for {len(per_event)} events") if args.out: with open(args.out.rsplit('.', 1)[0] + ".payloads.json", "w") as fh: json.dump(payload_dump, fh, indent=1, sort_keys=True) print(f"wrote {args.out.rsplit('.', 1)[0]}.payloads.json") scored = [e for e in per_event if e["R"] is not None] if scored: rs = [e["R"] for e in scored] results = [score_predictor("base-rate only", [e["base_p"] if e["base_p"] is not None else None for e in scored], rs), score_predictor("v1 logged (mean)", [e["v1_mean"] for e in scored], rs)] for p in PERSONAS: results.append(score_predictor(f"v1 logged {p}", [e.get(f"v1_{p}") for e in scored], rs)) if not args.dry_run: results.append(score_predictor(f"NEW {ap.PANEL_PROMPT_VERSION} (mean)", [e.get("new_mean") for e in scored], rs)) for p in PERSONAS: results.append(score_predictor(f"NEW {p}", [e.get(f"new_{p}") for e in scored], rs)) print_summary(results) if args.out and per_event: pd.DataFrame(per_event).to_csv(args.out, index=False) print(f"wrote {args.out} ({len(per_event)} events)") return 0 if __name__ == "__main__": sys.exit(main())