#!/usr/bin/env python3 """ backfill_signal_outcomes.py — seed and advance advisory outcomes for every logged trade setup (not only panel rows), one per advisory event. Pass A (offline, no network): assign setup_key / advisory_event_id to all legacy setup rows, mark same-setup re-logs within 60 min as DUPLICATE, and seed a PENDING outcome on each canonical row that has none. Existing (panel-seeded) outcomes are never overwritten. Pass B: advance PENDING rows from the closed-candle streams already stored in the DB (union across every snapshot of the timeframe — no provider). Pass C (--reseed-unresolvable): bounded re-seed of continuity-broken rows, then Pass B again. Safe beside the live writers: short per-row transactions with a busy timeout. Read-only preview with --dry-run. python scripts/backfill_signal_outcomes.py --dry-run python scripts/backfill_signal_outcomes.py python scripts/backfill_signal_outcomes.py --reseed-unresolvable """ import argparse import os import sqlite3 import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import app_config # noqa: E402 from signal_log import SignalLogger # noqa: E402 def _status_counts(con: sqlite3.Connection) -> dict: return dict(con.execute( "SELECT COALESCE(advisory_outcome_status, 'NULL'), COUNT(*)" " FROM signals WHERE setup_json IS NOT NULL AND setup_json != 'null'" " GROUP BY 1").fetchall()) def main(argv=None) -> int: ap = argparse.ArgumentParser(description=__doc__.splitlines()[1]) ap.add_argument("--db", default=None, help="defaults to the signal_log.path setting in config.json") ap.add_argument("--dry-run", action="store_true") ap.add_argument("--reseed-unresolvable", action="store_true") ap.add_argument("--symbol", default="XAU/USD") args = ap.parse_args(argv) path = args.db or app_config.signal_log_path() if not os.path.exists(path): print(f"no database at {path}") return 1 if args.dry_run: con = sqlite3.connect(f"file:{os.path.abspath(path)}?mode=ro", uri=True) print("setup rows by outcome status:", _status_counts(con)) have = {r[1] for r in con.execute("PRAGMA table_info(signals)")} if "setup_key" in have: n_keyless = con.execute( "SELECT COUNT(*) FROM signals WHERE setup_json IS NOT NULL" " AND setup_json != 'null' AND setup_key IS NULL").fetchone()[0] print(f"setup rows without setup_key: {n_keyless}") else: print("setup_key column not present yet (schema migrates on" " the first non-dry run)") con.close() return 0 lg = SignalLogger(path) try: print("before:", _status_counts(lg._conn)) print("pass A (seed/dedup):", lg.backfill_seeds()) total = 0 for _ in range(50): n = lg.resolve_from_store(args.symbol) total += n if n == 0: break print(f"pass B (advance from stored bars): {total} rows changed") if args.reseed_unresolvable: reseeded = lg.reseed_unresolvable() print(f"pass C (re-seed): {len(reseeded)} rows re-seeded") for _ in range(50): if lg.resolve_from_store(args.symbol) == 0: break print("after:", _status_counts(lg._conn)) finally: lg.close() return 0 if __name__ == "__main__": sys.exit(main())