Harwinder Sekhon
985ec83ef3
Zero callers anywhere (tests included), verified by grep/AST: - dashboard_publisher.DashboardPublisher.publish_all (run_cycle inlines both publishes) - signal_log.SignalLogger.advance_with_bars (the backfill script uses backfill_seeds / resolve_from_store / reseed_unresolvable) and the never-read SignalLogger.path property plus its write-only self._path - llm_client.reset_llm_client (tests reset the singleton by assignment) - market_outlook._last_run_ts, residue of the old "now - last < 3600" dedup; _last_run_hour (the UTC hour bucket) is the live key - news_engine FundamentalContext.event_impact_on_pairs: written once, read by nothing; its only feeder NewsEngine._get_affected_pairs and that helper's unit test go with it - position_sizing.LIVE_DEPTH_BARS (referenced only by its own docstring, which now states the depths inline), the unused Iterable import and the unused atr_ok parameter of _fill_geometry - analyze_signal_log: unused `field` import and the unused `rows` parameter of persona_scorecard / news_outcome_buckets (both read the DB directly) The two backfill/label scripts' --db help text now names the config.json setting it actually falls back to. Suite: 463 passed, 0 failed (one test removed with its subject). Claude-Session: https://claude.ai/code/session_0156YhogLytzj9P2iH1skGbL
116 lines
4.5 KiB
Python
Executable File
116 lines
4.5 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
label_scan_outcomes.py — batch driver for the scan_outcomes table.
|
|
|
|
Labels model-free forward returns (1/4/24 h, $ and ATR-normalized, plus
|
|
MFE/MAE, a 3xATR burst flag and a completeness flag) for every logged scan
|
|
whose 24 h window has closed, from the closed-candle streams already
|
|
stored in the DB. No provider, no network.
|
|
|
|
The live loggers do the same work opportunistically at the end of every
|
|
scan (time-boxed to ~2 s); this script is the catch-up pass for a backlog
|
|
— a freshly imported database, or rows whose bars only arrived later.
|
|
|
|
Idempotent: a row is rewritten only when its numbers change, so a settled
|
|
database costs one read pass. Safe beside the live writers (short per-row
|
|
transactions with a busy timeout). Read-only preview with --dry-run.
|
|
|
|
python scripts/label_scan_outcomes.py --dry-run
|
|
python scripts/label_scan_outcomes.py
|
|
python scripts/label_scan_outcomes.py --db logs/signals.db --budget 5000
|
|
"""
|
|
import argparse
|
|
import os
|
|
import sqlite3
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
import app_config # noqa: E402
|
|
from signal_log import SCAN_LABEL_MIN_AGE_H, SignalLogger # noqa: E402
|
|
|
|
CHUNK = 2000
|
|
|
|
|
|
def _counts(con: sqlite3.Connection, symbol: str,
|
|
labelled: bool = True) -> dict:
|
|
"""Candidate / labelled / complete / pending counts (read-only).
|
|
|
|
The age filter uses logged_at (always an ISO string) as a proxy for
|
|
the bar close — the labeller itself parses the real bar label."""
|
|
scans = ("SELECT COUNT(*) FROM signals s WHERE s.symbol = ?"
|
|
" AND s.indicators_json IS NOT NULL AND s.price > 0")
|
|
due = (scans + " AND s.logged_at <"
|
|
f" datetime('now', '-{int(SCAN_LABEL_MIN_AGE_H)} hours')")
|
|
out = {
|
|
"scan rows with indicators": con.execute(scans, (symbol,)).fetchone()[0],
|
|
"of those, window elapsed": con.execute(due, (symbol,)).fetchone()[0],
|
|
}
|
|
if labelled:
|
|
out["labelled"] = con.execute(
|
|
"SELECT COUNT(*) FROM scan_outcomes").fetchone()[0]
|
|
out["complete"] = con.execute(
|
|
"SELECT COUNT(*) FROM scan_outcomes WHERE complete = 1"
|
|
).fetchone()[0]
|
|
out["pending (labelled, incomplete)"] = con.execute(
|
|
"SELECT COUNT(*) FROM scan_outcomes WHERE complete = 0"
|
|
).fetchone()[0]
|
|
return out
|
|
|
|
|
|
def main(argv=None) -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__.splitlines()[1])
|
|
ap.add_argument("--db", default=None, help="defaults to the signal_log.path setting in config.json")
|
|
ap.add_argument("--dry-run", action="store_true",
|
|
help="read-only candidate counts, writes nothing")
|
|
ap.add_argument("--budget", type=int, default=100000,
|
|
help="maximum rows to write in this run")
|
|
ap.add_argument("--symbol", default="XAU/USD")
|
|
args = ap.parse_args(argv)
|
|
path = args.db or app_config.signal_log_path()
|
|
if not os.path.exists(path):
|
|
print(f"no database at {path}")
|
|
return 1
|
|
|
|
if args.dry_run:
|
|
con = sqlite3.connect(f"file:{os.path.abspath(path)}?mode=ro", uri=True)
|
|
try:
|
|
have = {r[0] for r in con.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='table'")}
|
|
if "scan_outcomes" not in have:
|
|
print("scan_outcomes table not present yet (created on the"
|
|
" first non-dry run / next logged scan)")
|
|
for key, value in _counts(con, args.symbol,
|
|
labelled="scan_outcomes" in have).items():
|
|
print(f" {key}: {value}")
|
|
finally:
|
|
con.close()
|
|
return 0
|
|
|
|
lg = SignalLogger(path)
|
|
try:
|
|
print("before:", _counts(lg._conn, args.symbol))
|
|
written = 0
|
|
previous_complete = -1
|
|
while written < args.budget:
|
|
n = lg.label_scan_outcomes(
|
|
budget=min(CHUNK, args.budget - written), time_budget=120.0)
|
|
written += n
|
|
complete = lg._conn.execute(
|
|
"SELECT COUNT(*) FROM scan_outcomes WHERE complete = 1"
|
|
).fetchone()[0]
|
|
# Stop on a pass that wrote nothing, or one that could not
|
|
# complete a single further row (only re-labelling stragglers).
|
|
if n == 0 or complete == previous_complete:
|
|
break
|
|
previous_complete = complete
|
|
print(f"labelled (rows written): {written}")
|
|
print("after:", _counts(lg._conn, args.symbol))
|
|
finally:
|
|
lg.close()
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|