ForexBot2/scripts/label_scan_outcomes.py

#!/usr/bin/env python3
"""
label_scan_outcomes.py — batch driver for the scan_outcomes table.

Labels model-free forward returns (1/4/24 h, $ and ATR-normalized, plus
MFE/MAE, a 3xATR burst flag and a completeness flag) for every logged scan
whose 24 h window has closed, from the closed-candle streams already
stored in the DB. No provider, no network.

The live loggers do the same work opportunistically at the end of every
scan (time-boxed to ~2 s); this script is the catch-up pass for a backlog
— a freshly imported database, or rows whose bars only arrived later.

Idempotent: a row is rewritten only when its numbers change, so a settled
database costs one read pass. Safe beside the live writers (short per-row
transactions with a busy timeout). Read-only preview with --dry-run.

    python scripts/label_scan_outcomes.py --dry-run
    python scripts/label_scan_outcomes.py
    python scripts/label_scan_outcomes.py --db logs/signals.db --budget 5000
"""
import argparse
import os
import sqlite3
import sys
from datetime import datetime, timedelta, timezone
from typing import Optional

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import app_config  # noqa: E402
from signal_log import SCAN_LABEL_MIN_AGE_H, SignalLogger  # noqa: E402

CHUNK = 2000


def _counts(con: sqlite3.Connection, symbol: str,
            labelled: bool = True, now: Optional[datetime] = None) -> dict:
    """Candidate / labelled / complete / pending counts (read-only).

    The age filter uses logged_at (always an ISO string) as a proxy for
    the bar close — the labeller itself parses the real bar label. The
    cutoff is built here in logged_at's own format (UTC, 'T' separator),
    the labeller's convention: SQLite's datetime() prints a space, which
    sorts before the 'T', so every row logged on the cutoff's calendar
    day was left out of the count."""
    moment = now or datetime.now(timezone.utc)
    if moment.tzinfo is None:
        moment = moment.replace(tzinfo=timezone.utc)
    cutoff = moment.astimezone(timezone.utc) - timedelta(
        hours=SCAN_LABEL_MIN_AGE_H)
    scans = ("SELECT COUNT(*) FROM signals s WHERE s.symbol = ?"
             " AND s.indicators_json IS NOT NULL AND s.price > 0")
    due = scans + " AND s.logged_at < ?"
    out = {
        "scan rows with indicators": con.execute(scans, (symbol,)).fetchone()[0],
        "of those, window elapsed": con.execute(
            due, (symbol, cutoff.isoformat())).fetchone()[0],
    }
    if labelled:
        out["labelled"] = con.execute(
            "SELECT COUNT(*) FROM scan_outcomes").fetchone()[0]
        out["complete"] = con.execute(
            "SELECT COUNT(*) FROM scan_outcomes WHERE complete = 1"
        ).fetchone()[0]
        out["pending (labelled, incomplete)"] = con.execute(
            "SELECT COUNT(*) FROM scan_outcomes WHERE complete = 0"
        ).fetchone()[0]
    return out


def main(argv=None) -> int:
    ap = argparse.ArgumentParser(description=__doc__.splitlines()[1])
    ap.add_argument("--db", default=None, help="defaults to the signal_log.path setting in config.json")
    ap.add_argument("--dry-run", action="store_true",
                    help="read-only candidate counts, writes nothing")
    ap.add_argument("--budget", type=int, default=100000,
                    help="maximum rows to write in this run")
    ap.add_argument("--symbol", default="XAU/USD")
    args = ap.parse_args(argv)
    path = args.db or app_config.signal_log_path()
    if not os.path.exists(path):
        print(f"no database at {path}")
        return 1

    if args.dry_run:
        con = sqlite3.connect(f"file:{os.path.abspath(path)}?mode=ro", uri=True)
        try:
            have = {r[0] for r in con.execute(
                "SELECT name FROM sqlite_master WHERE type='table'")}
            if "scan_outcomes" not in have:
                print("scan_outcomes table not present yet (created on the"
                      " first non-dry run / next logged scan)")
            for key, value in _counts(con, args.symbol,
                                      labelled="scan_outcomes" in have).items():
                print(f"  {key}: {value}")
        finally:
            con.close()
        return 0

    lg = SignalLogger(path)
    try:
        print("before:", _counts(lg._conn, args.symbol))
        written = 0
        previous_complete = -1
        while written < args.budget:
            n = lg.label_scan_outcomes(
                budget=min(CHUNK, args.budget - written), time_budget=120.0)
            written += n
            complete = lg._conn.execute(
                "SELECT COUNT(*) FROM scan_outcomes WHERE complete = 1"
            ).fetchone()[0]
            # Stop on a pass that wrote nothing, or one that could not
            # complete a single further row (only re-labelling stragglers).
            if n == 0 or complete == previous_complete:
                break
            previous_complete = complete
        print(f"labelled (rows written): {written}")
        print("after:", _counts(lg._conn, args.symbol))
    finally:
        lg.close()
    return 0


if __name__ == "__main__":
    sys.exit(main())