docs: architecture diagram (PNG + matplotlib generator)

End-to-end map: external services → providers/news layer → signal core
(engine + factory + aggregator) → SignalReport → four delivery surfaces,
plus the backtester replaying the same core. Regenerate with
venv/bin/python docs/architecture_diagram.py after architecture changes.

Claude-Session: https://claude.ai/code/session_01Aq1jZ6jBsBdEC2xT3u46SA

.gitignore

@@ -3,6 +3,9 @@
 .env.local
 config.json
 
+# Oanda daemon state
+oanda_state.json
+
 # Editor / tooling
 .claude/
 .vscode/
@@ -14,7 +17,6 @@ __pycache__/
 *.egg-info/
 .venv/
 venv/
-.coverage
 
 # OS
 .DS_Store
@@ -26,7 +28,12 @@ Thumbs.db
 # Backtesting data (keep scripts, ignore generated files)
 backtest/cache/
 backtest/*.csv
+backtest/research/*.parquet
+backtest/research/duka_m5_boundary_spreads.csv
+backtest/trailgrid/
+backtest/trailgrid5m/
 backtest/__pycache__/
 .env.*
 logs/
-backtest/research/sizing_2026-09/
+backtest/research/commit_ladder_2026-08/
+backtest/research/llm_news_eval_2026-08/

advisory_outcome.py

@@ -1,354 +0,0 @@
-"""Deterministic advisory trade outcomes from stored closed candles.
-
-This module measures the hypothetical setup reviewed by the LLM panel. It
-never changes a signal, panel verdict, order, or daemon state. The model uses
-the same next-open reanchoring and close-only trailing rules as the backtester,
-but its evidence is the cleaned mid-candle snapshots stored in signals.db.
-"""
-
-from __future__ import annotations
-
-import logging
-import math
-from datetime import datetime, timedelta
-from typing import Any, Dict, Iterable, List, Optional, Tuple
-from zoneinfo import ZoneInfo
-
-logger = logging.getLogger(__name__)
-
-MODEL = "stored-closed-mid-next-open-trail-v1"
-MAX_HOLD_BARS = 300
-REFERENCE_COST = 0.30
-REALISTIC_COST = 0.65
-
-_TF_ALIASES = {
-    "M1": "1min", "1MIN": "1min", "1min": "1min",
-    "M5": "5min", "5MIN": "5min", "5min": "5min",
-    "M15": "15min", "15MIN": "15min", "15min": "15min",
-    "M30": "30min", "30MIN": "30min", "30min": "30min",
-    "H1": "1h", "1H": "1h", "1h": "1h",
-    "H4": "4h", "4H": "4h", "4h": "4h",
-    "D1": "1day", "1D": "1day", "1d": "1day",
-    "1DAY": "1day", "1day": "1day",
-}
-
-_TF_SECONDS = {
-    "1min": 60,
-    "5min": 5 * 60,
-    "15min": 15 * 60,
-    "30min": 30 * 60,
-    "1h": 60 * 60,
-    "4h": 4 * 60 * 60,
-    "1day": 24 * 60 * 60,
-}
-
-_NY_TZ = ZoneInfo("America/New_York")
-
-
-def _is_forex_weekend(now: datetime) -> bool:
-    """NY-anchored XAU weekend closure; test-locked to the signal copies."""
-    ny = now.astimezone(_NY_TZ)
-    weekday = ny.weekday()
-    if weekday == 4:
-        return ny.hour >= 17
-    if weekday == 5:
-        return True
-    if weekday == 6:
-        return ny.hour < 18
-    return False
-
-
-def normalize_timeframe(value: Any) -> str:
-    text = str(getattr(value, "value", value) or "")
-    return _TF_ALIASES.get(text, _TF_ALIASES.get(text.upper(), text))
-
-
-def timeframe_value(mapping: Any, timeframe: Any) -> Any:
-    """Read a timeframe-keyed snapshot regardless of enum/storage alias."""
-    if not isinstance(mapping, dict):
-        return None
-    wanted = normalize_timeframe(timeframe)
-    for key, value in mapping.items():
-        if normalize_timeframe(key) == wanted:
-            return value
-    return None
-
-
-def _finite(value: Any) -> Optional[float]:
-    try:
-        number = float(value)
-    except (TypeError, ValueError):
-        return None
-    return number if math.isfinite(number) else None
-
-
-def _timestamp(value: Any) -> Optional[datetime]:
-    try:
-        return datetime.fromisoformat(str(value).replace(" ", "T"))
-    except (TypeError, ValueError):
-        return None
-
-
-def initial_outcome(
-    setup: Optional[Dict[str, Any]],
-    indicators: Optional[Dict[str, Any]],
-    timeframe: Any,
-) -> Tuple[str, Dict[str, Any]]:
-    """Freeze the setup and deterministic evaluation policy."""
-    if not setup:
-        return "NOT_APPLICABLE", {
-            "model": MODEL,
-            "status": "NOT_APPLICABLE",
-            "reason": "panel row has no trade setup",
-        }
-
-    tf = normalize_timeframe(timeframe)
-    analysis = timeframe_value(indicators, tf)
-    signal_bar = analysis.get("timestamp") if isinstance(analysis, dict) else None
-    direction = str(setup.get("direction", ""))
-    entry = _finite(setup.get("entry"))
-    stop = _finite(setup.get("stop_loss"))
-    tp1 = _finite(setup.get("take_profit_1"))
-    arm = _finite(setup.get("trail_arm_r"))
-    distance = _finite(setup.get("trail_dist_r"))
-    risk = abs(entry - stop) if entry is not None and stop is not None else 0.0
-    if (direction not in ("BUY", "SELL") or not signal_bar
-            or _timestamp(signal_bar) is None or risk <= 0 or tp1 is None
-            or arm is None or distance is None or arm < 0 or distance < 0):
-        return "UNRESOLVABLE", {
-            "model": MODEL,
-            "status": "UNRESOLVABLE",
-            "reason": "invalid or incomplete setup/outcome seed",
-        }
-
-    state = {
-        "model": MODEL,
-        "status": "PENDING",
-        "timeframe": tf,
-        "signal_bar_at": str(signal_bar),
-        "setup": {
-            "direction": direction,
-            "reported_entry": entry,
-            "stop_distance": risk,
-            "tp1_distance": abs(tp1 - entry),
-            "strategy": setup.get("strategy"),
-            "setup_pattern": setup.get("setup_pattern"),
-        },
-        "policy": {
-            "entry": "next_primary_bar_open_reanchored",
-            "max_hold_bars": MAX_HOLD_BARS,
-            "trail_arm_r": arm,
-            "trail_dist_r": distance,
-            "tp1_is_advisory": arm > 0,
-            "reference_cost": REFERENCE_COST,
-            "realistic_cost": REALISTIC_COST,
-        },
-        "progress": {
-            "last_processed_bar": str(signal_bar),
-            "entry_at": None,
-            "entry_price": None,
-            "initial_stop": None,
-            "current_stop": None,
-            "peak": None,
-            "bars_held": 0,
-            "mfe_r": 0.0,
-            "mae_r": 0.0,
-        },
-        "result": None,
-    }
-    return "PENDING", state
-
-
-def _valid_bars(bars: Iterable[Dict[str, Any]]) -> List[Dict[str, Any]]:
-    valid = []
-    for bar in bars:
-        stamp = _timestamp(bar.get("timestamp")) if isinstance(bar, dict) else None
-        values = [_finite(bar.get(key)) for key in ("open", "high", "low", "close")]
-        if stamp is None or any(value is None for value in values):
-            continue
-        valid.append({
-            "timestamp": str(bar["timestamp"]),
-            "_timestamp": stamp,
-            "open": values[0], "high": values[1],
-            "low": values[2], "close": values[3],
-        })
-    valid.sort(key=lambda item: item["_timestamp"])
-    return valid
-
-
-def _has_continuity(
-    bars: List[Dict[str, Any]], last: datetime, timeframe: str
-) -> bool:
-    """Reject missing tradable bars while allowing scheduled XAU closures."""
-    seconds = _TF_SECONDS.get(normalize_timeframe(timeframe))
-    if seconds is None:
-        return False
-    step = timedelta(seconds=seconds)
-    previous = last
-    seen = {last}
-    for bar in bars:
-        stamp = bar["_timestamp"]
-        if stamp <= previous or stamp in seen:
-            continue
-        expected = previous + step
-        while expected < stamp:
-            ny = expected.astimezone(_NY_TZ)
-            daily_maintenance = ny.weekday() < 4 and ny.hour == 17
-            if not (_is_forex_weekend(expected) or daily_maintenance):
-                return False
-            expected += step
-        if expected != stamp:
-            return False
-        previous = stamp
-        seen.add(stamp)
-    return True
-
-
-def _finish(
-    state: Dict[str, Any], reason: str, exit_at: str, exit_price: float
-) -> None:
-    setup = state["setup"]
-    progress = state["progress"]
-    entry = float(progress["entry_price"])
-    risk = float(setup["stop_distance"])
-    pnl = ((exit_price - entry) if setup["direction"] == "BUY"
-           else (entry - exit_price))
-    gross_r = pnl / risk
-    state["status"] = "RESOLVED"
-    state["result"] = {
-        "exit_at": exit_at,
-        "exit_price": exit_price,
-        "exit_reason": reason,
-        "bars_held": progress["bars_held"],
-        "gross_r": gross_r,
-        "net_r_030": (pnl - REFERENCE_COST) / risk,
-        "net_r_065": (pnl - REALISTIC_COST) / risk,
-        "mfe_r": progress["mfe_r"],
-        "mae_r": progress["mae_r"],
-    }
-
-
-def advance_outcome(
-    state: Dict[str, Any], bars: Iterable[Dict[str, Any]]
-) -> Dict[str, Any]:
-    """Advance one pending outcome with overlapping closed-bar snapshots."""
-    if state.get("status") != "PENDING":
-        return state
-    try:
-        progress = state["progress"]
-        setup = state["setup"]
-        policy = state["policy"]
-        last = _timestamp(progress["last_processed_bar"])
-        signal_bar = _timestamp(state["signal_bar_at"])
-        if last is None or signal_bar is None:
-            raise ValueError("invalid progress timestamp")
-
-        buy = setup["direction"] == "BUY"
-        risk = float(setup["stop_distance"])
-        trail_arm = float(policy["trail_arm_r"])
-        trail_distance = float(policy["trail_dist_r"])
-
-        valid_bars = _valid_bars(bars)
-        has_future = any(bar["_timestamp"] > last for bar in valid_bars)
-        has_overlap = any(bar["_timestamp"] == last for bar in valid_bars)
-        if has_future and not has_overlap:
-            state["status"] = "UNRESOLVABLE"
-            state["result"] = None
-            state["reason"] = "stored candle snapshots lost continuity"
-            return state
-        if has_future and not _has_continuity(valid_bars, last,
-                                              state["timeframe"]):
-            state["status"] = "UNRESOLVABLE"
-            state["result"] = None
-            state["reason"] = "stored candle snapshots contain an internal gap"
-            return state
-
-        for bar in valid_bars:
-            stamp = bar["_timestamp"]
-            if stamp <= signal_bar or stamp <= last:
-                continue
-
-            if progress["entry_price"] is None:
-                entry = bar["open"]
-                initial_stop = entry - risk if buy else entry + risk
-                progress.update({
-                    "entry_at": bar["timestamp"],
-                    "entry_price": entry,
-                    "initial_stop": initial_stop,
-                    "current_stop": initial_stop,
-                    "peak": entry,
-                })
-
-            entry = float(progress["entry_price"])
-            current_stop = float(progress["current_stop"])
-            initial_stop = float(progress["initial_stop"])
-            progress["bars_held"] += 1
-            progress["last_processed_bar"] = bar["timestamp"]
-            last = stamp
-
-            if buy:
-                mfe = (bar["high"] - entry) / risk
-                mae = (entry - bar["low"]) / risk
-                stop_hit = bar["low"] <= current_stop
-            else:
-                mfe = (entry - bar["low"]) / risk
-                mae = (bar["high"] - entry) / risk
-                stop_hit = bar["high"] >= current_stop
-            progress["mfe_r"] = max(float(progress["mfe_r"]), max(0.0, mfe))
-            progress["mae_r"] = max(float(progress["mae_r"]), max(0.0, mae))
-
-            if stop_hit:
-                exit_price = current_stop
-                if (buy and bar["open"] < current_stop) or \
-                        (not buy and bar["open"] > current_stop):
-                    exit_price = bar["open"]
-                reason = "SL" if current_stop == initial_stop else "TRAIL"
-                _finish(state, reason, bar["timestamp"], exit_price)
-                return state
-
-            if trail_arm <= 0:
-                target = entry + setup["tp1_distance"] if buy \
-                    else entry - setup["tp1_distance"]
-                target_hit = bar["high"] >= target if buy else bar["low"] <= target
-                if target_hit:
-                    exit_price = target
-                    if (buy and bar["open"] > target) or \
-                            (not buy and bar["open"] < target):
-                        exit_price = bar["open"]
-                    _finish(state, "TP1", bar["timestamp"], exit_price)
-                    return state
-            else:
-                favorable = bar["high"] if buy else bar["low"]
-                peak = max(float(progress["peak"]), favorable) if buy \
-                    else min(float(progress["peak"]), favorable)
-                progress["peak"] = peak
-                excursion = ((peak - entry) if buy else (entry - peak)) / risk
-                if excursion >= trail_arm:
-                    level = peak - trail_distance * risk if buy \
-                        else peak + trail_distance * risk
-                    progress["current_stop"] = max(current_stop, level) if buy \
-                        else min(current_stop, level)
-
-            if progress["bars_held"] >= int(policy["max_hold_bars"]):
-                _finish(state, "TIMEOUT", bar["timestamp"], bar["close"])
-                return state
-        return state
-    except Exception as exc:
-        # Still fail-safe (never raises into a scan), but no longer
-        # SILENT: a real defect here used to be indistinguishable from a
-        # legitimate weekend/downtime gap, discoverable only by grepping
-        # advisory_outcome_json. Callers persist the state either way.
-        logger.exception(
-            "advisory outcome evaluator failed for the %s outcome seeded "
-            "at %s — recorded UNRESOLVABLE",
-            state.get("timeframe"), state.get("signal_bar_at"))
-        state["status"] = "UNRESOLVABLE"
-        state["result"] = None
-        state["reason"] = f"outcome evaluator error: {type(exc).__name__}"
-        return state
-
-
-def canonical_r(state: Dict[str, Any]) -> Optional[float]:
-    result = state.get("result")
-    value = result.get("net_r_065") if isinstance(result, dict) else None
-    return _finite(value)

app_config.py

@@ -21,6 +21,8 @@ the config.json.example defaults, every produced PlatformConfig is
 byte-identical to the pure-code defaults. apply_pair_overrides() runs
 in the pair factory BEFORE the validated per-timeframe deltas, so the
 bot, CLI and backtester all see the same overlay (live == backtest).
+
+Spec: docs/superpowers/specs/2026-07-21-centralized-config-design.md
 """
 
 import json
@@ -80,54 +82,22 @@ load_dotenv()
 _VALID_QUALITIES = ("A+", "A", "B", "C")
 _VALID_LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL")
 # Timeframes the bot's candle-boundary scheduler supports (must have
-# TF_CONFIRMATIONS entries in telegram_bot.py).
+# AUTOSCAN_CONFIRMATIONS entries in telegram_bot.py).
 _SCANNABLE_TFS = ("M5", "M15", "H1", "H4")
 
 _KNOWN_TOP = {"telegram", "data_providers", "scanning", "pairs", "log_level",
-               "oanda", "deepseek", "agent_panel", "market_outlook",
-               "violent_watch", "signal_log", "news_archive"}
+              "oanda", "deepseek"}
 _KNOWN_OANDA = {
-    # "account_id" is TOLERATED but ignored: it belonged to the auto-
-    # execution daemon removed 2026-09-03 (last commit 31002cb). The candle
-    # provider needs only api_token + base_url; older config.json files
-    # still carry the key and must keep loading.
-    "account_id", "api_token", "base_url",
+    "account_id", "api_token", "base_url", "lot_size",
+    "quality_gate", "max_hold_bars",
 }
 _KNOWN_DEEPSEEK = {
     "api_key", "model", "enabled", "timeout_seconds", "daily_request_cap",
 }
-_KNOWN_AGENT_PANEL = {
-    "enabled", "telegram", "dashboard", "min_quality",
-    "max_panels_per_day", "reuse_ttl_seconds",
-    "candles_enabled", "candle_bars_primary", "candle_bars_confirm",
-    "verdict_source",
-}
-_KNOWN_MARKET_OUTLOOK = {"enabled", "telegram", "max_per_day", "candles_bars"}
-_KNOWN_VIOLENT_WATCH = {"enabled", "alert_line", "headsup",
-                        "max_headsup_per_day", "headsup_min_level"}
-_KNOWN_SIGNAL_LOG = {"enabled", "path", "log_blocked_scans"}
-_KNOWN_NEWS_ARCHIVE = {"enabled", "path"}
 _KNOWN_TELEGRAM = {"token", "chat_id", "channel_id", "alert_min_quality"}
 _KNOWN_SCANNING = {"pairs", "timeframes"}
 _KNOWN_PAIR_KEYS = {"blocked_hours_utc"}
 
-_MARKET_OUTLOOK_DEFAULTS = {
-    "enabled": False,
-    "telegram": False,
-    "max_per_day": 12,
-    "candles_bars": 20,
-}
-
-# Violent-move watch (display-only heads-up) — dark by default.
-_VIOLENT_WATCH_DEFAULTS = {
-    "enabled": False,
-    "alert_line": True,
-    "headsup": True,
-    "max_headsup_per_day": 12,
-    "headsup_min_level": "hot",
-}
-_VALID_WATCH_LEVELS = ("watch", "hot")
-
 
 class AppConfigError(Exception):
     """config.json is present but invalid — fail fast at boot."""
@@ -148,13 +118,6 @@ class AppConfig:
     pair_overrides: Dict[str, Dict[str, Any]] = field(default_factory=dict)
     oanda_settings: Dict[str, Any] = field(default_factory=dict)
     deepseek_settings: Dict[str, Any] = field(default_factory=dict)
-    agent_panel_settings: Dict[str, Any] = field(default_factory=dict)
-    market_outlook_settings: Dict[str, Any] = field(
-        default_factory=lambda: dict(_MARKET_OUTLOOK_DEFAULTS))
-    violent_watch_settings: Dict[str, Any] = field(
-        default_factory=lambda: dict(_VIOLENT_WATCH_DEFAULTS))
-    signal_log_settings: Dict[str, Any] = field(default_factory=dict)
-    news_archive_settings: Dict[str, Any] = field(default_factory=dict)
     log_level: str = "INFO"
 
 
@@ -274,6 +237,20 @@ def load_app_config(path: str = CONFIG_PATH) -> AppConfig:
 
     oanda = raw.get("oanda", {})
     _check_keys(oanda, _KNOWN_OANDA, '"oanda"')
+    if "lot_size" in oanda:
+        ls = oanda["lot_size"]
+        if not isinstance(ls, int) or isinstance(ls, bool) or ls < 1:
+            raise AppConfigError(
+                "config.json: oanda.lot_size must be a positive integer")
+    if "quality_gate" in oanda and oanda["quality_gate"] not in _VALID_QUALITIES:
+        raise AppConfigError(
+            f'config.json: oanda.quality_gate "{oanda["quality_gate"]}" '
+            f"not in {_VALID_QUALITIES}")
+    if "max_hold_bars" in oanda:
+        mhb = oanda["max_hold_bars"]
+        if not isinstance(mhb, int) or isinstance(mhb, bool) or mhb < 1:
+            raise AppConfigError(
+                "config.json: oanda.max_hold_bars must be a positive integer")
     app.oanda_settings = dict(oanda)
 
     deepseek = raw.get("deepseek", {})
@@ -299,107 +276,6 @@ def load_app_config(path: str = CONFIG_PATH) -> AppConfig:
                 "config.json: deepseek.model must be a non-empty string")
     app.deepseek_settings = dict(deepseek)
 
-    panel = raw.get("agent_panel", {})
-    _check_keys(panel, _KNOWN_AGENT_PANEL, '"agent_panel"')
-    for flag in ("enabled", "telegram", "dashboard"):
-        if flag in panel and not isinstance(panel[flag], bool):
-            raise AppConfigError(
-                f"config.json: agent_panel.{flag} must be true or false")
-    if "min_quality" in panel and panel["min_quality"] not in _VALID_QUALITIES:
-        raise AppConfigError(
-            f'config.json: agent_panel.min_quality "{panel["min_quality"]}" '
-            f"not in {_VALID_QUALITIES}")
-    if "max_panels_per_day" in panel:
-        mpd = panel["max_panels_per_day"]
-        if isinstance(mpd, bool) or not isinstance(mpd, int) or mpd < 1:
-            raise AppConfigError(
-                "config.json: agent_panel.max_panels_per_day must be a "
-                "positive integer")
-    if "reuse_ttl_seconds" in panel:
-        ttl = panel["reuse_ttl_seconds"]
-        if isinstance(ttl, bool) or not isinstance(ttl, (int, float)) \
-                or ttl < 0:
-            raise AppConfigError(
-                "config.json: agent_panel.reuse_ttl_seconds must be a "
-                "non-negative number")
-    if "candles_enabled" in panel \
-            and not isinstance(panel["candles_enabled"], bool):
-        raise AppConfigError(
-            "config.json: agent_panel.candles_enabled must be true or false")
-    for key in ("candle_bars_primary", "candle_bars_confirm"):
-        if key in panel:
-            n = panel[key]
-            if isinstance(n, bool) or not isinstance(n, int) \
-                    or not 1 <= n <= 100:
-                raise AppConfigError(
-                    f"config.json: agent_panel.{key} must be an integer "
-                    "between 1 and 100")
-    if "verdict_source" in panel and panel["verdict_source"] not in \
-            ("macro", "chair"):
-        raise AppConfigError(
-            'config.json: agent_panel.verdict_source must be "macro" or '
-            '"chair"')
-    app.agent_panel_settings = dict(panel)
-
-    outlook = raw.get("market_outlook", {})
-    _check_keys(outlook, _KNOWN_MARKET_OUTLOOK, '"market_outlook"')
-    for flag in ("enabled", "telegram"):
-        if flag in outlook and not isinstance(outlook[flag], bool):
-            raise AppConfigError(
-                f"config.json: market_outlook.{flag} must be true or false")
-    for key in ("max_per_day", "candles_bars"):
-        if key in outlook:
-            value = outlook[key]
-            if isinstance(value, bool) or not isinstance(value, int) or value < 1:
-                raise AppConfigError(
-                    f"config.json: market_outlook.{key} must be a positive integer")
-    app.market_outlook_settings.update(outlook)
-
-    watch = raw.get("violent_watch", {})
-    _check_keys(watch, _KNOWN_VIOLENT_WATCH, '"violent_watch"')
-    for flag in ("enabled", "alert_line", "headsup"):
-        if flag in watch and not isinstance(watch[flag], bool):
-            raise AppConfigError(
-                f"config.json: violent_watch.{flag} must be true or false")
-    if "max_headsup_per_day" in watch:
-        cap = watch["max_headsup_per_day"]
-        if isinstance(cap, bool) or not isinstance(cap, int) or cap < 0:
-            raise AppConfigError(
-                "config.json: violent_watch.max_headsup_per_day must be a "
-                "non-negative integer")
-    if ("headsup_min_level" in watch
-            and watch["headsup_min_level"] not in _VALID_WATCH_LEVELS):
-        raise AppConfigError(
-            f'config.json: violent_watch.headsup_min_level '
-            f'"{watch["headsup_min_level"]}" not in {_VALID_WATCH_LEVELS}')
-    app.violent_watch_settings.update(watch)
-
-    slog = raw.get("signal_log", {})
-    _check_keys(slog, _KNOWN_SIGNAL_LOG, '"signal_log"')
-    if "enabled" in slog and not isinstance(slog["enabled"], bool):
-        raise AppConfigError(
-            "config.json: signal_log.enabled must be true or false")
-    if "path" in slog and (not isinstance(slog["path"], str)
-                           or not slog["path"]):
-        raise AppConfigError(
-            "config.json: signal_log.path must be a non-empty string")
-    if ("log_blocked_scans" in slog
-            and not isinstance(slog["log_blocked_scans"], bool)):
-        raise AppConfigError(
-            "config.json: signal_log.log_blocked_scans must be true or false")
-    app.signal_log_settings = dict(slog)
-
-    narch = raw.get("news_archive", {})
-    _check_keys(narch, _KNOWN_NEWS_ARCHIVE, '"news_archive"')
-    if "enabled" in narch and not isinstance(narch["enabled"], bool):
-        raise AppConfigError(
-            "config.json: news_archive.enabled must be true or false")
-    if "path" in narch and (not isinstance(narch["path"], str)
-                            or not narch["path"]):
-        raise AppConfigError(
-            "config.json: news_archive.path must be a non-empty string")
-    app.news_archive_settings = dict(narch)
-
     level = raw.get("log_level", app.log_level)
     if level not in _VALID_LOG_LEVELS:
         raise AppConfigError(
@@ -473,6 +349,12 @@ def provider_api_key(
 
 # ── Oanda resolution helpers (env → config.json → default) ───────────
 
+def oanda_account_id(app: Optional[AppConfig] = None) -> str:
+    app = app or get_app_config()
+    return os.environ.get("OANDA_ACCOUNT_ID") or str(
+        app.oanda_settings.get("account_id", ""))
+
+
 def oanda_api_token(app: Optional[AppConfig] = None) -> str:
     app = app or get_app_config()
     return os.environ.get("OANDA_API_TOKEN") or str(
@@ -493,92 +375,6 @@ def deepseek_api_key(app: Optional[AppConfig] = None) -> str:
         app.deepseek_settings.get("api_key", ""))
 
 
-# ── AI market-outlook resolution helper ──────────────────────────────
-
-def market_outlook_enabled(app: Optional[AppConfig] = None) -> bool:
-    """Whether the advisory outlook feature is enabled (env wins)."""
-    env = os.environ.get("MARKET_OUTLOOK_ENABLED")
-    if env:
-        return env.strip().lower() in ("1", "true", "yes", "on")
-    app = app or get_app_config()
-    return bool(app.market_outlook_settings.get("enabled", False))
-
-
-# ── Violent-move watch resolution helper ─────────────────────────────
-
-def violent_watch_enabled(app: Optional[AppConfig] = None) -> bool:
-    """Whether the display-only violent-move watch is on (env wins)."""
-    env = os.environ.get("VIOLENT_WATCH_ENABLED")
-    if env:
-        return env.strip().lower() in ("1", "true", "yes", "on")
-    app = app or get_app_config()
-    return bool(app.violent_watch_settings.get("enabled", False))
-
-
-# ── Signal-log resolution helpers (env → config.json → default) ──────
-
-def signal_log_enabled(app: Optional[AppConfig] = None) -> bool:
-    """Default ON — signal history is the platform's audit trail."""
-    env = os.environ.get("SIGNAL_LOG_ENABLED")
-    if env:
-        return env.strip().lower() in ("1", "true", "yes", "on")
-    app = app or get_app_config()
-    return bool(app.signal_log_settings.get("enabled", True))
-
-
-def signal_log_blocked_scans(app: Optional[AppConfig] = None) -> bool:
-    """Whether weekend/day-filter early returns are logged (as minimal
-    rows without news/candles). Default ON — they are liveness evidence;
-    turn off to keep the history table to real scans only."""
-    env = os.environ.get("SIGNAL_LOG_BLOCKED_SCANS")
-    if env:
-        return env.strip().lower() in ("1", "true", "yes", "on")
-    app = app or get_app_config()
-    return bool(app.signal_log_settings.get("log_blocked_scans", True))
-
-
-def signal_log_path(app: Optional[AppConfig] = None) -> str:
-    """Absolute path to the SQLite signal-history DB. A relative config
-    value resolves against the repo root so CWD never matters."""
-    env = os.environ.get("SIGNAL_LOG_PATH")
-    if env:
-        path = env
-    else:
-        app = app or get_app_config()
-        path = str(app.signal_log_settings.get("path", "logs/signals.db"))
-    if not os.path.isabs(path):
-        path = os.path.join(os.path.dirname(os.path.abspath(__file__)), path)
-    return path
-
-
-# ── News-archive resolution helpers (env → config.json → default) ────
-
-def news_archive_enabled(app: Optional[AppConfig] = None) -> bool:
-    """Default ON — the headline corpus is training data for a local
-    scorer and cannot be backfilled once a fetch window has passed. It
-    lives in its own DB so a signals.db rotation cannot take it with it
-    (which is how three weeks of news were lost on 2026-09-04)."""
-    env = os.environ.get("NEWS_ARCHIVE_ENABLED")
-    if env:
-        return env.strip().lower() in ("1", "true", "yes", "on")
-    app = app or get_app_config()
-    return bool(app.news_archive_settings.get("enabled", True))
-
-
-def news_archive_path(app: Optional[AppConfig] = None) -> str:
-    """Absolute path to the SQLite headline corpus. A relative config
-    value resolves against the repo root so CWD never matters."""
-    env = os.environ.get("NEWS_ARCHIVE_PATH")
-    if env:
-        path = env
-    else:
-        app = app or get_app_config()
-        path = str(app.news_archive_settings.get("path", "logs/newsdata.db"))
-    if not os.path.isabs(path):
-        path = os.path.join(os.path.dirname(os.path.abspath(__file__)), path)
-    return path
-
-
 # ── PlatformConfig overlay ────────────────────────────────────────────
 
 def apply_pair_overrides(

backtest/diagnose_filters.py

@@ -45,6 +45,7 @@ from gpu_backtest import (  # noqa: E402
     BAR_SECONDS,
     GPUTechnicalEngine,
     PAIR_ALIASES,
+    BacktestConfig,
     IndicatorCache,
     fetch_history,
     precompute_indicators,
@@ -220,6 +221,7 @@ def run_diagnosis(
     platform_cfg = PlatformConfig(
         symbol=pair,
         display_name=display,
+        pip_value=pair_info["pip_value"],
         primary_timeframe=primary_tf,
         confirmation_timeframes=list(confirm_tfs),
     )

backtest/gpu_backtest.py

@@ -62,7 +62,6 @@ from config import (  # noqa: E402
 from providers import BaseDataProvider, make_data_provider  # noqa: E402
 from engine import IndicatorSignal, TechnicalAnalysis, TechnicalEngine  # noqa: E402
 from signal_aggregator import SignalAggregator, SignalReport, TradeSetup  # noqa: E402
-import position_sizing  # noqa: E402
 
 # ─────────────────────────────────────────────────────────────────────
 #  GPU shim — CuPy if available AND CUDA runtime works, else NumPy.
@@ -963,6 +962,8 @@ class GPUTechnicalEngine(TechnicalEngine):
         result.key_levels["kc_upper"] = kc_upper_val
         result.key_levels["kc_lower"] = kc_lower_val
         result.key_levels["kc_mid"] = kc_mid_val
+        result._kc_squeeze = is_squeeze
+        result._kc_squeeze_released = squeeze_released
 
 
 # ═════════════════════════════════════════════════════════════════════
@@ -1073,13 +1074,6 @@ class Trade:
     indicators_neutral: int = 0
     indicators_total: int = 0      # total scored indicators
 
-    # RESEARCH (--export-features only): decision-time feature vector
-    # (position_sizing.FEATURE_ORDER) and the stake multiplier the setup
-    # carried. Empty / 1.0 on every normal run, so the default book is
-    # untouched.
-    features: Dict[str, float] = field(default_factory=dict)
-    stake_multiplier: float = 1.0
-
     # Filled on exit:
     exit_time: Optional[pd.Timestamp] = None
     exit_price: Optional[float] = None
@@ -1460,11 +1454,6 @@ class BacktestConfig:
     # afterwards in a fast ordered pass. Results are identical to the
     # sequential path by construction.
     workers: int = 0
-    # RESEARCH: also compute position_sizing.build_features() at each
-    # entry and append them to --export-csv. Off by default — the extra
-    # columns would break comparison against the pinned 63-column books,
-    # and the feature build costs time per candidate.
-    export_features: bool = False
 
 
 def _is_xau_blocked_by_calendar(ts: pd.Timestamp, cfg: PlatformConfig) -> Optional[str]:
@@ -1621,17 +1610,6 @@ def _replay_span(span: Tuple[int, int]):
             adj_tp1 = entry_price - tp1_distance
             adj_tp2 = entry_price - tp2_distance
 
-        # RESEARCH: decision-time features (same frames the aggregator
-        # just saw, so live == backtest by construction). Deliberately
-        # NOT wrapped in try/except — build_features already swallows its
-        # own failures, so anything escaping here is a real bug.
-        feats: Dict[str, float] = {}
-        if bt_cfg.export_features:
-            feats = position_sizing.build_features(
-                report, ts_setup, slice_dfs, bt_cfg.primary_tf,
-                list(bt_cfg.confirmation_tfs),
-            )
-
         candidates.append((pos, Trade(
             pair=bt_cfg.pair,
             entry_time=entry_time,
@@ -1657,10 +1635,6 @@ def _replay_span(span: Tuple[int, int]):
             indicators_opposing=oppose,
             indicators_neutral=neutral,
             indicators_total=total,
-            features=feats,
-            stake_multiplier=float(ts_setup.stake_multiplier
-                                   if ts_setup.stake_multiplier is not None
-                                   else 1.0),
             trail_arm_r=(platform_cfg.signals.pb_trail_arm_r
                          if tag == "XAU_PB" else 0.0),
             trail_dist_r=(platform_cfg.signals.pb_trail_dist_r
@@ -1763,6 +1737,7 @@ def run_backtest(
         platform_cfg = PlatformConfig(
             symbol=bt_cfg.pair,
             display_name=PAIR_INFO[bt_cfg.pair]["display"],
+            pip_value=PAIR_INFO[bt_cfg.pair]["pip_value"],
             primary_timeframe=bt_cfg.primary_tf,
             confirmation_timeframes=list(bt_cfg.confirmation_tfs),
         )
@@ -2182,17 +2157,11 @@ def print_trade_reasoning(trades: List[Trade], limit: int = 5) -> None:
         print(f"  MFE / MAE: {t.mfe_r:.2f}R / {t.mae_r:.2f}R")
 
 
-def trades_to_dataframe(trades: List[Trade],
-                        include_features: bool = False) -> pd.DataFrame:
+def trades_to_dataframe(trades: List[Trade]) -> pd.DataFrame:
     """Export trades for downstream analysis (e.g. parquet/CSV).
 
     Includes per-instructions: entry reasoning, indicator-by-indicator
     signal/value/agreement, and overall agreement count.
-
-    `include_features` (research, --export-features) appends
-    stake_multiplier, r_sized and the position_sizing.FEATURE_ORDER
-    columns AFTER the existing ones. Default off so the column set stays
-    byte-identical to the pinned books.
     """
     rows = []
     for t in trades:
@@ -2231,22 +2200,8 @@ def trades_to_dataframe(trades: List[Trade],
             row[f"ind_{name}_value"] = b["value"]
             row[f"ind_{name}_conf"] = b["confidence"]
             row[f"ind_{name}_agrees"] = b["agrees"]
-        if include_features:
-            row["stake_multiplier"] = t.stake_multiplier
-            row["r_sized"] = t.r_multiple * t.stake_multiplier
-            for name in position_sizing.FEATURE_ORDER:
-                row[name] = t.features.get(name, float("nan"))
         rows.append(row)
-    df = pd.DataFrame(rows)
-    if include_features and len(df.columns):
-        # A later trade can introduce an indicator column the first trade
-        # lacked (pandas orders columns by first appearance), which would
-        # land AFTER the extras. Force the extras to the tail so the default
-        # export is always a strict prefix of the feature export.
-        extras = ["stake_multiplier", "r_sized", *position_sizing.FEATURE_ORDER]
-        base_cols = [c for c in df.columns if c not in extras]
-        df = df[base_cols + [c for c in extras if c in df.columns]]
-    return df
+    return pd.DataFrame(rows)
 
 
 # ═════════════════════════════════════════════════════════════════════
@@ -2308,11 +2263,6 @@ def _parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace:
                    help="Round-trip cost in price units per trade (default: pair's "
                         "default_spread from PAIR_INFO — XAU $0.30; 0 disables)")
     p.add_argument("--export-csv", help="Optional path to dump per-trade CSV (incl. indicator breakdown)")
-    p.add_argument("--export-features", action="store_true",
-                   help="Append the 65 decision-time f_* feature columns "
-                        "(+ stake_multiplier, r_sized) to --export-csv; "
-                        "research flag, default off so pinned 63-column "
-                        "books stay comparable")
     p.add_argument("--detail", action="store_true",
                    help="Print full per-trade reasoning (indicators, signals, agreement, why entered)")
     p.add_argument("--workers", type=int, default=0,
@@ -2401,6 +2351,7 @@ def main(argv: Optional[List[str]] = None) -> int:
         platform_cfg = PlatformConfig(
             symbol=pair,
             display_name=PAIR_INFO[pair]["display"],
+            pip_value=PAIR_INFO[pair]["pip_value"],
             primary_timeframe=primary_tf,
             confirmation_timeframes=confirm_tfs or [Timeframe.H1, Timeframe.H4],
         )
@@ -2465,14 +2416,7 @@ def main(argv: Optional[List[str]] = None) -> int:
         spread=spread,
         workers=args.workers,
         fixed_rr_tp=args.fixed_rr_tp or 0.0,
-        export_features=args.export_features,
     )
-    if args.export_features:
-        logger.info(
-            f"--export-features ON: appending stake_multiplier, r_sized and "
-            f"{len(position_sizing.FEATURE_ORDER)} f_* feature columns to the "
-            f"CSV export (research flag; book itself is unchanged)"
-        )
 
     logger.info(
         f"Backend: {'CuPy GPU' if _GPU_AVAILABLE else 'NumPy CPU fallback'} | "
@@ -2486,7 +2430,7 @@ def main(argv: Optional[List[str]] = None) -> int:
     print_trade_reasoning(trades, limit=args.reasoning_samples)
 
     if args.export_csv:
-        df = trades_to_dataframe(trades, include_features=args.export_features)
+        df = trades_to_dataframe(trades)
         df.to_csv(args.export_csv, index=False)
         logger.info(f"Exported {len(df)} trades → {args.export_csv}")
 

backtest/instructions_for_backtest.txt

@@ -1,26 +1,59 @@
-Backtest reporting checklist — ForexBot2 (XAU/USD)
-==================================================
-Every backtest report must record:
-- The exact command line (window, --primary-tf, --spread, and any --set /
-  --quality / --strategies / --confirm-tfs overrides) and the "Backend:" log line.
-- Trades, total R, win rate, profit factor and max drawdown for the whole
-  window AND for both halves, at both cost conventions ($0.65 measured-Oanda
-  headline, $0.30 cross-book comparability) — CLAUDE.md rule 1.
-- Per trade (the exported CSV already carries these): entry/exit time and
-  price, exit reason, R multiple, MFE/MAE, entry reasons, and every
-  indicator's signal / value / agreement at decision time.
-- Breakdowns by session/hour, weekday, regime and volatility regime, and,
-  for any candidate change, the removed/added/shifted trade diff against the
-  control book (the 5-bar cooldown displaces entries, so judge run-level
-  numbers, never the added-trade bucket alone).
-- The md5 of the exported book whenever the run is a parity or regression check.
-Replays go through the real signal code (analyze_from_dataframes) — never a
-re-implementation.
+Note down entery and exit Why we took trade?
+What were each indicator signal on that specific time when we took trade
+How many indicators agreed at trade time
+Always backtest using engine.py and signal_aggregator.py
+Dont take trades on weekends and Fridays
+Keep TP small 1.3 Reward ratio
+At the end of test make table of results and print all values you observed.
+Trade context
+
+Time of day and trading session (Asian/London/NY/overlap)
+Day of week (track Mon/Tue/Wed/Thu performance separately)
+Market regime at entry: trending, ranging, or choppy (use ATR or ADX)
+Volatility level (ATR value or ATR percentile vs recent average)
+Pair/instrument traded
+News proximity (high-impact news within 30 min before/after entry)
+
+Trade execution details
+
+Spread at entry in pips
+Slippage on entry and exit (planned vs actual fill)
+Time in trade (minutes/hours from entry to exit)
+Maximum Adverse Excursion (MAE): how far against you it went
+Maximum Favorable Excursion (MFE): how far in profit before pullback
+
+Risk metrics
+
+Max consecutive losses (streak)
+Max consecutive wins (streak)
+Expectancy: (Win% x Avg Win) minus (Loss% x Avg Loss)
+Expectancy per hour in trade
+Recovery factor: Total R divided by Max DD
+
+Indicator/signal analysis
+
+Each indicator's standalone win rate when it agreed with the trade
+Confluence level performance (2 vs 3 vs 4+ indicators agreeing)
+Which indicator disagreed on losing trades (find the "canary")
+
+Decision quality
+
+Rate each setup A/B/C at entry, before knowing outcome
+Verify A-grade setups actually win more often
+Track system trades vs discretionary deviations separately
+
+Process notes
+
+Screenshot or chart snapshot at entry and exit
+Emotional state (calm, FOMO, revenge, tired)
+What would have happened if you held longer or exited sooner
+
+Aggregate analysis at end of backtest
+
+Equity curve, not just total R
+Performance breakdown by session, day, volatility regime
+Win rate over rolling 20-trade windows (check for edge decay)
+
+Critical extra
+"Do not trade" log: setups flagged but skipped, and what happened after
 
---- 2026-09-06: decision-time feature export ---
-- `--export-features` appends stake_multiplier, r_sized and the 65 `f_*` decision-time
-  features (position_sizing.FEATURE_ORDER) AFTER the 63 pinned columns; default export is
-  byte-identical to the pinned books.
-- Any sizing report must show the FLAT book and the SIZED book side by side and the
-  label-shuffle null; the 2026-09-06 campaign verdict (nothing ships,
-  backtest/research/sizing_campaign_2026-09.md) is the reference.

backtest/research/__init__.py

@@ -0,0 +1,5 @@
+"""Research tooling: calendar sourcing/store and edge-screening utilities.
+
+Nothing here is imported by the live bot; these modules support
+backtesting and edge validation only.
+"""

backtest/research/ai_signal_enrichment_2026-08-15.md

@@ -1,116 +0,0 @@
-# AI signal enrichment — analysis + build record (2026-08-15)
-
-Owner questions:
-1. If the AI panel also received indicator values and (limited) candle data, would the system be more accurate?
-2. If AI reviewed every candle alongside the local score system, would prediction accuracy justify the token burn — or is there a middle ground?
-
-## What the panel already received (pre-change baseline)
-
-The four personas (`agent_panel.py`) already got a *distilled* picture:
-per-indicator signal+confidence, MTF signals/scores, key levels,
-exhaustion count, full trade-setup math, warnings, headlines/events/
-sentiment. Missing: raw indicator **values** and any **candle** data.
-
-Cost guardrails already in place: 5 requests/panel, `max_panels_per_day`
-20 (≤100 requests/day), 3600 s reuse-TTL, per-bar exact cache, shared
-500 req/day DeepSeek cap, static prompts on the ~50× prefix cache.
-
-## Evidence used
-
-- **LLM is strong at text, weak at raw numerics** (measured in-repo,
-  `llm_news_eval_2026-08-06.md`): headline sentiment 100% (78/78) vs
-  keyword scorer 31%; but ~11% of event-direction calls misread close
-  numeric comparisons until code pre-computed the comparison
-  (`vs_forecast`) and the model only mapped it to doctrine.
-- **The book any filter would act on** (M5 canonical,
-  `oanda_m5_12mo_065_nopb_20260814.csv`): 577 trades, +106.82R,
-  PF 1.51, win rate 52.9%; A-grade 59.5%, B-grade 51.4% (+67.8R of the
-  total), C-grade 43.8% (−0.23R, 16 trades). Average edge ≈ 0.185R/trade.
-
-## Answer 1 — enrich the existing panel? YES (cheap, safe)
-
-- A compact candle is ~15–25 tokens; 20 primary-TF + 10 confirm-TF bars
-  ≈ 500–1,500 tokens per persona call → cents/day at the 20-panel cap.
-  **Tokens are not the constraint; the request cap is.**
-- Measured after the build (dry-test wire log): enriched technical
-  payload with 20 candles ≈ **~380 tokens**; replies ≈ 25–35 tokens
-  under the new 15-word note cap.
-- Caveats: (a) the panel is advisory-only — richer payloads improve the
-  human's context, not the book's P&L; (b) raw prices are the model's
-  weakest input, so candles are sent **ATR-normalized** (code does the
-  arithmetic, LLM reads the shape) — the same pattern that fixed the
-  event-direction misreads.
-
-## Answer 2 — AI on every candle? NO
-
-- **Feasibility**: M5+M15+H1+H4 ≈ 414 bars/day → one call/bar nearly
-  exhausts the shared 500/day cap (news needs it too); a panel per bar
-  ≈ 2,070/day = 4× over. Most of that spend reviews HOLD bars.
-- **Parity (rule 4)**: an AI gate cannot be replayed by the backtester
-  (news off; no recorded verdicts) — every pinned book becomes
-  non-comparable, and model-version drift silently mutates the strategy.
-- **Edge**: the engine already compresses 380 bars into 9 tuned
-  indicators + regime + MTF validated over 5 years; an LLM's generic
-  prior on raw candles won't beat fitted thresholds, and the margin it
-  would filter is ~0.185R/trade. B-grade holds +67.8R at 51.4% win —
-  a blunt veto is net-negative.
-
-## The middle ground (adopted)
-
-1. **Enrich payloads** (built today, see below) — near-zero cost, no
-   rule-1 exposure.
-2. **Log panel verdicts to signals.db** (built today) — creates the
-   "does OPPOSE predict losers?" dataset with zero behavior change.
-3. **After 4–8 weeks**: join logged verdicts to outcomes; simulate an
-   OPPOSE-veto against the pinned book offline. Only then consider any
-   gating, as a full rule-1 revalidation.
-
-## Build record (2026-08-15, owner-approved)
-
-- `agent_panel.py`
-  - Technical persona payload now carries indicator **values** (units
-    legend added to the prompt) and **ATR-normalized candles** per TF:
-    `{"ref_close", "atr", "bars": [[o,h,l,c]…]}` oldest-first, each price
-    as `(price − ref_close) / atr`, 1 decimal. Window: 20 bars primary /
-    10 per confirmation TF.
-  - New config keys (validated in `app_config.py`, in
-    `config.json.example`): `candles_enabled` (default true),
-    `candle_bars_primary` (20), `candle_bars_confirm` (10).
-  - Panel-private provider singleton (`_dp`, patchable in tests);
-    TTL cache keeps fetches near-free; any failure → candles silently
-    omitted, panel unaffected.
-  - Prompt hardening: training data declared stale and OFF-LIMITS;
-    "if a fact is not in the JSON, treat it as unknown"; thin evidence
-    → neutral < 30 (unchanged); note cap 25→**15 words**, rationale
-    40→**30 words** (output-token savings).
-  - `attach_panel` now calls `signal_log.log_panel_update(report)` after
-    every successful attach (fresh, exact-cache, and reuse-TTL paths).
-- `signal_log.py`
-  - `log()` returns the rowid; `log_signal` stashes it on the report as
-    `_signal_log_rowid`; new `log_panel_update()` fills the 4 panel
-    columns on that row post-hoc. No rowid (backtest replay, logging
-    disabled, other process) → silent no-op. Fail-safe contract kept.
-- Tests: 381 → **388 passed** (7 new: candle normalization math, payload
-  values/candles, context windows + ATR gating + disable flag, attach
-  injection (technical persona only), prompt-contract assertions,
-  panel row update, rowid hand-off + no-op paths). Existing attach tests
-  stub `ap._dp` to stay off the network.
-- **Live dry test** (`panel_payload_drytest_2026-08-15.py` — script removed 2026-09-06, raw replies kept in the `.jsonl` — fabricated
-  numbers only, 13 real DeepSeek requests total): **7/7 probes pass** —
-  coherent-bullish → support@70; bearish-vs-BUY → oppose@85; genuinely
-  mixed → neutral@30; empty payload → neutral@20 (contract); no invented
-  numbers in any note; all notes ≤ 15 words; synthesizer AGREE with a
-  21-word rationale. Raw wire log:
-  `backtest/research/panel_payload_drytest_2026-08-15.jsonl`.
-  - Notable: an early P3 variant with H1 SELL against a BUY brief got
-    oppose@60 — *correct* per the prompt (higher-TF disagreement is a
-    listed conflict), so the probe was tightened to a genuinely mixed
-    scenario. Documented so nobody "fixes" the prompt for it later.
-
-## What this deliberately did NOT change
-
-- No gate reads panel fields; the panel remains advisory-only.
-- No backtest-path change (`analyze_from_dataframes` untouched; parity
-  test still locks panel fields empty in replay).
-- No verdict-based filtering — that decision waits for the signals.db
-  dataset (4–8 weeks) and an offline veto replay against the pinned book.

backtest/research/book_diff.py

@@ -0,0 +1,86 @@
+"""Trade-book diff: relaxed/gated run vs its control.
+
+Usage:
+    python backtest/research/book_diff.py CONTROL.csv CANDIDATE.csv BAR_MINUTES
+
+Decision numbers are RUN-LEVEL (ΔtotalR, ΔPF, ΔmaxDD, Δtrades) — never
+the added-trade bucket alone: the 5-bar entry cooldown means new
+entries displace existing ones. The added/removed decomposition and the
+per-decision-hour histogram of added trades are diagnostics.
+
+Shifted trades = removed+added pairs in the same direction within
+±5 bars (cooldown displacement, not a genuinely new/lost signal).
+"""
+import sys
+
+import pandas as pd
+
+
+def _metrics(df):
+    r = df["r_multiple"]
+    gains, losses = r[r > 0].sum(), -r[r <= 0].sum()
+    cum = r.cumsum()
+    return dict(trades=len(df), total_r=r.sum(),
+                pf=gains / max(losses, 1e-9),
+                wr=(r > 0).mean() * 100 if len(df) else 0.0,
+                max_dd=(cum.cummax() - cum).max() if len(cum) else 0.0)
+
+
+def diff_books(control_csv, candidate_csv, bar_minutes, split="2026-01-29"):
+    ctrl, cand = pd.read_csv(control_csv), pd.read_csv(candidate_csv)
+    for df in (ctrl, cand):
+        df["_t"] = pd.to_datetime(df["entry_time"], utc=True, format="mixed")
+        df["_key"] = df["entry_time"].astype(str) + "|" + df["direction"]
+
+    added = cand[~cand["_key"].isin(ctrl["_key"])].copy()
+    removed = ctrl[~ctrl["_key"].isin(cand["_key"])].copy()
+
+    # cooldown displacement: same-direction removed/added within ±5 bars
+    tol = pd.Timedelta(minutes=5 * bar_minutes)
+    shifted = 0
+    rem_pool = removed.copy()
+    for _, a in added.iterrows():
+        m = rem_pool[(rem_pool["direction"] == a["direction"])
+                     & ((rem_pool["_t"] - a["_t"]).abs() <= tol)]
+        if len(m):
+            shifted += 1
+            rem_pool = rem_pool.drop(m.index[0])
+
+    mc, mk = _metrics(ctrl), _metrics(cand)
+    split_ts = pd.Timestamp(split, tz="UTC")
+    halves = (cand[cand["_t"] < split_ts]["r_multiple"].sum(),
+              cand[cand["_t"] >= split_ts]["r_multiple"].sum())
+    hour_hist = None
+    if len(added):
+        added["dec_hour"] = (added["_t"]
+                             - pd.Timedelta(minutes=bar_minutes)).dt.hour
+        hour_hist = added.groupby("dec_hour")["r_multiple"] \
+                         .agg(["count", "sum"]).round(2)
+    return dict(
+        control=mc, candidate=mk,
+        delta_r=mk["total_r"] - mc["total_r"],
+        delta_pf=mk["pf"] - mc["pf"],
+        delta_dd=mk["max_dd"] - mc["max_dd"],
+        delta_trades=mk["trades"] - mc["trades"],
+        added_n=len(added), added_r=added["r_multiple"].sum(),
+        removed_n=len(removed), removed_r=removed["r_multiple"].sum(),
+        shifted=shifted, halves=halves, added_by_hour=hour_hist,
+    )
+
+
+if __name__ == "__main__":
+    d = diff_books(sys.argv[1], sys.argv[2], int(sys.argv[3]))
+    c, k = d["control"], d["candidate"]
+    print(f"control  : {c['trades']:4d} trades  {c['total_r']:+7.1f}R  "
+          f"PF {c['pf']:.2f}  WR {c['wr']:.1f}%  DD {c['max_dd']:.1f}")
+    print(f"candidate: {k['trades']:4d} trades  {k['total_r']:+7.1f}R  "
+          f"PF {k['pf']:.2f}  WR {k['wr']:.1f}%  DD {k['max_dd']:.1f}")
+    print(f"ΔR {d['delta_r']:+.1f}  ΔPF {d['delta_pf']:+.2f}  "
+          f"ΔDD {d['delta_dd']:+.1f}  Δtrades {d['delta_trades']:+d}")
+    print(f"added {d['added_n']} ({d['added_r']:+.1f}R)  "
+          f"removed {d['removed_n']} ({d['removed_r']:+.1f}R)  "
+          f"shifted≈{d['shifted']}")
+    print(f"candidate halves: H1 {d['halves'][0]:+.1f}R  H2 {d['halves'][1]:+.1f}R")
+    if d["added_by_hour"] is not None:
+        print("\nadded trades by decision hour:")
+        print(d["added_by_hour"].to_string())

backtest/research/burst_profile.py

@@ -0,0 +1,42 @@
+"""Burst-candle frequency by UTC hour — the empirical "hedge fund hours".
+
+Usage:
+    python backtest/research/burst_profile.py PARQUET_GLOB
+
+Per UTC bar-close hour of the given bars: % of bars whose range is
+>= K x the PRIOR bar's Wilder ATR(14), for K in {3,4,5}; mean bar
+range; and mean signed drift over the 6 bars following a >=4x burst
+(does price continue or snap back?). Analysis tooling only.
+"""
+import glob
+import sys
+
+import pandas as pd
+
+
+def burst_table(parquet_glob: str, bar_minutes: int = 5):
+    files = sorted(glob.glob(parquet_glob))
+    if not files:
+        raise SystemExit(f"no parquet matches {parquet_glob}")
+    df = pd.concat([pd.read_parquet(f) for f in files]).sort_index()
+    df = df[~df.index.duplicated(keep="last")]
+    h, l, c = df["High"], df["Low"], df["Close"]
+    tr = pd.concat([h - l, (h - c.shift()).abs(), (l - c.shift()).abs()],
+                   axis=1).max(axis=1)
+    atr = tr.ewm(alpha=1 / 14, min_periods=14, adjust=False).mean()
+    ratio = (h - l) / atr.shift(1)
+    close_hour = (df.index + pd.Timedelta(minutes=bar_minutes)).hour
+    fwd6 = c.shift(-6) - c  # signed move over the 6 bars after this one
+    out = pd.DataFrame({
+        "bars": ratio.groupby(close_hour).size(),
+        **{f"pct_ge_{k}x": (ratio >= k).groupby(close_hour).mean() * 100
+           for k in (3, 4, 5)},
+        "mean_range": (h - l).groupby(close_hour).mean(),
+        "post4x_drift6": fwd6[ratio >= 4].groupby(close_hour[ratio >= 4]).mean(),
+    })
+    return out.round(3)
+
+
+if __name__ == "__main__":
+    print(burst_table(sys.argv[1],
+                      int(sys.argv[2]) if len(sys.argv) > 2 else 5).to_string())

backtest/research/commit_ladder.py

@@ -0,0 +1,178 @@
+"""Commit-ladder campaign driver (2026-08).
+
+Question: of all signal-code states shipped in the last two months, which
+configuration is the most accurate? Historical in-era backtest numbers are
+not comparable (the backtester's realism model changed on 07-08/07-16 and
+the data feed switched to Oanda on 07-28), so every historical
+configuration is recreated on TODAY's harness via --set overrides and
+replayed on the same pinned Oanda bars. Bug-fix commits stay on; the
+ladder compares shipped tuning decisions only.
+
+Usage:
+    python backtest/research/commit_ladder.py            # full matrix
+    python backtest/research/commit_ladder.py --tf 5min  # one timeframe
+    python backtest/research/commit_ladder.py --windows 12mo,oos
+
+Resumable: a run whose CSV already exists in the output dir is skipped.
+Outputs: backtest/research/commit_ladder_2026-08/<variant>_<window>.csv
+         + .log per run, summary.csv, and a printed ranking table.
+"""
+from __future__ import annotations
+
+import argparse
+import subprocess
+import sys
+from pathlib import Path
+
+import pandas as pd
+
+ROOT = Path(__file__).resolve().parents[2]
+BT = ROOT / "backtest" / "gpu_backtest.py"
+OUT = ROOT / "backtest" / "research" / "commit_ladder_2026-08"
+
+WINDOWS = {
+    "12mo": ("2025-07-30", "2026-07-30"),
+    "oos": ("2024-06-01", "2025-03-30"),
+    "5yr": ("2021-07-30", "2026-07-30"),
+}
+SPLIT_12MO = pd.Timestamp("2026-01-29", tz="UTC")  # book_diff.py's split
+
+# (variant, "commit date what-it-represents", [--set overrides])
+VARIANTS: dict[str, list[tuple[str, str, list[str]]]] = {
+    "5min": [
+        ("M5-A", "dad2432 07-09 gates+trailing+pullback",
+         ["xau_strategy.blocked_hours_utc=[0,1,2,10,11,13,16,21]",
+          "xau_strategy.cp_enabled=false",
+          "signals.trail_dist_r=0.75",
+          "xau_strategy.m5_min_abs_score=30",
+          "xau_strategy.m5_min_confirmation_score=10"]),
+        ("M5-B", "f3ba4d5 07-11 +hour-9 block",
+         ["xau_strategy.blocked_hours_utc=[0,1,2,10,11,13,16,21,9]",
+          "xau_strategy.cp_enabled=false",
+          "signals.trail_dist_r=0.75",
+          "xau_strategy.m5_min_abs_score=30",
+          "xau_strategy.m5_min_confirmation_score=10"]),
+        ("M5-C", "de6424e 07-16 +chart patterns",
+         ["xau_strategy.blocked_hours_utc=[0,1,2,10,11,13,16,21,9]",
+          "signals.trail_dist_r=0.75",
+          "xau_strategy.m5_min_abs_score=30",
+          "xau_strategy.m5_min_confirmation_score=10"]),
+        ("M5-D", "3ee79b5 07-21 +trail dist 0.85",
+         ["xau_strategy.blocked_hours_utc=[0,1,2,10,11,13,16,21,9]",
+          "xau_strategy.m5_min_abs_score=30",
+          "xau_strategy.m5_min_confirmation_score=10"]),
+        ("M5-E", "0396970 07-30 relaxation (HEAD)", []),
+    ],
+    "15min": [
+        ("M15-A", "987ec5b 07-09 pre-campaigns",
+         ["signals.tp_floor_rr=0.5",
+          "xau_strategy.blocked_regimes=[]",
+          "xau_strategy.sell_blocked_hours_utc=[]",
+          "signals.trail_arm_r=0.60"]),
+        ("M15-B", "f3ba4d5/f88e39f 07-11 1R floor + regime block",
+         ["xau_strategy.sell_blocked_hours_utc=[]",
+          "signals.trail_arm_r=0.60"]),
+        ("M15-C", "7d3fd95 07-16 +SELL hour 15/17 block",
+         ["signals.trail_arm_r=0.60"]),
+        ("M15-D", "bf8e5fb 07-21 trail arm 0.70 (HEAD)", []),
+    ],
+    "1h": [
+        ("H1-A", "pre-d95d31a 07-08 TP-tightening model",
+         ["signals.min_rr_ratio=0.4",
+          "signals.tp_tightening_factor=0.8"]),
+        ("H1-B", "HEAD", []),
+    ],
+}
+
+
+def run_one(tf: str, name: str, overrides: list[str], wkey: str,
+            spread: str | None = None) -> Path:
+    suffix = f"_s{spread}" if spread else ""
+    csv_path = OUT / f"{name}_{wkey}{suffix}.csv"
+    log_path = OUT / f"{name}_{wkey}{suffix}.log"
+    if csv_path.exists():
+        print(f"  [skip] {csv_path.name} exists")
+        return csv_path
+    start, end = WINDOWS[wkey]
+    cmd = [sys.executable, str(BT), "--pair", "XAUUSD",
+           "--primary-tf", tf, "--start", start, "--end", end,
+           "--export-csv", str(csv_path)]
+    if spread:
+        cmd += ["--spread", spread]
+    for ov in overrides:
+        cmd += ["--set", ov]
+    print(f"  [run ] {csv_path.name} ...", flush=True)
+    with open(log_path, "w") as lf:
+        rc = subprocess.run(cmd, stdout=lf, stderr=subprocess.STDOUT,
+                            cwd=ROOT).returncode
+    if rc != 0:
+        raise SystemExit(f"backtest failed (rc={rc}) — see {log_path}")
+    log = log_path.read_text()
+    n_ov = log.count("--set override")
+    if n_ov != len(overrides):
+        raise SystemExit(
+            f"{name}_{wkey}: expected {len(overrides)} --set overrides, "
+            f"log shows {n_ov} — see {log_path}")
+    if "Loading cached" not in log and wkey != "sanity":
+        print(f"  [warn] {log_path.name}: no 'Loading cached' line — "
+              f"data was fetched live for at least one timeframe")
+    return csv_path
+
+
+def metrics(csv_path: Path, wkey: str) -> dict:
+    df = pd.read_csv(csv_path, parse_dates=["entry_time"])
+    if df.empty:
+        return {"trades": 0, "total_r": 0.0, "wr": 0.0, "pf": 0.0,
+                "max_dd": 0.0, "h1_r": 0.0, "h2_r": 0.0, "years_pos": ""}
+    if df["entry_time"].dt.tz is None:
+        df["entry_time"] = df["entry_time"].dt.tz_localize("UTC")
+    r = df["r_multiple"]
+    pos, neg = r[r > 0].sum(), r[r < 0].sum()
+    cum = r.cumsum()
+    dd = (pd.concat([pd.Series([0.0]), cum]).cummax()[1:] - cum).max()
+    out = {
+        "trades": len(df),
+        "total_r": round(r.sum(), 1),
+        "wr": round((r > 0).mean() * 100, 1),
+        "pf": round(pos / abs(neg), 2) if neg else float("inf"),
+        "max_dd": round(dd, 1),
+        "h1_r": "", "h2_r": "", "years_pos": "",
+    }
+    if wkey == "12mo":
+        h1 = r[df["entry_time"] < SPLIT_12MO].sum()
+        out["h1_r"] = round(h1, 1)
+        out["h2_r"] = round(r.sum() - h1, 1)
+    if wkey == "5yr":
+        by_year = r.groupby(df["entry_time"].dt.year).sum()
+        out["years_pos"] = f"{(by_year > 0).sum()}/{len(by_year)}"
+    return out
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--tf", choices=list(VARIANTS), default=None)
+    ap.add_argument("--windows", default="12mo,oos,5yr")
+    args = ap.parse_args()
+    OUT.mkdir(parents=True, exist_ok=True)
+    wkeys = [w.strip() for w in args.windows.split(",")]
+    tfs = [args.tf] if args.tf else list(VARIANTS)
+
+    rows = []
+    for tf in tfs:
+        for name, label, overrides in VARIANTS[tf]:
+            for wkey in wkeys:
+                csv_path = run_one(tf, name, overrides, wkey)
+                m = metrics(csv_path, wkey)
+                rows.append({"tf": tf, "variant": name, "represents": label,
+                             "window": wkey, **m})
+                print(f"    {name} {wkey}: {m['trades']} trades, "
+                      f"{m['total_r']}R, PF {m['pf']}, WR {m['wr']}%")
+
+    summary = pd.DataFrame(rows)
+    summary.to_csv(OUT / "summary.csv", index=False)
+    print("\n=== SUMMARY (also in summary.csv) ===")
+    print(summary.to_string(index=False))
+
+
+if __name__ == "__main__":
+    main()

backtest/research/duka_spread.py

@@ -0,0 +1,194 @@
+"""Dukascopy XAU/USD tick download + bid/ask spread study.
+
+Validates ForexBot2's $0.30 round-trip cost assumption (config:
+SignalConfig.round_trip_cost / PAIR_INFO default_spread). Round-trip
+cost of a market entry+exit = one ask-bid spread width (buy at ask,
+sell at bid), so the assumption holds iff typical spread ~ $0.30.
+
+bi5 format: LZMA 'alone' stream of 20-byte big-endian records
+(ms_in_hour uint32, ask uint32, bid uint32, ask_vol f32, bid_vol f32);
+XAUUSD prices scaled x1000. URL months are 0-indexed.
+"""
+import lzma
+import struct
+import sys
+from concurrent.futures import ThreadPoolExecutor
+from datetime import date, timedelta
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import requests
+
+SCALE = 1000.0
+CACHE = Path(__file__).parent / "tick_cache"
+CACHE.mkdir(exist_ok=True)
+URL = "https://datafeed.dukascopy.com/datafeed/XAUUSD/{y}/{m:02d}/{d:02d}/{h:02d}h_ticks.bi5"
+
+# Bot's tradable bar-close hours: session 03:00-23:00 minus XAU blocked
+# hours [0,1,2,10,11,13,16,21] (M5 additionally blocks 9 — reported
+# separately below).
+BLOCKED = {0, 1, 2, 10, 11, 13, 16, 21}
+TRADABLE = sorted(set(range(3, 23)) - BLOCKED)
+
+# ── Day sample ────────────────────────────────────────────────────────
+recent = [date(2026, 7, 6) + timedelta(days=i) for i in range(17)]
+recent = [d for d in recent if d.weekday() < 5]              # Mon-Fri Jul 6-22
+monthly = [date(2025, 8, 13), date(2025, 9, 10), date(2025, 10, 15),
+           date(2025, 11, 12), date(2025, 12, 10), date(2026, 1, 14),
+           date(2026, 2, 11), date(2026, 3, 11), date(2026, 4, 15),
+           date(2026, 5, 13), date(2026, 6, 10)]             # mid-month Wednesdays
+nfp = [date(2026, 6, 5), date(2026, 7, 3)]                   # NFP Fridays
+sundays = [date(2026, 7, 12), date(2026, 7, 19)]             # Sun 22-24 open
+DAYS = sorted(set(recent + monthly + nfp + sundays))
+assert all(d.weekday() < 5 for d in recent + monthly + nfp)
+
+
+_tls = __import__("threading").local()
+
+
+def _session() -> requests.Session:
+    if not hasattr(_tls, "s"):
+        _tls.s = requests.Session()
+        _tls.s.headers["User-Agent"] = "Mozilla/5.0 (research; spread-study)"
+    return _tls.s
+
+
+def fetch(day: date, hour: int) -> Path:
+    p = CACHE / f"{day.isoformat()}_{hour:02d}.bi5"
+    if p.exists():
+        return p
+    u = URL.format(y=day.year, m=day.month - 1, d=day.day, h=hour)
+    last = None
+    for attempt in range(4):
+        try:
+            r = _session().get(u, timeout=30)
+            if r.status_code != 200:
+                raise RuntimeError(f"{u} -> HTTP {r.status_code}")
+            p.write_bytes(r.content)
+            return p
+        except (requests.ConnectionError, requests.Timeout, RuntimeError) as exc:
+            last = exc
+            __import__("time").sleep(1.5 * (attempt + 1))
+            if hasattr(_tls, "s"):
+                _tls.s.close()
+                del _tls.s
+    raise RuntimeError(f"giving up on {u}: {last}")
+
+
+def decode(p: Path, day: date, hour: int) -> pd.DataFrame:
+    blob = p.read_bytes()
+    if not blob:
+        return pd.DataFrame()
+    raw = lzma.decompress(blob)
+    n = len(raw) // 20
+    a = np.frombuffer(raw[: n * 20], dtype=">u4").reshape(n, 5)
+    f = np.frombuffer(raw[: n * 20], dtype=">f4").reshape(n, 5)
+    df = pd.DataFrame({
+        "ms": a[:, 0].astype(np.int64),
+        "ask": a[:, 1] / SCALE,
+        "bid": a[:, 2] / SCALE,
+    })
+    df["spread"] = df["ask"] - df["bid"]
+    df["date"], df["hour"] = day.isoformat(), hour
+    df["weekday"] = day.weekday()
+    return df
+
+
+def load_all() -> pd.DataFrame:
+    jobs = [(d, h) for d in DAYS for h in range(24)]
+    with ThreadPoolExecutor(max_workers=3) as ex:
+        paths = list(ex.map(lambda j: fetch(*j), jobs))
+    frames = [decode(p, d, h) for p, (d, h) in zip(paths, jobs)]
+    df = pd.concat([f for f in frames if not f.empty], ignore_index=True)
+    return df
+
+
+def wstats(g: pd.DataFrame) -> pd.Series:
+    """Tick stats + time-weighted mean (each tick's spread held until the
+    next tick; last tick held to hour end)."""
+    tw_num = tw_den = 0.0
+    keys = [k for k in ("date", "hour") if k in g.columns]
+    for _, grp in g.groupby(keys, sort=False):
+        ms = grp["ms"].to_numpy()
+        sp = grp["spread"].to_numpy()
+        w = np.diff(np.append(ms, 3_600_000)).clip(min=0)
+        tw_num += float((sp * w).sum())
+        tw_den += float(w.sum())
+    s = g["spread"]
+    return pd.Series({
+        "ticks": len(s),
+        "tw_mean": tw_num / tw_den if tw_den else np.nan,
+        "mean": s.mean(), "median": s.median(),
+        "p75": s.quantile(0.75), "p90": s.quantile(0.90),
+        "p99": s.quantile(0.99), "max": s.max(),
+        "pct_le_030": (s <= 0.30).mean() * 100,
+        "pct_ge_060": (s >= 0.60).mean() * 100,
+    })
+
+
+def m5_boundary_spreads(df: pd.DataFrame) -> pd.DataFrame:
+    """Spread of the FIRST tick at/after each M5 boundary — what a
+    next-open fill actually pays."""
+    rows = []
+    for (day, hour), grp in df.groupby(["date", "hour"], sort=False):
+        ms = grp["ms"].to_numpy()
+        sp = grp["spread"].to_numpy()
+        wd = grp["weekday"].iat[0]
+        for b in range(0, 3_600_000, 300_000):
+            i = np.searchsorted(ms, b)
+            if i < len(ms) and ms[i] - b <= 30_000:   # tick within 30 s
+                rows.append((day, wd, hour, b // 60_000, sp[i]))
+    return pd.DataFrame(rows, columns=["date", "weekday", "hour", "minute", "spread"])
+
+
+def main() -> None:
+    df = load_all()
+    print(f"days={df['date'].nunique()}  ticks={len(df):,}  "
+          f"price range {df['bid'].min():.0f}-{df['ask'].max():.0f}")
+
+    print("\n── Overall (all sampled hours) ──")
+    print(wstats(df).round(3).to_string())
+
+    trad = df[df["hour"].isin(TRADABLE) & (df["weekday"] < 5)]
+    print("\n── Bot-tradable hours (session 03-23 minus blocked, Mon-Fri) ──")
+    print(wstats(trad).round(3).to_string())
+
+    roll = df[df["hour"].isin([21, 22, 23])]
+    print("\n── Rollover window 21-24 UTC ──")
+    print(wstats(roll).round(3).to_string())
+
+    print("\n── Per UTC hour ──")
+    per_hour = df[df["weekday"] < 5].groupby("hour").apply(wstats, include_groups=False).round(3)
+    per_hour["tradable"] = ["Y" if h in TRADABLE else "-" for h in per_hour.index]
+    print(per_hour.to_string())
+    per_hour.to_csv(Path(__file__).parent / "duka_spread_by_hour.csv")
+
+    print("\n── M5 next-open boundary fills ──")
+    mb = m5_boundary_spreads(df)
+    mb_trad = mb[mb["hour"].isin(TRADABLE) & (mb["weekday"] < 5)]
+    for name, d in [("all hours", mb), ("tradable hours", mb_trad)]:
+        s = d["spread"]
+        print(f"{name}: n={len(s):,} mean={s.mean():.3f} median={s.median():.3f} "
+              f"p90={s.quantile(0.9):.3f} p99={s.quantile(0.99):.3f} "
+              f"%<=0.30={(s <= 0.30).mean()*100:.1f} %>=0.60={(s >= 0.60).mean()*100:.1f}")
+    mb.to_csv(Path(__file__).parent / "duka_m5_boundary_spreads.csv", index=False)
+
+    print("\n── By month (tradable hours, tick median / tw_mean) ──")
+    trad2 = trad.copy()
+    trad2["month"] = trad2["date"].str[:7]
+    bym = trad2.groupby("month").apply(wstats, include_groups=False).round(3)
+    print(bym[["ticks", "tw_mean", "median", "p90", "pct_le_030", "pct_ge_060"]].to_string())
+
+    print("\n── NFP Fridays 12:00-15:00 UTC vs same hours other days ──")
+    nfp_days = {"2026-06-05", "2026-07-03"}
+    win = df[df["hour"].isin([12, 13, 14])]
+    for name, d in [("NFP", win[win["date"].isin(nfp_days)]),
+                    ("non-NFP", win[~win["date"].isin(nfp_days) & (win["weekday"] < 5)])]:
+        s = d["spread"]
+        print(f"{name}: n={len(s):,} tick-median={s.median():.3f} p90={s.quantile(0.9):.3f} "
+              f"p99={s.quantile(0.99):.3f} max={s.max():.2f}")
+
+
+if __name__ == "__main__":
+    sys.exit(main())

backtest/research/duka_spread_by_hour.csv

@@ -0,0 +1,25 @@
+hour,ticks,tw_mean,mean,median,p75,p90,p99,max,pct_le_030,pct_ge_060,tradable
+0,362893.0,0.749,0.768,0.79,0.82,0.86,1.1,7.69,0.053,94.561,-
+1,540182.0,0.735,0.742,0.74,0.81,0.87,1.0,5.59,0.073,90.449,-
+2,431589.0,0.716,0.722,0.73,0.8,0.86,0.95,2.96,0.1,85.27,-
+3,343952.0,0.708,0.711,0.71,0.79,0.84,0.95,1.67,0.081,82.939,Y
+4,246262.0,0.704,0.702,0.71,0.78,0.82,0.94,1.86,0.098,82.627,Y
+5,388365.0,0.712,0.716,0.72,0.79,0.82,0.95,2.17,0.076,86.745,Y
+6,368999.0,0.65,0.651,0.65,0.69,0.74,0.94,2.32,0.079,71.668,Y
+7,313149.0,0.601,0.598,0.59,0.65,0.69,0.994,3.08,0.239,42.904,Y
+8,347276.0,0.612,0.616,0.6,0.66,0.74,1.0,3.04,0.308,48.349,Y
+9,321619.0,0.612,0.615,0.6,0.66,0.72,1.0,5.95,0.288,50.644,Y
+10,295271.0,0.611,0.618,0.6,0.66,0.71,1.0,9.36,0.164,49.253,-
+11,336963.0,0.619,0.624,0.6,0.67,0.79,1.0,4.97,0.203,50.178,-
+12,476773.0,0.66,0.697,0.63,0.72,0.877,1.367,14.98,0.257,59.633,Y
+13,651425.0,0.641,0.648,0.63,0.69,0.787,1.487,11.407,0.296,60.297,-
+14,673391.0,0.625,0.625,0.63,0.68,0.77,0.9,11.98,0.458,59.085,Y
+15,598398.0,0.636,0.638,0.64,0.7,0.78,0.91,4.49,0.433,66.296,Y
+16,485023.0,0.648,0.644,0.64,0.7,0.757,0.917,4.09,0.118,68.496,-
+17,450119.0,0.626,0.628,0.627,0.7,0.726,0.84,4.08,0.154,63.337,Y
+18,411538.0,0.626,0.63,0.637,0.7,0.71,0.79,4.16,0.107,67.734,Y
+19,435396.0,0.638,0.643,0.65,0.7,0.72,0.896,5.804,0.11,72.007,Y
+20,212654.0,0.684,0.665,0.66,0.7,0.76,1.217,7.15,0.226,70.215,Y
+21,31319.0,0.627,0.622,0.61,0.68,0.75,1.02,2.08,0.089,53.153,-
+22,108879.0,0.829,0.834,0.797,0.84,0.98,2.05,10.11,0.056,95.434,Y
+23,161079.0,1.032,0.913,0.8,0.89,1.307,2.77,13.174,0.092,95.41,-

backtest/research/feed_divergence_2026-08-14.md

@@ -1,81 +0,0 @@
-# Feed Divergence Study — Oanda vs Tiingo vs Twelve Data (Phase 1)
-
-**Date:** 2026-08-14
-**Test plan:** `docs/data_source_change_test_plan.md` (removed 2026-09-06, in git history; the pending Phase 4 shadow scan is noted in CLAUDE.md)
-**Tool:** `scripts/feed_divergence_study.py` (script removed 2026-09-06, in git history) (fetch_range over 365d, parquet-cached)
-**Window:** 2025-08-14 → 2026-08-14 (12 months); Oanda-vs-TD truncated at TD cache head (2026-07-23)
-
-## Headline
-
-Three mid-price feeds for the *same* XAU/USD instrument are **not** interchangeable:
-
-| Pair | mean \|ΔClose\| | p95 \|ΔClose\| | direction agree (M5/M15/H1) | bias | extra bars |
-|---|---|---|---|---|---|
-| Oanda vs **Tiingo** | ~$0.13 | ~$0.29 | 97.7% / 98.3% / 98.5% | ~0 | Tiingo +3–5% (weekend) |
-| Oanda vs **Twelve Data** | ~$3.0 | ~$11.9 | 81.0% / 88.7% / 92.7% | −$1.06 (TD high) | — |
-
-- **Oanda ↔ Tiingo are near-identical** on shared bars (~1–2% of ATR on close), but Tiingo emits ~22 hours of Sunday bars/week that Oanda omits.
-- **Oanda ↔ Twelve Data diverge massively** — ~$3 mean close gap (≈30% of M5 ATR, ~30% of M15 ATR), a systematic ~$1 bias (TD quoted ~$1 high), and 19% of M5 bars even disagree on bar *direction*. This is the documented "Oanda/TD divergence" that motivated re-pinning the books on Oanda bars.
-
-## Oanda vs Tiingo — detailed
-
-Both feeds: same bar grid, same head/tail timestamps. Tiingo simply has more bars (Sunday bars).
-
-### Coverage (12 months)
-
-| TF | Oanda bars | Tiingo bars | shared | Oanda-only | Tiingo-only |
-|---|---|---|---|---|---|
-| M5 | 70,901 | 73,241 | 70,807 (96.6%) | 94 (0.13%) | 2,434 (3.32%) |
-| M15 | 23,640 | 24,922 | 23,615 (94.7%) | 25 (0.11%) | 1,307 (5.24%) |
-| H1 | 5,912 | 6,246 | 5,909 (94.6%) | 3 (0.05%) | 337 (5.40%) |
-| H4 | 1,597 | 1,657 | 1,597 (96.4%) | 0 | 60 (3.62%) |
-
-- **Tiingo-only bars cluster exclusively on Sundays** (e.g. M15: 88/Sunday; H1: 22/Sunday; H4: 5/Sunday). Tiingo's FX endpoint resamples continuously and does **not** apply the FX weekend calendar — it emits ~22h of flat/stale Sunday bars that Oanda omits.
-- Oanda-only bars cluster on holidays (2025-11-27 Thanksgiving, 2025-12-10, 2026-06-25) — a handful of bars, negligible.
-
-### Per-bar OHLC diffs (Oanda − Tiingo, $)
-
-| TF | Close mean | Close p95 | Close max | \|ΔClose\| / ATR14 (mean / p95) |
-|---|---|---|---|---|
-| M5 | $0.123 | $0.285 | $26.10 | 2.1% / 5.0% |
-| M15 | $0.128 | $0.290 | $26.10 | 1.3% / 2.9% |
-| H1 | $0.145 | $0.335 | $26.10 | 0.7% / 1.6% |
-| H4 | $0.168 | $0.340 | $17.53 | 0.4% / 0.8% |
-
-- Bias is ~zero on every TF (no systematic one-feed-high).
-- Tail outliers (p99/max up to $26–88) occur during fast moves — the two feeds' bar extremes disagree on the exact tick, as expected.
-- **Direction agreement is 97.7–98.4%**, return correlation 0.995–0.9997.
-- Divergence is concentrated at the Friday-close / Sunday-reopen boundary (UTC hours 20–22; mean |ΔClose| ~2–4× the 08–19h baseline).
-
-### Phase 1 pass/fail vs plan thresholds
-
-| Criterion | Plan threshold | Result | Verdict |
-|---|---|---|---|
-| timestamp overlap | ≥ 99.5% | 94.6–96.6% | **FAIL** (Tiingo weekend bars) |
-| Close p95 \|Δ\| | ≤ 0.01% of price | ~0.0067% | PASS |
-| no systematic bias | mean Δ ≈ 0 | ~0 | PASS |
-
-The overlap "failure" is a *coverage* difference, not a data-quality one: the shared bars agree tightly, but Tiingo's extra Sunday bars mean the feeds have a different bar *count*. The signal pipeline's own `_is_forex_weekend` filter (NY-anchored) gates those Sunday bars, so live impact depends on whether any code trusts the feed's raw bar set directly.
-
-## Oanda vs Twelve Data — detailed (M5/M15/H1, cached TD parquets)
-
-| TF | common | mean \|ΔClose\| | median \|ΔClose\| | p95 | max | bias (TD−Oanda) | dir agree |
-|---|---|---|---|---|---|---|---|
-| M5 | 66,246 | $2.999 | $1.118 | $11.934 | $101.85 | +1.05 | 80.96% |
-| M15 | 22,090 | $2.951 | $1.051 | $11.848 | $101.85 | +1.08 | 88.71% |
-| H1 | 5,526 | $2.902 | $0.985 | $11.908 | $70.94 | +1.06 | 92.67% |
-
-- The ~$3 mean / ~$1 median / ~$12 p95 gap is **~23× larger** than the Oanda–Tiingo gap, and the ~$1 bias is systematic (Twelve Data quotes gold ~$1 high vs Oanda's true mid).
-- M5 direction agreement of **81%** means one bar in five disagrees on direction — M5 signals on TD bars are effectively a *different instrument* than on Oanda bars.
-- This matches the prior Oanda/TD M15 study of 2026-07-24 (`oanda_td_divergence.py`, removed 2026-09-06 as superseded by this report; mean |ΔClose| ~$3.3) and confirms the standing rule: **pinned Twelve-Data books are not comparable to Oanda live signals without a re-pin.**
-
-## Conclusions for the data-source switch
-
-1. **Oanda → Tiingo (or back) is signal-safe in magnitude** (price diffs ~1–2% of ATR, 98% direction agreement), *provided* the weekend-bar coverage difference is handled — Tiingo's Sunday bars must be gated by the aggregator's NY-anchored weekend filter, not by raw bar count.
-2. **Oanda → Twelve Data is a recalibration, not a formality.** Phase 2/3 must quantify the actual signal/book delta; the $3/$1 bias alone guarantees a large fraction of marginal bars flip.
-3. Re-pinning the books on Oanda bars (Phase 3) remains mandatory before any TD-pinned baseline is treated as a live expectation.
-
-## Next steps
-
-- Phase 2: signal-level replay parity (`analyze_from_dataframes` on both feeds) to turn price-diff stats into flip rates.
-- Phase 3: backtest re-pin on Oanda bars at $0.30 and $0.65.

backtest/research/fetch_oanda_m15.py

@@ -0,0 +1,158 @@
+"""Fetch XAU/USD M15 candles from the Oanda v20 REST API.
+
+Mechanical data-fetching infrastructure only — no analysis here.
+Paginates GET /v3/instruments/XAU_USD/candles from a start cursor
+through "now" and writes the result to a parquet file.
+
+Run from the repo root:
+    venv/bin/python backtest/research/fetch_oanda_m15.py
+"""
+import sys
+import time
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+import pandas as pd
+import requests
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from app_config import oanda_api_token, oanda_base_url  # noqa: E402
+
+INSTRUMENT = "XAU_USD"
+GRANULARITY = "M15"
+START = datetime(2025, 7, 1, 0, 0, 0, tzinfo=timezone.utc)
+OUT_PATH = Path(__file__).parent / "oanda_xau_m15_20250701_now.parquet"
+COUNT = 5000
+REQUEST_SLEEP_S = 0.5
+FRESHNESS_WINDOW = timedelta(minutes=30)
+
+
+def _rfc3339(dt: datetime) -> str:
+    return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+
+
+def fetch_all_candles():
+    token = oanda_api_token()
+    base_url = oanda_base_url()
+    if not token:
+        raise RuntimeError(
+            "No Oanda API token resolved (env OANDA_API_TOKEN / "
+            "config.json oanda.api_token). Aborting.")
+
+    url = f"{base_url}/v3/instruments/{INSTRUMENT}/candles"
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Accept-Datetime-Format": "RFC3339",
+    }
+
+    cursor = START
+    include_first = True
+    all_rows = []
+    n_requests = 0
+
+    while True:
+        params = {
+            "granularity": GRANULARITY,
+            "price": "MBA",
+            "count": COUNT,
+            "dailyAlignment": 0,
+            "alignmentTimezone": "Etc/UTC",
+            "from": _rfc3339(cursor),
+            "includeFirst": "true" if include_first else "false",
+        }
+
+        resp = requests.get(url, headers=headers, params=params, timeout=30)
+        n_requests += 1
+        if resp.status_code != 200:
+            raise RuntimeError(
+                f"Oanda API request failed: status={resp.status_code}\n"
+                f"URL: {resp.url}\n"
+                f"Response body: {resp.text}")
+
+        payload = resp.json()
+        candles = payload.get("candles", [])
+
+        for c in candles:
+            if not c.get("complete", False):
+                continue
+            mid = c["mid"]
+            bid = c["bid"]
+            ask = c["ask"]
+            all_rows.append({
+                "Datetime": pd.Timestamp(c["time"]).tz_convert("UTC")
+                if pd.Timestamp(c["time"]).tzinfo is not None
+                else pd.Timestamp(c["time"]).tz_localize("UTC"),
+                "Open": float(mid["o"]),
+                "High": float(mid["h"]),
+                "Low": float(mid["l"]),
+                "Close": float(mid["c"]),
+                "Volume": int(c["volume"]),
+                "bid_o": float(bid["o"]),
+                "bid_h": float(bid["h"]),
+                "bid_l": float(bid["l"]),
+                "bid_c": float(bid["c"]),
+                "ask_o": float(ask["o"]),
+                "ask_h": float(ask["h"]),
+                "ask_l": float(ask["l"]),
+                "ask_c": float(ask["c"]),
+            })
+
+        print(f"  request {n_requests}: from={params['from']} "
+              f"got {len(candles)} candles (complete kept so far: "
+              f"{len(all_rows)})", file=sys.stderr)
+
+        if not candles:
+            break
+
+        last_time = pd.Timestamp(candles[-1]["time"])
+        if last_time.tzinfo is None:
+            last_time = last_time.tz_localize("UTC")
+        else:
+            last_time = last_time.tz_convert("UTC")
+
+        now = datetime.now(timezone.utc)
+        # Oanda's "count" includes the "from" boundary bar itself, so with
+        # includeFirst=false every full page comes back as count-1, not
+        # count (verified empirically: count=N, includeFirst=false always
+        # yields N-1 candles when a full page of data exists ahead of the
+        # cursor). Compare against the count actually expected for this
+        # request's includeFirst setting, not the raw requested count.
+        expected_full_page = COUNT if include_first else COUNT - 1
+        got_fewer_than_requested = len(candles) < expected_full_page
+        near_now = (now - last_time.to_pydatetime()) < FRESHNESS_WINDOW
+
+        if got_fewer_than_requested or near_now:
+            break
+
+        cursor = last_time.to_pydatetime()
+        include_first = False
+        time.sleep(REQUEST_SLEEP_S)
+
+    return all_rows, n_requests
+
+
+def main():
+    print(f"Fetching {INSTRUMENT} {GRANULARITY} candles from "
+          f"{_rfc3339(START)} through now...", file=sys.stderr)
+
+    rows, n_requests = fetch_all_candles()
+
+    if not rows:
+        raise RuntimeError("No complete candles were fetched.")
+
+    df = pd.DataFrame(rows)
+    df = df.set_index("Datetime")
+    df = df.sort_index()
+    df = df[~df.index.duplicated(keep="last")]
+
+    df.to_parquet(OUT_PATH)
+
+    print(f"\nSaved {len(df)} rows to {OUT_PATH}", file=sys.stderr)
+    print(f"Requests made: {n_requests}", file=sys.stderr)
+    print(f"First timestamp: {df.index[0]}", file=sys.stderr)
+    print(f"Last timestamp: {df.index[-1]}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    main()

backtest/research/ff_calendar_scraper.py

@@ -0,0 +1,144 @@
+"""Forex Factory calendar scraper -> backtest/cache/calendar.parquet.
+
+Fetches weekly calendar pages and parses the event JSON embedded in
+`window.calendarComponentStates` (the `days:` array). Event `dateline`
+is a unix epoch, so timestamps carry no timezone ambiguity — verified
+2026-07-11 against known release times (NFP 12:30/13:30 UTC across DST,
+ECB 12:15/13:15) and against EUR/USD M15 volatility: the bar containing
+a high-impact event averages 3.5x normal range, and shifting event
+times by ±1h destroys that alignment.
+
+Usage:
+    python backtest/research/ff_calendar_scraper.py --start 2024-01-01 --end 2026-07-13
+    python backtest/research/ff_calendar_scraper.py            # refresh: last stored week -> today
+
+Merges into the existing parquet on event (id, dateline), so refreshing
+is idempotent and also picks up revised `actual` values for recent weeks.
+Be polite: ~1.6s between requests (a full year is ~52 requests).
+"""
+import argparse
+import json
+import time
+from datetime import date, timedelta
+from pathlib import Path
+from typing import List, Optional
+
+import pandas as pd
+import requests
+
+CACHE_DIR = Path(__file__).resolve().parent.parent / "cache"
+PARQUET = CACHE_DIR / "calendar.parquet"
+
+UA = ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+      "(KHTML, like Gecko) Chrome/126.0 Safari/537.36")
+
+FIELDS = ["id", "dateline", "name", "country", "currency", "impactName",
+          "timeLabel", "timeMasked", "actual", "forecast", "previous",
+          "revision", "actualBetterWorse"]
+
+
+def week_url(d: date) -> str:
+    return (f"https://www.forexfactory.com/calendar?week="
+            f"{d.strftime('%b').lower()}{d.day}.{d.year}")
+
+
+def extract_days(html: str) -> Optional[list]:
+    """Balanced-bracket extraction of the `days:` JSON array."""
+    i = html.find("days:")
+    if i < 0:
+        return None
+    j = html.find("[", i)
+    depth, in_str, esc = 0, False, False
+    for k in range(j, len(html)):
+        ch = html[k]
+        if in_str:
+            if esc:
+                esc = False
+            elif ch == "\\":
+                esc = True
+            elif ch == '"':
+                in_str = False
+        else:
+            if ch == '"':
+                in_str = True
+            elif ch == "[":
+                depth += 1
+            elif ch == "]":
+                depth -= 1
+                if depth == 0:
+                    return json.loads(html[j:k + 1])
+    return None
+
+
+def fetch_week(sess: requests.Session, d: date, retries: int = 4) -> Optional[List[dict]]:
+    for attempt in range(retries):
+        try:
+            r = sess.get(week_url(d), timeout=30)
+            if r.status_code == 200:
+                days = extract_days(r.text)
+                if days is not None:
+                    return [{k: e.get(k) for k in FIELDS}
+                            for day in days for e in day.get("events", [])]
+        except requests.RequestException:
+            pass
+        time.sleep(5 * (attempt + 1))
+    return None
+
+
+def scrape(start: date, end: date) -> pd.DataFrame:
+    sess = requests.Session()
+    sess.headers["User-Agent"] = UA
+    rows: List[dict] = []
+    failed: List[date] = []
+    d = start
+    while d <= end:
+        week_rows = fetch_week(sess, d)
+        if week_rows is None:
+            failed.append(d)
+            print(f"FAILED week {d}")
+        else:
+            rows.extend(week_rows)
+        d += timedelta(days=7)
+        time.sleep(1.6)
+    if failed:
+        print(f"WARNING: {len(failed)} weeks failed: {failed}")
+    return pd.DataFrame(rows)
+
+
+def merge_into_store(new: pd.DataFrame, path: Path = PARQUET) -> pd.DataFrame:
+    if path.exists():
+        old = pd.read_parquet(path)
+        combined = pd.concat([old, new])
+    else:
+        combined = new
+    # keep="last": a refresh overwrites stale rows (actuals fill in
+    # after release, revisions land weeks later).
+    combined = (combined.drop_duplicates(subset=["id", "dateline"], keep="last")
+                .sort_values("dateline").reset_index(drop=True))
+    combined.to_parquet(path, index=False)
+    return combined
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--start", type=date.fromisoformat, default=None)
+    ap.add_argument("--end", type=date.fromisoformat, default=date.today())
+    args = ap.parse_args()
+
+    start = args.start
+    if start is None:
+        if PARQUET.exists():
+            last = pd.read_parquet(PARQUET).dateline.max()
+            start = pd.Timestamp(last, unit="s").date() - timedelta(days=7)
+        else:
+            start = date(2024, 1, 1)
+    print(f"Scraping {start} -> {args.end}")
+    new = scrape(start, args.end)
+    combined = merge_into_store(new)
+    print(f"Store: {len(combined)} events "
+          f"({pd.Timestamp(combined.dateline.min(), unit='s')} -> "
+          f"{pd.Timestamp(combined.dateline.max(), unit='s')}) -> {PARQUET}")
+
+
+if __name__ == "__main__":
+    main()

backtest/research/hour_analysis.py

@@ -0,0 +1,40 @@
+"""Per-hour P&L breakdown of a backtest trade book.
+
+Usage:
+    python backtest/research/hour_analysis.py BOOK.csv BAR_MINUTES
+
+Reports, per UTC hour (both entry hour and DECISION hour = entry_time
+minus one bar, the clock blocked_hours_utc evaluates): trade count,
+total R, mean R, win rate, profit factor. Also a per-(hour, direction)
+table. Analysis tooling only — never imported by the live code.
+"""
+import sys
+
+import pandas as pd
+
+
+def hour_tables(book_csv: str, bar_minutes: int):
+    df = pd.read_csv(book_csv)
+    et = pd.to_datetime(df["entry_time"], utc=True, format="mixed")
+    df["entry_hour"] = et.dt.hour
+    df["decision_hour"] = (et - pd.Timedelta(minutes=bar_minutes)).dt.hour
+
+    def table(group_cols):
+        g = df.groupby(group_cols)["r_multiple"]
+        out = g.agg(count="count", total_r="sum", mean_r="mean",
+                    wr=lambda r: (r > 0).mean() * 100)
+        out["pf"] = g.apply(
+            lambda r: r[r > 0].sum() / max(-r[r <= 0].sum(), 1e-9))
+        return out.round(2)
+
+    return (table("decision_hour"), table(["decision_hour", "direction"]),
+            table("entry_hour"))
+
+
+if __name__ == "__main__":
+    book, mins = sys.argv[1], int(sys.argv[2])
+    by_dec, by_dec_dir, by_ent = hour_tables(book, mins)
+    print(f"\n── {book} · by DECISION hour (bar-close clock) ──")
+    print(by_dec.to_string())
+    print(f"\n── by (decision hour, direction) ──")
+    print(by_dec_dir.to_string())

backtest/research/llm_news_eval.py

@@ -0,0 +1,697 @@
+"""
+DeepSeek news-analysis validation harness — XAU/USD.
+====================================================
+Research-only (no production code changes). Answers: is the DeepSeek
+layer's news analysis *correct*?
+
+Part A (events): replays every high-impact USD release with a genuine
+surprise (2024-01 → 2026-07, from backtest/cache/calendar.parquet)
+through the REAL `NewsEngine._llm_event_direction` — real prompt, real
+payload, real validation — and scores its "gold bullish/bearish" calls
+against gold's actual move (Oanda M15 mids) at +15m/30m/1h/4h/24h.
+Baselines: mechanical surprise-sign doctrine, FF actualBetterWorse,
+always-bullish/bearish (2024-26 is a gold bull regime), coin flip.
+
+Part B (headlines): a curated golden set (llm_news_golden.csv) through
+the REAL `NewsEngine._llm_headline_scores` in production-sized batches,
+vs the keyword scorer it overrides. Includes repeat-batch (determinism)
+and shuffled-order (cross-contamination) probes.
+
+Implementation notes that keep the replay faithful to production:
+  * `_llm_event_direction`/`_llm_headline_scores` resolve the client via
+    `get_llm_client()`, so we seed the module singleton with a research
+    client (raised daily cap: 586 events > production cap 500).
+  * The event cache key omits the forecast, so a repeat `actual` would
+    silently re-serve a direction computed for a different forecast —
+    the cache is cleared before every call.
+  * A "neutral" reply and a hard failure both return None; they are
+    disambiguated by reading the event cache back after the call
+    (absent = failure, "" = neutral, "bullish:/bearish: …" = call).
+  * Raw request/response bytes are captured with a requests response
+    hook — the request-building code path is untouched.
+
+Usage (from repo root):
+  venv/bin/python backtest/research/llm_news_eval.py events --limit 5
+  venv/bin/python backtest/research/llm_news_eval.py events --determinism 20
+  venv/bin/python backtest/research/llm_news_eval.py headlines
+  venv/bin/python backtest/research/llm_news_eval.py report
+
+Artifacts land in backtest/research/llm_news_eval_2026-08/ (JSONL replay
+logs are resume-safe: completed events are never re-paid). `report` is a
+pure offline function of the artifacts.
+"""
+
+import argparse
+import json
+import math
+import random
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(REPO_ROOT))
+
+import llm_client
+from app_config import deepseek_api_key, get_app_config
+from config import NewsConfig
+from llm_client import DEFAULT_MODEL, DeepSeekClient
+from news_engine import EconomicEvent, NewsEngine
+
+RESEARCH_DIR = Path(__file__).parent
+ART_DIR = RESEARCH_DIR / "llm_news_eval_2026-08"
+CALENDAR = REPO_ROOT / "backtest" / "cache" / "calendar.parquet"
+BARS = (REPO_ROOT / "backtest" / "cache" /
+        "oanda_XAU_USD_15min_20210531_20260804.parquet")
+GOLDEN = RESEARCH_DIR / "llm_news_golden.csv"
+EVENTS_JSONL = ART_DIR / "events_replay.jsonl"
+HEADLINES_JSONL = ART_DIR / "headlines_replay.jsonl"
+SCORED_CSV = ART_DIR / "events_scored.csv"
+
+HORIZONS = [("15m", 15), ("30m", 30), ("1h", 60), ("4h", 240),
+            ("24h", 1440)]
+NEUTRAL_BAND = 0.15  # |xau| below this = neutral prediction (Part B)
+BATCH_CAP = 25       # production batch cap (news_engine.py:783)
+EXPECTED_CORPUS = 586
+
+
+# ── shared plumbing ──────────────────────────────────────────────────
+
+class _Recorder:
+    """requests response hook capturing the exact wire traffic."""
+
+    def __init__(self):
+        self.last = None
+
+    def hook(self, resp, **kwargs):
+        body = resp.request.body
+        if isinstance(body, bytes):
+            body = body.decode("utf-8", "replace")
+        self.last = {
+            "request_body": body,
+            "http_status": resp.status_code,
+            "response_body": resp.text,
+            "elapsed_s": round(resp.elapsed.total_seconds(), 3),
+        }
+        return resp
+
+    def pop(self):
+        cap, self.last = self.last, None
+        return cap or {}
+
+
+def install_research_client(timeout: float):
+    """Build a research DeepSeekClient (raised cap) and seed the module
+    singleton so the production code paths use it. Returns (client,
+    recorder)."""
+    app = get_app_config()
+    key = deepseek_api_key(app)
+    ds = app.deepseek_settings
+    if not key:
+        sys.exit("No DeepSeek API key (env DEEPSEEK_API_KEY or "
+                 "config.json deepseek.api_key) — aborting before spend.")
+    if ds.get("enabled") is False:
+        sys.exit("deepseek.enabled is false in config.json — aborting.")
+    client = DeepSeekClient(
+        api_key=key,
+        model=str(ds.get("model", DEFAULT_MODEL)),
+        timeout=timeout,
+        daily_cap=10 ** 6,  # research: corpus 586 > production cap 500
+    )
+    rec = _Recorder()
+    client._session.hooks["response"].append(rec.hook)
+    with llm_client._client_lock:
+        llm_client._client = client
+    return client, rec
+
+
+def _s(x):
+    """Parquet str columns carry float NaN — EconomicEvent needs None."""
+    return None if pd.isna(x) else str(x)
+
+
+def load_corpus():
+    """High-impact USD releases with a genuine surprise, through the
+    production `EconomicEvent.surprise` gate. Asserts run before any
+    API spend."""
+    df = pd.read_parquet(CALENDAR)
+    df = df[(df["impactName"] == "high") & (df["currency"] == "USD")]
+    rows = []
+    for r in df.itertuples():
+        ev = EconomicEvent(
+            time=datetime.fromtimestamp(int(r.dateline), tz=timezone.utc),
+            currency="USD", impact="high", event=_s(r.name) or "",
+            forecast=_s(r.forecast), previous=_s(r.previous),
+            actual=_s(r.actual),
+        )
+        if ev.surprise not in ("positive", "negative"):
+            continue
+        rows.append({
+            "ffid": int(r.id), "dateline": int(r.dateline),
+            "utc_time": ev.time.isoformat(), "name": ev.event,
+            "actual": ev.actual, "forecast": ev.forecast,
+            "previous": ev.previous, "surprise": ev.surprise,
+            "better_worse": int(r.actualBetterWorse), "ev": ev,
+        })
+    ids = [d["ffid"] for d in rows]
+    if len(rows) != EXPECTED_CORPUS:
+        sys.exit(f"corpus size {len(rows)} != {EXPECTED_CORPUS} — "
+                 "calendar.parquet changed; re-verify before spending.")
+    if len(set(ids)) != len(ids):
+        sys.exit("FF ids not unique — resume keying would break.")
+    # Pin the actualBetterWorse mapping on two unambiguous NFP prints.
+    nfp = {d["utc_time"][:10]: d["better_worse"] for d in rows
+           if d["name"] == "Non-Farm Employment Change"}
+    if nfp.get("2024-01-05") != 1 or nfp.get("2024-05-03") != 2:
+        sys.exit("actualBetterWorse mapping pins failed (expected "
+                 "1=better-for-USD on 2024-01-05 beat, 2=worse on "
+                 "2024-05-03 miss).")
+    rows.sort(key=lambda d: (d["dateline"], d["ffid"]))
+    return rows
+
+
+def _load_jsonl(path: Path):
+    recs = []
+    if path.exists():
+        with open(path) as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    recs.append(json.loads(line))
+    return recs
+
+
+# ── Part A: event replay ─────────────────────────────────────────────
+
+def _extract_reason(cap):
+    try:
+        content = json.loads(cap["response_body"])[
+            "choices"][0]["message"]["content"]
+        return json.loads(content).get("reason")
+    except Exception:
+        return None
+
+
+def cmd_events(args):
+    ART_DIR.mkdir(exist_ok=True)
+    corpus = load_corpus()
+    if args.limit:
+        corpus = corpus[:args.limit]
+    engine = NewsEngine(NewsConfig())
+    existing = _load_jsonl(EVENTS_JSONL)
+    done1 = {r["ffid"] for r in existing
+             if r.get("run_pass") == 1 and r.get("llm_direction")}
+    done2 = {r["ffid"] for r in existing
+             if r.get("run_pass") == 2 and r.get("llm_direction")}
+    pending = [d for d in corpus if d["ffid"] not in done1]
+    print(f"corpus {len(corpus)} | already recorded {len(done1)} | "
+          f"pending {len(pending)} | determinism re-calls "
+          f"{args.determinism} | model calls are ~100 tokens each "
+          f"(deepseek-v4-flash, static-prefix cached) → cost ≈ cents")
+    if not pending and not args.determinism:
+        print("nothing to do")
+        return
+    _, rec = install_research_client(args.timeout)
+
+    out = open(EVENTS_JSONL, "a")
+
+    def call_one(d, run_pass):
+        # Cache key omits forecast → cleared per call so 196 colliding
+        # corpus rows can't be served a stale direction.
+        with NewsEngine._cache_lock:
+            NewsEngine._llm_event_cache.clear()
+        ev = d["ev"]
+        engine._llm_event_direction(ev)
+        key = (ev.event, ev.currency, str(ev.actual or ""))
+        with NewsEngine._cache_lock:
+            cached = NewsEngine._llm_event_cache.get(key)
+        cap = rec.pop()
+        if cached is None:
+            direction, text = None, None       # hard failure → retry
+        elif cached == "":
+            direction, text = "neutral", ""
+        else:
+            direction, text = cached.split(":", 1)[0], cached
+        recd = {k: d[k] for k in (
+            "ffid", "dateline", "utc_time", "name", "actual", "forecast",
+            "previous", "surprise", "better_worse")}
+        recd.update({
+            "llm_direction": direction, "llm_text": text,
+            "reason_raw": _extract_reason(cap), "run_pass": run_pass,
+            "ts": datetime.now(timezone.utc).isoformat(), **cap,
+        })
+        out.write(json.dumps(recd) + "\n")
+        out.flush()
+        return direction
+
+    remaining = pending
+    for npass in range(1, args.retry_passes + 1):
+        if not remaining:
+            break
+        failed = []
+        for i, d in enumerate(remaining):
+            if call_one(d, 1) is None:
+                failed.append(d)
+            if args.sleep:
+                time.sleep(args.sleep)
+            if (i + 1) % 25 == 0:
+                print(f"  pass {npass}: {i + 1}/{len(remaining)} "
+                      f"({len(failed)} failed)")
+        print(f"pass {npass} done: {len(remaining) - len(failed)} ok, "
+              f"{len(failed)} failed")
+        remaining = failed
+    if remaining:
+        print(f"WARNING: {len(remaining)} events still unrecorded after "
+              f"{args.retry_passes} passes — re-run `events` to resume.")
+
+    if args.determinism:
+        ok_ids = {x["ffid"] for x in remaining}
+        done_now = [d for d in corpus if d["ffid"] not in ok_ids]
+        done_now.sort(key=lambda d: (d["name"], d["dateline"]))
+        stride = max(1, len(done_now) // args.determinism)
+        sample = done_now[::stride][:args.determinism]
+        todo = [d for d in sample if d["ffid"] not in done2]
+        print(f"determinism probe: {len(todo)} re-calls "
+              f"({len(sample) - len(todo)} already recorded)")
+        for d in todo:
+            call_one(d, 2)
+            if args.sleep:
+                time.sleep(args.sleep)
+    out.close()
+    print("events replay complete →", EVENTS_JSONL)
+
+
+# ── Part B: golden headlines ─────────────────────────────────────────
+
+def _load_golden():
+    g = pd.read_csv(GOLDEN)
+    need = {"headline", "category", "expected_xau_sign", "rationale"}
+    if not need <= set(g.columns):
+        sys.exit(f"golden CSV missing columns {need - set(g.columns)}")
+    if not g["expected_xau_sign"].isin([-1, 0, 1]).all():
+        sys.exit("expected_xau_sign must be -1/0/+1")
+    if not g["headline"].is_unique:
+        sys.exit("duplicate headlines in golden CSV")
+    return g
+
+
+def cmd_headlines(args):
+    ART_DIR.mkdir(exist_ok=True)
+    g = _load_golden()
+    heads = list(g["headline"])
+    _, rec = install_research_client(args.timeout)
+    engine = NewsEngine(NewsConfig())
+    out = open(HEADLINES_JSONL, "w")  # tiny (~9 calls): fresh each run
+
+    def run_batch(texts, tag, batch_idx):
+        scores = None
+        for attempt in range(3):
+            scores = engine._llm_headline_scores(texts)
+            cap = rec.pop()
+            if scores is not None:
+                break
+            print(f"  {tag} batch {batch_idx} attempt {attempt + 1} "
+                  f"failed (http {cap.get('http_status')})")
+            time.sleep(2.0)
+        out.write(json.dumps({
+            "run_tag": tag, "batch": batch_idx, "headlines": texts,
+            "scores": scores,
+            "ts": datetime.now(timezone.utc).isoformat(), **cap,
+        }) + "\n")
+        out.flush()
+        return scores
+
+    batches = [heads[i:i + BATCH_CAP] for i in range(0, len(heads),
+                                                     BATCH_CAP)]
+    for bi, b in enumerate(batches):
+        ok = run_batch(b, "main", bi) is not None
+        print(f"main batch {bi} ({len(b)} headlines): "
+              f"{'ok' if ok else 'FAILED'}")
+        time.sleep(args.sleep)
+    run_batch(batches[0], "repeat", 0)
+    print("repeat probe done")
+    shuffled = heads[:]
+    random.Random(20260806).shuffle(shuffled)
+    for bi in range(0, len(shuffled), BATCH_CAP):
+        run_batch(shuffled[bi:bi + BATCH_CAP], "shuffled",
+                  bi // BATCH_CAP)
+    print("shuffled probe done →", HEADLINES_JSONL)
+    out.close()
+
+
+# ── report: pure offline function of the artifacts ───────────────────
+
+def _binom_p(k, n):
+    """Exact two-sided binomial p-value vs p=0.5 (no scipy)."""
+    if n == 0:
+        return float("nan")
+    dist = [math.comb(n, i) for i in range(n + 1)]
+    tot = float(2 ** n)
+    pk = dist[k]
+    return min(1.0, sum(c for c in dist if c <= pk) / tot)
+
+
+def _mcnemar_p(hit_a, hit_b):
+    """Exact McNemar via binomial on discordant pairs. Inputs: aligned
+    boolean lists."""
+    b = sum(1 for x, y in zip(hit_a, hit_b) if x and not y)
+    c = sum(1 for x, y in zip(hit_a, hit_b) if not x and y)
+    return b, c, _binom_p(min(b, c), b + c)
+
+
+def _latest_events(recs):
+    """Last pass-1 record per ffid (retries append; last wins)."""
+    out = {}
+    for r in recs:
+        if r.get("run_pass") == 1:
+            out[r["ffid"]] = r
+    return list(out.values())
+
+
+def _fmt_rate(hits, n, label=""):
+    if n == 0:
+        return f"{label:<24} n=0"
+    p = _binom_p(hits, n)
+    return (f"{label:<24} {hits:>4}/{n:<4} = {hits / n * 100:5.1f}%  "
+            f"(binom p={p:.3f})")
+
+
+def report_events():
+    recs = _latest_events(_load_jsonl(EVENTS_JSONL))
+    if not recs:
+        print("no event replay records — run `events` first")
+        return
+    bars = pd.read_parquet(BARS)
+    closes = bars["Close"].to_numpy()
+    close_ns = (bars.index + pd.Timedelta(minutes=15)).asi8
+
+    def anchor(t_ns, h_min):
+        i = np.searchsorted(close_ns, t_ns, side="right") - 1
+        if i < 0:
+            return None
+        stale_ns = max(45.0, h_min / 4.0) * 60e9
+        if t_ns - close_ns[i] > stale_ns:
+            return None
+        return closes[i]
+
+    rows, skip = [], {h: 0 for h, _ in HORIZONS}
+    base_skipped = 0
+    for r in recs:
+        t_ns = int(r["dateline"]) * 10 ** 9
+        base = anchor(t_ns, 0)
+        if base is None:
+            base_skipped += 1
+            continue
+        row = dict(r)
+        row.pop("request_body", None)
+        row.pop("response_body", None)
+        row["year"] = r["utc_time"][:4]
+        row["base"] = base
+        for hname, hmin in HORIZONS:
+            p = anchor(t_ns + hmin * 60 * 10 ** 9, hmin)
+            if p is None:
+                skip[hname] += 1
+                row[f"d_{hname}"] = np.nan
+            else:
+                row[f"d_{hname}"] = p - base
+        rows.append(row)
+    df = pd.DataFrame(rows)
+    df.to_csv(SCORED_CSV, index=False)
+
+    n_fail = int((df["llm_direction"].isna()).sum())
+    print("═" * 70)
+    print("PART A — event gold-direction vs actual XAU move")
+    print("═" * 70)
+    print(f"events replayed: {len(df)} (baseline-price skipped: "
+          f"{base_skipped}) | LLM hard failures: {n_fail}")
+    print("horizon skip counts (stale/closed):",
+          {k: v for k, v in skip.items() if v})
+    dist = df["llm_direction"].value_counts(dropna=False).to_dict()
+    print("LLM call distribution:", dist)
+
+    # Unconditional per-horizon noise scale (null): median |Δ| over all
+    # 2024+ bar times, position-shifted.
+    bars24 = bars[bars.index >= "2024-01-01"]
+    null_med = {}
+    for hname, hmin in HORIZONS:
+        k = hmin // 15
+        d = (bars24["Close"].shift(-k) - bars24["Close"]).abs().dropna()
+        null_med[hname] = float(d.median())
+    print("unconditional null median |Δ| ($):",
+          {k: round(v, 2) for k, v in null_med.items()})
+
+    # Strategy call columns: +1 gold up, −1 gold down, 0/NaN excluded.
+    df["c_llm"] = df["llm_direction"].map(
+        {"bullish": 1, "bearish": -1}).fillna(0)
+    df["c_doctrine"] = df["surprise"].map({"positive": -1, "negative": 1})
+    df["c_ffbw"] = df["better_worse"].map({1: -1, 2: 1}).fillna(0)
+    df["c_bull"] = 1
+    df["c_bear"] = -1
+
+    strategies = [("LLM", "c_llm"), ("doctrine(surprise)", "c_doctrine"),
+                  ("FF betterWorse", "c_ffbw"), ("always-bullish", "c_bull"),
+                  ("always-bearish", "c_bear")]
+
+    for hname, _ in HORIZONS:
+        dcol = f"d_{hname}"
+        sub = df[df[dcol].notna()]
+        print(f"\n── horizon {hname} (n scored bars={len(sub)}) ──")
+        zeros = int((sub[dcol] == 0).sum())
+        if zeros:
+            print(f"  Δ==0 count (scored as miss): {zeros}")
+        for label, col in strategies:
+            s = sub[sub[col] != 0]
+            hits = int((np.sign(s[dcol]) == s[col]).sum())
+            print("  " + _fmt_rate(hits, len(s), label))
+        # noise-filtered LLM hit rate
+        s = sub[(sub["c_llm"] != 0) &
+                (sub[dcol].abs() >= null_med[hname])]
+        hits = int((np.sign(s[dcol]) == s["c_llm"]).sum())
+        print("  " + _fmt_rate(hits, len(s),
+                               f"LLM |Δ|≥null-med"))
+        # event-window magnitude vs null
+        mag = sub[dcol].abs().median()
+        print(f"  event-window median |Δ| = ${mag:.2f} "
+              f"(null ${null_med[hname]:.2f})")
+        # paired McNemar: LLM vs each baseline on common directional set
+        for label, col in [("doctrine", "c_doctrine"),
+                           ("FF betterWorse", "c_ffbw"),
+                           ("always-bullish", "c_bull")]:
+            s = sub[(sub["c_llm"] != 0) & (sub[col] != 0)]
+            ha = list(np.sign(s[dcol]) == s["c_llm"])
+            hb = list(np.sign(s[dcol]) == s[col])
+            b, c, p = _mcnemar_p(ha, hb)
+            print(f"  McNemar LLM vs {label:<15} b={b:>3} c={c:>3} "
+                  f"p={p:.3f}")
+
+    # neutral-call flatness (15m)
+    neu = df[(df["llm_direction"] == "neutral") & df["d_15m"].notna()]
+    dire = df[(df["c_llm"] != 0) & df["d_15m"].notna()]
+    if len(neu):
+        print(f"\nneutral calls (n={len(neu)}): median |Δ_15m| "
+              f"${neu['d_15m'].abs().median():.2f} vs directional "
+              f"(n={len(dire)}) ${dire['d_15m'].abs().median():.2f} "
+              f"(null ${null_med['15m']:.2f})")
+
+    # breakdowns at 1h
+    dcol = "d_1h"
+    sub = df[df[dcol].notna() & (df["c_llm"] != 0)]
+    print("\n── LLM hit rate by event name (1h, top 10 by count) ──")
+    for name, grp in sorted(sub.groupby("name"), key=lambda kv: -len(kv[1]))[:10]:
+        hits = int((np.sign(grp[dcol]) == grp["c_llm"]).sum())
+        print("  " + _fmt_rate(hits, len(grp), name[:24]))
+    print("── LLM hit rate by year (1h) ──")
+    for yr, grp in sub.groupby("year"):
+        hits = int((np.sign(grp[dcol]) == grp["c_llm"]).sum())
+        print("  " + _fmt_rate(hits, len(grp), yr))
+
+    # cluster view: shared datelines
+    print("\n── same-timestamp clusters ──")
+    clus = df.groupby("dateline").filter(lambda g: len(g) > 1)
+    groups = list(clus.groupby("dateline"))
+    consistent = 0
+    net_hits, net_n = 0, 0
+    for _, g in groups:
+        signs = set(g.loc[g["c_llm"] != 0, "c_llm"])
+        if len(signs) <= 1:
+            consistent += 1
+        net = int(g["c_llm"].sum())
+        d = g["d_1h"].iloc[0]
+        if net != 0 and pd.notna(d) and d != 0:
+            net_n += 1
+            if np.sign(d) == np.sign(net):
+                net_hits += 1
+    print(f"clusters: {len(groups)} datelines "
+          f"({len(clus)} events, {len(clus) / max(len(df), 1) * 100:.0f}% of "
+          f"corpus) | internally consistent: {consistent}/{len(groups)}")
+    print("  " + _fmt_rate(net_hits, net_n, "cluster net-call (1h)"))
+
+    # determinism probe
+    p2 = {r["ffid"]: r for r in _load_jsonl(EVENTS_JSONL)
+          if r.get("run_pass") == 2 and r.get("llm_direction")}
+    if p2:
+        p1 = {r["ffid"]: r for r in recs}
+        agree = same_reason = n = 0
+        for fid, r2 in p2.items():
+            r1 = p1.get(fid)
+            if not r1 or not r1.get("llm_direction"):
+                continue
+            n += 1
+            if r1["llm_direction"] == r2["llm_direction"]:
+                agree += 1
+            if r1.get("reason_raw") == r2.get("reason_raw"):
+                same_reason += 1
+        print(f"\ndeterminism probe: direction agreement {agree}/{n}, "
+              f"identical reason text {same_reason}/{n}")
+
+    # stratified reason examples
+    print("\n── reason samples (1h hit/miss) ──")
+    sub = df[df[dcol].notna() & (df["c_llm"] != 0)].copy()
+    sub["hit"] = np.sign(sub[dcol]) == sub["c_llm"]
+    for verdict in (True, False):
+        picked, seen = [], set()
+        for _, r in sub[sub["hit"] == verdict].iterrows():
+            if r["name"] in seen:
+                continue
+            seen.add(r["name"])
+            picked.append(r)
+            if len(picked) == 10:
+                break
+        tag = "HIT " if verdict else "MISS"
+        for r in picked:
+            print(f"  [{tag}] {r['utc_time'][:16]} {r['name'][:30]:<30} "
+                  f"a={r['actual']} f={r['forecast']} → "
+                  f"{r['llm_direction']:<7} Δ1h={r[dcol]:+7.2f}  "
+                  f"“{(r['reason_raw'] or '')[:60]}”")
+    print(f"\nscored table → {SCORED_CSV}")
+
+
+def report_headlines():
+    recs = _load_jsonl(HEADLINES_JSONL)
+    if not recs:
+        print("\n(no headline replay records — run `headlines` first)")
+        return
+    g = _load_golden()
+    engine = NewsEngine(NewsConfig())
+
+    def collect(tag):
+        m = {}
+        for r in recs:
+            if r["run_tag"] == tag and r["scores"]:
+                for h, sc in zip(r["headlines"], r["scores"]):
+                    m[h] = tuple(sc)
+        return m
+
+    main, shuf = collect("main"), collect("shuffled")
+    print("\n" + "═" * 70)
+    print("PART B — golden headline set: LLM vs keyword baseline")
+    print("═" * 70)
+    missing = [h for h in g["headline"] if h not in main]
+    if missing:
+        print(f"WARNING: {len(missing)} headlines missing from main run")
+
+    def cls(x):
+        if x >= NEUTRAL_BAND:
+            return 1
+        if x <= -NEUTRAL_BAND:
+            return -1
+        return 0
+
+    rows = []
+    for _, r in g.iterrows():
+        h = r["headline"]
+        if h not in main:
+            continue
+        llm_xau = main[h][1]
+        kw = engine._analyze_headline_sentiment(h, for_currency="XAU")
+        rows.append({
+            "headline": h, "category": r["category"],
+            "exp": int(r["expected_xau_sign"]),
+            "llm_xau": llm_xau, "llm_cls": cls(llm_xau),
+            "kw": kw, "kw_cls": cls(kw),
+        })
+    t = pd.DataFrame(rows)
+    for who in ("llm", "kw"):
+        acc = (t[f"{who}_cls"] == t["exp"]).mean()
+        print(f"{who.upper():<4} 3-class accuracy (band ±{NEUTRAL_BAND}): "
+              f"{(t[f'{who}_cls'] == t['exp']).sum()}/{len(t)} "
+              f"= {acc * 100:.1f}%")
+    ha = list(t["llm_cls"] == t["exp"])
+    hb = list(t["kw_cls"] == t["exp"])
+    b, c, p = _mcnemar_p(ha, hb)
+    print(f"paired McNemar LLM vs keyword: b={b} c={c} p={p:.3f}")
+
+    print("\nper-category accuracy (LLM | keyword):")
+    for cat, grp in t.groupby("category"):
+        a1 = (grp["llm_cls"] == grp["exp"]).sum()
+        a2 = (grp["kw_cls"] == grp["exp"]).sum()
+        print(f"  {cat:<18} {a1:>2}/{len(grp):<2} | {a2:>2}/{len(grp)}")
+
+    print("\nLLM confusion (rows=expected, cols=predicted -1/0/+1):")
+    cm = pd.crosstab(t["exp"], t["llm_cls"]).reindex(
+        index=[-1, 0, 1], columns=[-1, 0, 1], fill_value=0)
+    print(cm.to_string())
+
+    print("\nLLM misclassified headlines:")
+    for _, r in t[t["llm_cls"] != t["exp"]].iterrows():
+        print(f"  exp {r['exp']:+d} got {r['llm_xau']:+.2f} "
+              f"[{r['category']}] {r['headline'][:70]}")
+
+    # repeat-batch determinism
+    m_b0 = [r for r in recs if r["run_tag"] == "main" and r["batch"] == 0]
+    r_b0 = [r for r in recs if r["run_tag"] == "repeat"]
+    if m_b0 and r_b0 and m_b0[0]["scores"] and r_b0[0]["scores"]:
+        a = np.array(m_b0[0]["scores"], dtype=float)
+        b_ = np.array(r_b0[0]["scores"], dtype=float)
+        exact = int((a == b_).all(axis=1).sum())
+        print(f"\nrepeat-batch: {exact}/{len(a)} score pairs identical, "
+              f"max |diff| = {np.abs(a - b_).max():.3f}")
+
+    # shuffled-order drift
+    if shuf:
+        both = [h for h in t["headline"] if h in shuf]
+        drift = [abs(main[h][1] - shuf[h][1]) for h in both]
+        if drift:
+            print(f"shuffled-order xau drift over {len(both)} headlines: "
+                  f"mean {np.mean(drift):.3f}, max {np.max(drift):.3f}")
+
+
+def cmd_report(args):
+    report_events()
+    report_headlines()
+
+
+# ── CLI ──────────────────────────────────────────────────────────────
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__.split("\n")[1])
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    e = sub.add_parser("events", help="replay release surprises")
+    e.add_argument("--limit", type=int, default=0,
+                   help="smoke-test on first N events")
+    e.add_argument("--determinism", type=int, default=0,
+                   help="re-call N stratified events a second time")
+    e.add_argument("--sleep", type=float, default=0.3)
+    e.add_argument("--retry-passes", type=int, default=3)
+    e.add_argument("--timeout", type=float, default=30.0)
+    e.set_defaults(fn=cmd_events)
+
+    h = sub.add_parser("headlines", help="golden-set probes")
+    h.add_argument("--sleep", type=float, default=0.3)
+    h.add_argument("--timeout", type=float, default=30.0)
+    h.set_defaults(fn=cmd_headlines)
+
+    r = sub.add_parser("report", help="offline scoring report")
+    r.set_defaults(fn=cmd_report)
+
+    args = ap.parse_args()
+    args.fn(args)
+
+
+if __name__ == "__main__":
+    main()

backtest/research/llm_news_golden.csv

@@ -0,0 +1,79 @@
+headline,category,expected_xau_sign,rationale
+"Powell says restrictive policy needed for longer as inflation proves sticky",hawkish_fed,-1,"higher-for-longer rates lift real yields, gold-bearish"
+"Fed minutes show officials leaning toward two more rate hikes this year",hawkish_fed,-1,"more hikes = stronger dollar/yields"
+"Hot CPI print fuels bets on additional Fed tightening",hawkish_fed,-1,"tightening expectations weigh on gold"
+"Fed's Waller: higher-for-longer rates warranted by resilient economy",hawkish_fed,-1,"hawkish guidance"
+"FOMC raises rates 50bp, flags further increases ahead",hawkish_fed,-1,"outsized hike plus hawkish guidance"
+"Treasury yields spike as Fed pushes back on rate cut hopes",hawkish_fed,-1,"cut hopes dashed, yields up"
+"Core inflation accelerates, cementing case for hawkish Fed stance",hawkish_fed,-1,"hawkish repricing"
+"Fed dot plot shifts higher; markets price out easing for the year",hawkish_fed,-1,"easing priced out"
+"Fed cuts rates 25bp and signals more easing to come",dovish_fed,1,"easing cycle is gold-bullish"
+"Powell hints at rate cuts as inflation cools toward target",dovish_fed,1,"cut guidance"
+"FOMC minutes reveal growing support for policy easing",dovish_fed,1,"dovish minutes"
+"Fed's Williams says rate cuts appropriate later this year",dovish_fed,1,"cut guidance"
+"Soft inflation data boosts odds of a dovish Fed pivot",dovish_fed,1,"pivot odds up"
+"Fed pauses hikes, markets price aggressive easing cycle",dovish_fed,1,"easing priced in"
+"Central banks turn dovish as global growth falters",dovish_fed,1,"global easing backdrop"
+"Fed to restart bond purchases to support flagging economy",dovish_fed,1,"QE is gold-bullish"
+"Dollar index hits 12-month high on robust US economic data",strong_usd,-1,"strong dollar pressures gold"
+"US GDP grows 4.1% annualized, blowing past expectations",strong_usd,-1,"strong economy, hawkish implication"
+"Blockbuster jobs report sends dollar surging across the board",strong_usd,-1,"strong jobs lift dollar and yields"
+"US retail sales smash forecasts, underscoring consumer strength",strong_usd,-1,"strong data"
+"Greenback rallies as US data outshines Europe and Asia",strong_usd,-1,"dollar strength"
+"ISM services index jumps to two-year high on strong demand",strong_usd,-1,"strong data"
+"Consumer confidence surges to post-pandemic high in the US",strong_usd,-1,"strong data"
+"Dollar tumbles to eight-month low after dismal manufacturing data",weak_usd,1,"weak dollar lifts gold"
+"US economy adds far fewer jobs than expected; dollar slides",weak_usd,1,"weak jobs, dovish implication"
+"Greenback weakens as US retail sales unexpectedly contract",weak_usd,1,"weak data"
+"US GDP growth revised down sharply, stoking slowdown fears",weak_usd,1,"slowdown is dovish"
+"Dollar slumps as trade deficit balloons to record",weak_usd,1,"dollar weakness"
+"Soft services PMI adds to evidence of cooling US economy",weak_usd,1,"cooling economy"
+"US consumer sentiment plunges to two-year low",weak_usd,1,"weak data"
+"Real yields climb to multi-year highs as TIPS sell off",real_yields,-1,"rising real yields raise gold's opportunity cost"
+"10-year Treasury yield surges past 5% on supply glut",real_yields,-1,"yield spike"
+"Rising real interest rates dim appeal of non-yielding assets",real_yields,-1,"explicit opportunity-cost story"
+"Treasury yields sink as investors flock to bonds",real_yields,1,"falling yields support gold"
+"Real yields turn negative as inflation outpaces rates",real_yields,1,"negative real yields are gold's best backdrop"
+"Bond rally drives 10-year yield to lowest level in a year",real_yields,1,"falling yields"
+"Geopolitical tensions escalate in Middle East, investors seek safety",safe_haven,1,"safe-haven bid"
+"Stocks crater worldwide as banking crisis fears spread",safe_haven,1,"crisis flight to safety"
+"Escalating conflict drives flight to safe-haven assets",safe_haven,1,"explicit haven flow"
+"Investors dump risk assets after surprise sovereign default",safe_haven,1,"default shock"
+"Military escalation on NATO border rattles global markets",safe_haven,1,"geopolitical stress"
+"Wall Street plunges 4% in worst session since pandemic",safe_haven,1,"risk-off"
+"Currency crisis in emerging markets sparks contagion fears",safe_haven,1,"contagion, haven bid"
+"Nuclear rhetoric intensifies, sending investors to havens",safe_haven,1,"extreme geopolitical stress"
+"Ceasefire agreement lifts global markets, haven demand fades",risk_on,-1,"haven unwind"
+"Stocks roar to record highs as trade deal signed",risk_on,-1,"risk-on drains haven demand"
+"Risk appetite returns as banking fears subside",risk_on,-1,"risk-on"
+"Global equities rally on strong earnings and easing tensions",risk_on,-1,"risk-on"
+"Gold hits record high as central banks keep buying bullion",gold_direct,1,"direct demand"
+"Gold ETFs post biggest weekly inflows in three years",gold_direct,1,"investment inflows"
+"China's central bank extends gold buying spree to 18th month",gold_direct,1,"official-sector demand"
+"Miners struggle to meet gold demand as supply deficit widens",gold_direct,1,"supply deficit"
+"Gold slides 2% as investors book profits after record run",gold_direct,-1,"profit-taking pressure"
+"Gold tumbles below key support as ETF outflows accelerate",gold_direct,-1,"outflows"
+"Bullion demand weakens as Indian jewellery buying stalls",gold_direct,-1,"physical demand soft"
+"Speculators slash bullish gold bets as dollar strengthens",gold_direct,-1,"positioning unwind"
+"Fed holds rates steady, signals no rate hike on the horizon",negation_traps,1,"dovish hold; keyword layer sees the phrase rate hike and flips it gold-bearish"
+"Powell says no rate cuts until inflation shows sustained cooling",negation_traps,-1,"hawkish hold; keyword layer sees rate cut and calls it gold-bullish"
+"Recession fears fade as US data streak beats estimates",negation_traps,-1,"fading fear is risk-on; keywords see recession plus beats and cancel out"
+"Inflation fails to accelerate, easing pressure on the Fed",negation_traps,1,"disinflation opens door to cuts"
+"Fed's Bowman: not yet confident inflation is beaten, cuts premature",negation_traps,-1,"hawkish pushback phrased around negations"
+"Central bank stops short of hiking, but warns tightening may resume",negation_traps,-1,"hawkish hold with tightening bias"
+"No safe-haven bid for gold despite equity wobble, analysts say",negation_traps,-1,"explicitly negated haven demand"
+"Jobs report not as weak as feared, dollar recovers",negation_traps,-1,"less-bad data is dollar-positive"
+"US manufacturing output plunges as new orders dry up",inverted_metrics,1,"weak economy is dovish/gold-bullish; keyword layer reads plunges as bearish"
+"US factory activity slumps to three-year low",inverted_metrics,1,"weak data; keyword reads slumps as bearish"
+"Consumer spending falls for third straight month",inverted_metrics,1,"weak data; keyword reads falls as bearish"
+"Private payrolls growth slides to weakest pace since 2020",inverted_metrics,1,"weak jobs; keyword reads slides as bearish"
+"Jobless claims hit two-year high as layoffs spread",inverted_metrics,1,"higher claims = weaker economy = gold-bullish"
+"US unemployment rate hits 4.5%, highest since 2021",inverted_metrics,1,"higher unemployment = dovish = gold-bullish"
+"Lakers clinch playoff berth with overtime win over Suns",distractors,0,"irrelevant"
+"New Marvel film smashes box office records on opening weekend",distractors,0,"irrelevant"
+"Apple unveils thinner MacBook with faster chip lineup",distractors,0,"irrelevant"
+"Scientists discover new deep-sea species off Pacific coast",distractors,0,"irrelevant"
+"Heavy rain forecast for the Midwest through the weekend",distractors,0,"irrelevant"
+"Streaming service raises subscription prices in Europe",distractors,0,"irrelevant"
+"Electric vehicle startup recalls 12,000 cars over software glitch",distractors,0,"irrelevant"
+"Local elections see record turnout in several US states",distractors,0,"irrelevant"

backtest/research/news_sentiment_value_2026-09-04.md

@@ -1,129 +0,0 @@
-# News sentiment vs. price — does the news score carry information? — 2026-09-04
-
-**Source evidence:** `logs/signals.db` 2026-08-13 → 2026-09-03 (14,683 rows, 13,947 with
-a price; candle snapshots from 08-18, full-shape headline blobs from 08-19), read with
-`mode=ro`. A continuous H1/M15/M5 candle stream was rebuilt from the stored 14-bar
-`candles_json` snapshots (H1: 293 bars, 08-18 05:00 → 09-03 22:00). Headlines were
-resolved through `news_snapshots` (4,054 distinct headlines, 2,024 gold-tagged).
-Outcomes are the repo's own advisory model (`advisory_r_multiple` on the 144 `RESOLVED`
-rows, R at $0.65). One-year context from the Oanda H1 parquet
-(`backtest/cache/oanda_XAU_USD_1h_20250814_20260814.parquet`). No backtest: the
-backtester runs with news off, and no headline history exists beyond these 3 weeks, so a
-replay cannot test news. The same check reruns with
-`python analyze_signal_log.py --stored-candles --news`.
-
-Question asked by the owner: the log reads bullish most of the time — what did the
-indicators and the news say when gold fell, and is news scoring a waste?
-
-## Findings
-
-**The numeric news score is a rear-view mirror.** `news_sentiment` (= mean gold-headline
-score − mean dollar-headline score over the last 12 h) tracks what gold did over the
-*previous* day and says nothing stable about the next one.
-
-| hourly mean sentiment vs. gold return (n≈293 h) | Pearson |
-|---|---|
-| past 24 h | **+0.67** |
-| past 8 h / 4 h / 1 h | +0.32 / +0.20 / +0.07 |
-| next 1 h / 4 h / 8 h / 24 h | +0.05 / +0.10 / +0.21 / +0.14 |
-| next 8 h, first half (08-18→25) / second half (08-26→09-03) | **−0.37 / +0.12** |
-| next 24 h, first half / second half | −0.12 / −0.09 |
-| daily mean vs same-day / previous-day / next-day move | +0.39 / +0.37 / **+0.03** |
-| *change* in sentiment vs next 4 h / 8 h / 24 h | −0.09 / −0.03 / −0.01 |
-
-Partial correlation with the next 8 h after removing the past-24 h return is +0.26 pooled
-but flips sign between halves. Alternative aggregates (gold-specific score only, dropping
-price-report headlines, only headlines < 2 h old) show the same shape: past-24 h
-correlation 0.59–0.82, no stable forward correlation. A naive "hold in the direction of
-sentiment" rule made +$272 over 279 hours vs +$120 buy-and-hold, but trading the *change*
-in sentiment lost −$200: the gain is trend persistence (sentiment ≈ trailing return), not
-news.
-
-**Why the log reads bullish.** 62% of rows have sentiment ≥ +0.1, 21% ≤ −0.1. Component
-means: gold headlines +0.158, dollar headlines +0.048. The window was a 16% August rally,
-so the average headline was bullish because price had been rising. Of the gold-tagged
-headlines, 42% merely restate the chart (price-up reports 489, mean +0.37; price-down 366,
-mean −0.18) and 18% are long-horizon forecast pieces ("$5,400 target", "$50,000/oz"; 60%
-positive, 21% of all positive score mass). The per-headline LLM labels are correct — every
-regex-flagged "mis-score" was a correctly scored "yields fall → gold up" headline — which
-matches the 78/78 golden set of 2026-08-06. The averaging, the 12 h window and the
-headline universe are what destroy the information.
-
-**What news and indicators said on the way down** (top $4,697 on 08-25 00:00 UTC, low
-$4,283 on 09-02 03:00; an 8.8% drop in a week):
-
-| moment | price | news sentiment | engine (M15) |
-|---|---|---|---|
-| top, 08-25 00:00 | 4,697 | +0.30, rising to **+0.44** two hours after the top (window high) | ranging, composite +5.9; 9 BUY setups that day: −7.4R, 11% win |
-| 08-26/27 | −$60 | still +0.15…+0.37; first bearish hour 08-26 20:00, **44 h after the top**, price already −$102 | trending_down 134/138 rows, composite −4.6/−7.8; SELL setups 0/3 on 08-27 |
-| 08-28 Warsh speech 13:00–16:00 | −$122 in 3 h, −$140 day | +0.24…+0.30 overnight, **0.00 at 13:00**, −0.17 only at 16:00; summary all morning: "a hawkish tone could pressure gold, dovish hints may lift it" | composite −6.2; SELL setups 4/4, +3.3R |
-| 09-01 | −$128 day | −0.04…−0.20, "gold under pressure" all day (correct, describing the move) | trending_down 275 rows, composite −12.2, 51 SELL rows; SELL setups 7/7, +3.5R |
-| low, 09-02 03:00 | 4,283 | −0.16 at the low, −0.19 six hours later; first bullish hour 09-02 19:00, **16 h after the low**, price already +$108 | SELL setups at the bottom 0/2, −2.05R |
-
-Sentiment peaked at the top and troughed at the bottom. The engine also lagged (it is
-trend-following) but flipped regime and composite within hours; its SELL setups won on the
-trend days and lost at the turns, the same as its BUYs.
-
-**Outcomes** (144 RESOLVED, +37.8R): BUY 121 / +43.2R / 70% win; SELL 23 / −5.4R / 48%.
-
-| direction × sentiment bucket (≥+0.15 bull, ≤−0.05 bear) | n | R | mean | win |
-|---|---|---|---|---|
-| BUY, bullish news | 81 | +32.0 | +0.40 | 73% |
-| BUY, neutral | 34 | +10.6 | +0.31 | 68% |
-| BUY, bearish news | 6 | +0.65 | +0.11 | 50% |
-| SELL, bearish news | 14 | +0.13 | +0.01 | 71% |
-| SELL, neutral | 5 | −1.46 | −0.29 | 20% |
-| SELL, bullish news | 4 | −4.08 | −1.02 | 0% |
-
-The splits are the trend confound again ("BUY with bullish news" = "BUY in an uptrend").
-Cleanest counter-example: 08-25, the most bullish news day of the drop period (+0.25),
-9 BUYs at the top lost −7.4R. corr(sentiment, R): BUY +0.10, SELL −0.45 (n=23).
-
-**Macro persona** (reads the same headlines plus high-impact events): macro=support 51
-BUYs → +32.6R, 90% win; macro=neutral 16 → −3.2R, 38%. Within bullish-sentiment BUYs:
-support 42 → +25.2R (88%) vs neutral 7 → −2.9R (29%). On 08-25 all 7 panel rows were
-macro=neutral ("awaiting PCE, mixed") while the numeric aggregate said +0.25 bullish.
-Confounded with regime (64 of 84 supports were trending_up) and n≈16 on the informative
-side. Not enough to promote or demote; the standing rule (re-score at ~300 outcomes)
-applies.
-
-**Event risk** cannot be evaluated here: only 4 event windows in the sample
-(08-19 17:00, 08-26 12:00, 08-28 13:00, 09-01 13:00), none produced a setup. The standing
-evidence remains the 2026-07-11 audit (event gates would cost +4.6…+7.2R/yr).
-
-**One-year context:** daily moves ≤ −$100 happened 17 times in the year to 2026-08-14;
-15 of 47 rolling 3-week windows contained a ≥ 8% drawdown. This sample is one ordinary
-rally-then-drop regime.
-
-**Simulated 4 h window + neutral drop** (the change shipped with this note, on 11,136
-full-shape scans): never leaves a scan empty (median 19 directional headlines, 8
-gold-specific; worst hour 02:00 UTC has < 3 gold headlines 28% of the time), the aggregate
-correlates 0.78 with the old one and becomes bolder (share |score| ≥ 0.3 rises 17% → 41%),
-with the same lack of forward information (next-24 h corr +0.20; next-8 h halves
-−0.22 / −0.10).
-
-## Method
-
-- Hourly join convention: a scan in hour H (boundary + 10 s) sees the bar closed at H,
-  i.e. bar H's open; forward returns run from bar H's open, past returns end there.
-- Correlations are Pearson on hourly means (no scipy); halves split at 2026-08-26 00:00.
-- Headline classes by regex on the title (price-up / price-down / forecast-style); the
-  gold-specific score is `score_by_currency.XAU` where present, else `score`.
-- Outcome buckets use the representative row's `news_sentiment`; macro stance from
-  `panel_takes_json`.
-- Everything above reproduces from the DB alone; nothing depends on live feeds.
-
-## Decisions
-
-- `sentiment_affects_signals` and `event_risk_caps_signals` stay `False`. No gate is
-  built in either direction (follow or fade): 3 weeks, one regime.
-- The headline labelling is not the problem; the pool and the averaging are. Shipped the
-  same day: `headline_max_age_seconds` 12 h → 4 h with undated items dropped, and
-  `get_fundamental_context` drops headlines whose every relevant score is inside ±0.1
-  (`_is_neutral_headline`). The market-summary prompt now writes for someone who has
-  never traded, sends headlines newest-first with the platform's own gold direction, and
-  caps at 400 chars. **`news_sentiment` rows from the deploy date onward are a mean over
-  directional ≤ 4 h headlines — log comparisons must split at that date.**
-- Keep the macro persona running and logging; re-check with `--news` and the persona
-  scorecard at ~300 outcomes.
-- Refuted hypothesis, do not re-propose without new data: "news sentiment leads price".

backtest/research/oanda_td_divergence.py

@@ -0,0 +1,199 @@
+"""
+Oanda vs Twelve Data bar-divergence study — XAU/USD M15, 12 months.
+====================================================================
+Compares the mid-price OHLC bars the signal engine would see from each
+provider over the same window. Purpose: quantify whether migrating the
+signal feed from Twelve Data to Oanda is a formality (bars ~identical)
+or a recalibration (bars differ at a scale that moves indicators).
+
+Inputs:
+  --td      Twelve Data parquet (raw bars from backtest/cache/)
+  --oanda   Oanda parquet (from fetch_oanda_m5.py / fetch_oanda_m15.py)
+  --start / --end  analysis window (UTC dates)
+
+Outputs:
+  printed report
+  backtest/research/oanda_td_divergence_by_hour.csv
+  backtest/research/oanda_td_bar_diffs.parquet  (per-bar diffs, common index)
+"""
+
+import argparse
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+RESEARCH_DIR = Path(__file__).parent
+PIP = 0.10  # XAU pip in $
+
+
+def wilder_atr(df: pd.DataFrame, period: int = 14) -> pd.Series:
+    hl = df["High"] - df["Low"]
+    hc = (df["High"] - df["Close"].shift(1)).abs()
+    lc = (df["Low"] - df["Close"].shift(1)).abs()
+    tr = pd.concat([hl, hc, lc], axis=1).max(axis=1)
+    return tr.ewm(alpha=1.0 / period, adjust=False).mean()
+
+
+def pct_stats(s: pd.Series) -> dict:
+    a = s.abs()
+    return {
+        "mean": a.mean(), "median": a.median(),
+        "p90": a.quantile(0.90), "p95": a.quantile(0.95),
+        "p99": a.quantile(0.99), "max": a.max(),
+        "bias(mean signed)": s.mean(),
+    }
+
+
+def fmt_stats(name: str, s: pd.Series) -> str:
+    st = pct_stats(s)
+    return (f"  {name:<6} mean ${st['mean']:.3f}  med ${st['median']:.3f}  "
+            f"p95 ${st['p95']:.3f}  p99 ${st['p99']:.3f}  max ${st['max']:.2f}  "
+            f"bias {st['bias(mean signed)']:+.4f}")
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--td", default=str(RESEARCH_DIR.parent / "cache" /
+                                        "XAU_USD_15min_20240522_20260723.parquet"))
+    ap.add_argument("--oanda", default=str(RESEARCH_DIR /
+                                           "oanda_xau_m15_20250701_now.parquet"))
+    ap.add_argument("--start", default="2025-07-24")
+    ap.add_argument("--end", default="2026-07-23")
+    args = ap.parse_args()
+
+    td = pd.read_parquet(args.td)
+    oa = pd.read_parquet(args.oanda)
+
+    start = pd.Timestamp(args.start, tz="UTC")
+    end = pd.Timestamp(args.end, tz="UTC") + pd.Timedelta(days=1)
+    start = max(start, td.index[0], oa.index[0])
+    end = min(end, td.index[-1], oa.index[-1])
+    td = td.loc[(td.index >= start) & (td.index <= end)]
+    oa = oa.loc[(oa.index >= start) & (oa.index <= end)]
+
+    print("=" * 78)
+    print("OANDA vs TWELVE DATA — XAU/USD M15 bar divergence")
+    print(f"Window: {start} -> {end}  ({(end - start).days} days)")
+    print("=" * 78)
+
+    # ── Coverage ─────────────────────────────────────────────────────
+    common = td.index.intersection(oa.index)
+    td_only = td.index.difference(oa.index)
+    oa_only = oa.index.difference(td.index)
+    print(f"\nCOVERAGE")
+    print(f"  Twelve Data bars : {len(td):>7,}")
+    print(f"  Oanda bars       : {len(oa):>7,}")
+    print(f"  common           : {len(common):>7,}")
+    print(f"  TD-only          : {len(td_only):>7,}  "
+          f"({100 * len(td_only) / max(len(td), 1):.2f}% of TD)")
+    print(f"  Oanda-only       : {len(oa_only):>7,}  "
+          f"({100 * len(oa_only) / max(len(oa), 1):.2f}% of Oanda)")
+    for label, idx in (("TD-only", td_only), ("Oanda-only", oa_only)):
+        if len(idx):
+            by_date = pd.Series(1, index=idx).groupby(idx.date).sum()
+            top = by_date.sort_values(ascending=False).head(8)
+            print(f"  {label} bars cluster on: "
+                  + ", ".join(f"{d} ({n})" for d, n in top.items()))
+            by_hour = pd.Series(1, index=idx).groupby(idx.hour).sum()
+            top_h = by_hour.sort_values(ascending=False).head(6)
+            print(f"  {label} by UTC hour: "
+                  + ", ".join(f"{h:02d}h ({n})" for h, n in top_h.items()))
+
+    t = td.loc[common]
+    o = oa.loc[common]
+
+    # ── Scale references ─────────────────────────────────────────────
+    atr = wilder_atr(t).dropna()
+    mean_price = t["Close"].mean()
+    print(f"\nSCALE REFERENCES (Twelve Data feed)")
+    print(f"  mean close      : ${mean_price:,.2f}")
+    print(f"  mean bar range  : ${(t['High'] - t['Low']).mean():.3f}")
+    print(f"  mean ATR14      : ${atr.mean():.3f}   median ${atr.median():.3f}")
+
+    # ── Per-field diffs (TD − Oanda mid) ─────────────────────────────
+    print(f"\nPER-BAR DIFFS  (TD − Oanda, $; pip = $0.10)")
+    diffs = {}
+    for col in ("Open", "High", "Low", "Close"):
+        d = t[col] - o[col]
+        diffs[col] = d
+        print(fmt_stats(col, d))
+
+    dclose = diffs["Close"].abs()
+    print(f"\n  |ΔClose| as % of mean ATR14 : "
+          f"mean {100 * dclose.mean() / atr.mean():.1f}%   "
+          f"p95 {100 * dclose.quantile(0.95) / atr.mean():.1f}%")
+
+    print(f"\nTHRESHOLD EXCEEDANCE  (|ΔClose|)")
+    for thr in (0.10, 0.30, 0.65, 1.00, 2.00, 5.00):
+        pct = 100 * (dclose > thr).mean()
+        print(f"  > ${thr:<5.2f}: {pct:6.2f}% of bars  ({(dclose > thr).sum():,})")
+
+    # ── Direction / return agreement ─────────────────────────────────
+    t_ret = t["Close"].pct_change()
+    o_ret = o["Close"].pct_change()
+    # only consecutive common bars (gap = one M15 step)
+    step_ok = pd.Series(common, index=common).diff().eq(pd.Timedelta(minutes=15))
+    tr, orr = t_ret[step_ok], o_ret[step_ok]
+    corr = tr.corr(orr)
+    t_dir = np.sign(t["Close"] - t["Open"])
+    o_dir = np.sign(o["Close"] - o["Open"])
+    both_nonflat = (t_dir != 0) & (o_dir != 0)
+    dir_agree = 100 * (t_dir[both_nonflat] == o_dir[both_nonflat]).mean()
+    print(f"\nAGREEMENT")
+    print(f"  close-return correlation (consecutive bars): {corr:.5f}  "
+          f"(n={step_ok.sum():,})")
+    print(f"  bar direction (sign of close−open) agree   : {dir_agree:.2f}%")
+
+    # ── By-hour breakdown ────────────────────────────────────────────
+    hours = common.hour
+    rows = []
+    for h in range(24):
+        m = hours == h
+        if not m.any():
+            continue
+        row = {"hour_utc": h, "bars": int(m.sum()),
+               "mean_abs_dclose": dclose[m].mean(),
+               "p95_abs_dclose": dclose[m].quantile(0.95),
+               "mean_abs_dhigh": diffs["High"][m].abs().mean(),
+               "mean_abs_dlow": diffs["Low"][m].abs().mean()}
+        if "bid_c" in o.columns:
+            row["oanda_median_spread"] = (
+                (o["ask_c"] - o["bid_c"])[m].median())
+        rows.append(row)
+    by_hour = pd.DataFrame(rows)
+    out_hour = RESEARCH_DIR / "oanda_td_divergence_by_hour.csv"
+    by_hour.to_csv(out_hour, index=False, float_format="%.4f")
+    print(f"\nBY UTC HOUR  (written to {out_hour.name})")
+    print(by_hour.to_string(index=False,
+                            float_format=lambda x: f"{x:.3f}"))
+
+    # ── By-month drift ───────────────────────────────────────────────
+    print(f"\nBY MONTH  (mean |ΔClose|, $)")
+    by_month = dclose.groupby(common.to_period("M")).mean()
+    for p, v in by_month.items():
+        print(f"  {p}: {v:.3f}")
+
+    # ── Oanda real spread (bonus, ties to Dukascopy study) ───────────
+    if "bid_c" in o.columns:
+        spread = o["ask_c"] - o["bid_c"]
+        liquid = ~o.index.hour.isin([21, 22, 23, 0])
+        print(f"\nOANDA REAL SPREAD (close bid/ask, $)")
+        print(f"  all bars      : median {spread.median():.3f}  "
+              f"p25 {spread.quantile(0.25):.3f}  p75 {spread.quantile(0.75):.3f}")
+        print(f"  liquid hours  : median {spread[liquid].median():.3f}")
+        print(f"  21-00 UTC     : median {spread[~liquid].median():.3f}")
+
+    # ── Persist per-bar diffs ────────────────────────────────────────
+    out = pd.DataFrame({
+        "td_close": t["Close"], "oa_close": o["Close"],
+        "d_open": diffs["Open"], "d_high": diffs["High"],
+        "d_low": diffs["Low"], "d_close": diffs["Close"],
+    })
+    out_path = RESEARCH_DIR / "oanda_td_bar_diffs.parquet"
+    out.to_parquet(out_path)
+    print(f"\nPer-bar diffs written to {out_path}")
+
+
+if __name__ == "__main__":
+    main()

backtest/research/oanda_td_divergence_by_hour.csv

@@ -0,0 +1,25 @@
+hour_utc,bars,mean_abs_dclose,p95_abs_dclose,mean_abs_dhigh,mean_abs_dlow,oanda_median_spread
+0,1029,3.3483,13.9460,3.2906,3.4310,0.7900
+1,1028,3.3483,12.8292,3.3477,3.6398,0.7700
+2,1028,3.0288,10.8960,2.9469,3.1317,0.7700
+3,1028,2.8783,11.1300,2.8419,2.9992,0.7600
+4,1028,2.8232,10.4710,2.8165,2.8455,0.7400
+5,1028,2.9268,10.6027,2.9234,3.0038,0.7400
+6,1028,3.0466,10.9265,2.9487,3.0796,0.7200
+7,1028,3.0286,11.3512,3.0301,3.0797,0.6700
+8,1028,3.0223,11.1483,2.8989,3.0801,0.6600
+9,1028,3.0460,11.6345,2.9307,3.0609,0.6600
+10,1028,2.9280,10.9015,2.8647,3.0029,0.6600
+11,1028,3.0340,11.4712,2.9591,2.9472,0.6600
+12,1028,3.0986,11.3282,2.9412,3.1407,0.7000
+13,1028,3.4014,12.3343,3.1077,3.4228,0.7300
+14,1028,3.5074,11.3628,3.3960,3.6569,0.6900
+15,1028,3.4514,11.8982,3.4124,3.6733,0.6700
+16,1028,3.1275,12.5155,3.2038,3.3061,0.6700
+17,1020,2.9826,12.6435,3.0064,3.1008,0.6900
+18,1015,2.8721,12.5480,2.9101,2.9996,0.6900
+19,1004,2.8637,12.1335,2.8824,2.9454,0.6900
+20,996,2.8006,11.9325,2.8486,2.8577,0.7600
+21,335,1.1085,4.8600,1.1212,1.2034,0.9400
+22,676,4.0827,13.6787,4.8422,4.6761,0.8300
+23,1027,3.2722,12.5725,3.5617,3.7747,0.8700

backtest/research/repin_verification_2026-08-14.md

@@ -1,40 +0,0 @@
-# Backtest Re-pin on Oanda bars — verification (Phase 3)
-
-**Date:** 2026-08-14 (verification; original books were generated 2026-08-04, current-HEAD M5 canonical book exported 2026-08-14)
-**Scope:** confirm the pinned Oanda books exist, satisfy rule 1, and quantify the Oanda↔TD delta that motivated them.
-
-## Re-pin state
-
-The re-pin is **already done** — full Oanda-bar books exist in `backtest/oanda_*.csv` for M5/M15/H1/H4, at both cost assumptions, with in-sample (12mo), out-of-sample (2024-06 → 2025-03), and 5-year variants. The shipped configs are `m5` (pullback OFF — `_nopb`) and `m15`.
-
-### M5 canonical-reference reconciliation (2026-08-14)
-
-The dated current-HEAD export `backtest/oanda_m5_12mo_065_nopb_20260814.csv` is the canonical M5 12-month reference going forward: **577 trades, +106.82R, PF 1.51**, with both 12-month halves positive (+50.80R / +56.02R; PF 1.46 / 1.55). It covers 2025-08-14 → 2026-08-14, runs the shipped M5 factory (XAU_PB off), charges $0.65 round trip, and uses next-open/raw-path/close-only-trail realism unchanged.
-
-Earlier M5 snapshots (`oanda_m5_12mo_065_nopb.csv`: 586 / +98.11R / 1.45; `m5_12mo_shipped.csv`: 588 / +96.60R / 1.44; and the 555 / +98.64R ledger entry) remain historical evidence from different HEADs around the weekend-calendar and related QA work. They must not be cited as the current-HEAD reference.
-
-### Rule-1 summary at realistic cost ($0.65 round trip)
-
-| TF | 12mo in-sample | 12mo H1 / H2 | OOS (2024-06→2025-03) |
-|---|---|---|---|
-| M15 | +73.93R PF 1.53 | +66.71 / +7.23 (PF 1.10) | +38.77R PF 1.36 |
-| M5 (shipped, nopb) | +106.82R PF 1.51 (current HEAD) | +50.80 / +56.02 (PF 1.46 / 1.55) | +9.52R PF 1.34 |
-| H1 | +23.97R PF 1.55 | — | +5.15R PF 1.13 |
-| H4 | +16.14R PF 2.28 | — | +1.31R PF 1.14 |
-
-- **M5 (shipped)** passes rule 1 cleanly: both current-HEAD 12mo halves positive with PF ≥ 1.3, and OOS positive with PF 1.34.
-- **M15** passes in-sample and OOS (PF 1.53 / 1.36) but its second half is soft (PF 1.10) — positive R, below the 1.3 bar. Pre-existing condition, not a re-pin artifact.
-- **H1/H4** pass in-sample but OOS is marginal (PF 1.13–1.14) — consistent with the standing "H4 unproven for auto-execution" note.
-
-## Oanda ↔ Twelve Data trade-level delta
-
-Not worth a trade-by-trade book diff: Phase 1 already showed the two feeds disagree on **bar direction** 19% of the time on M5 (11% on M15), with a ~$3 mean / ~$1 systematic close gap. A TD-pinned book is a different instrument's book — this is the documented re-pin motivator, and the Oanda books above are the replacement baseline. A formal trade-level reconciliation would show near-total mismatch and add no decision value.
-
-## Deliverable status
-
-- [x] Re-pinned books exist on Oanda bars (12mo + OOS + 5yr, $0.30 + $0.65).
-- [x] Rule 1 verified for the shipped TFs (M5 clean; M15 in-sample+OOS pass, H2 soft; H1/H4 OOS marginal).
-- [x] Cost model: $0.65 matches measured Oanda spread (~$0.65–0.72); $0.30 retained for cross-book comparability.
-- [x] Oanda↔TD delta quantified (Phase 1) — no further book diff needed.
-
-Remaining Phase-3 follow-up: none. Owner decision 2026-08-14 retained dual reporting: $0.65 is the realistic headline while $0.30 remains for TD-era cross-book comparability and the live TP spread gate.

backtest/research/signal_log_campaign_2026-09.md

@@ -1,182 +0,0 @@
-# Signal-log audit → prediction-quality campaign — 2026-09-03
-
-**Source evidence:** `logs/signals.db` 2026-08-13 → 2026-09-03 (14,683 rows, candle
-snapshots from 08-18), replayed with the repo's own advisory outcome model
-(`analyze_signal_log.py --stored-candles`; the replay reproduces
-`advisory_outcome.py` to 5e-17 on the 49 panel-seeded rows), cross-checked
-against the pinned Oanda books (`backtest/oanda_*_065*.csv`).
-
-## Live-log findings that motivated the campaign
-
-| View (3 weeks, $0.65 rt) | n | WR | R | PF |
-|---|---|---|---|---|
-| every unique setup event | 144 | 67% | +37.8 | 1.93 |
-| opportunities (first per tf+direction, 60 min) | 66 | 67% | +18.3 | 1.92 |
-| one-position-per-TF (5-bar cooldown) | 32 | 62% | +6.6 | 1.53 |
-| direction BUY / SELL | 121 / 23 | 70% / 48% | +43.2 / −5.4 | 2.53 / 0.56 |
-
-- Two rally days (08-19, 08-21) contribute essentially all of the R.
-- SELL is negative in every view; in the books SELL is the weakest bucket on
-  every TF (H1 SELL −16.7R/5yr PF 0.77, −11.5R OOS PF 0.30; M15 SELL −11.7R OOS;
-  M5 SELL in `transitioning` −7.9R/12mo PF 0.21, −5.2R/5yr PF 0.49).
-- Bollinger is anti-predictive on all four TFs (forward-return hit 0.26–0.50 at
-  12 bars) and "opposes" 93% of taken book trades; divergence is inverted on
-  M15/H1 live and trades where it opposed outperform in the books.
-- Refuted live hypotheses (books disagree): ADX<20 gate, exhaustion gate,
-  NEAR-EMA200 gate, EXTREME-volatility gate, capping SL/ATR.
-
-## Method (rule 1)
-
-Harness: `backtest/gpu_backtest.py --pair XAUUSD --spread 0.65` on the CPU
-backend (CuPy present but the GPU was not reachable; the M5 12mo control
-reproduces the pinned `oanda_m5_12mo_065_nopb_20260814.csv` trade-for-trade
-except the two trades on the end date, which `--end` excludes). Every candidate
-is compared with a CONTROL produced on the same harness, never with the pinned
-book directly. Windows (cache start = start − 62 days):
-
-| window | `--start` | `--end` |
-|---|---|---|
-| 12mo (M15/H1) | 2025-08-04 | 2026-08-04 |
-| 12mo (M5 canonical) | 2025-08-14 | 2026-08-14 |
-| 5yr | 2021-08-01 | 2026-08-04 |
-| OOS | 2024-06-01 | 2025-03-30 |
-
-Acceptance: positive total R in both 12mo halves, PF ≥ 1.3, OOS not worse
-than control, and a candidate that removes a net-positive bucket for a PF-only
-gain is rejected (big-candle precedent). Runner:
-`<scratchpad>/campaign/run_campaign.sh`; summary: `summarize.py` (scratchpad
-tooling, not retained — both shipped gates are reproduced by a plain pair-factory
-run at HEAD; see the parity recipe in `sizing_campaign_2026-09.md`).
-
-Candidates (all `--set`, no code):
-
-- **B1 SELL gating** — H1 `xau_strategy.sell_min_quality=[]` (block all H1
-  shorts); M5 `xau_strategy.sell_blocked_regimes=["transitioning"]`; M15
-  `xau_strategy.sell_min_quality=["B"]`.
-- **B2 divergence off** — `technical.disabled_indicators=["divergence"]` on
-  M15 and H1 (M5 divergence-down was rejected 2026-08-04; not re-run).
-- **B3 Bollinger weight 0 in trending regimes** — full
-  `technical.regime_weights_override` with bollinger 0.0 in
-  trending_up/trending_down/trending, ranging/transitioning unchanged; M5, M15, H1.
-
-## Results
-
-Every control reproduced its pinned book: M15 12mo (`ctl_m15_12mo`) is exact
-at 397/+73.93R/PF 1.53; all three H1 controls are exact; the M5 12mo control
-(`base_m5_12mo`) is exact except for the two trades that fall on the end
-date, which `--end` excludes on this harness (see Method above).
-
-| book | n | R | PF | DD | H1 | H2 | sellN | sellR | ΔR vs control | ΔPF |
-|---|---|---|---|---|---|---|---|---|---|---|
-| b1_h1_nosell_12mo | 91 | +22.2 | 1.79 | 4.1 | +20.6 | +1.6 | 0 | +0.0 | -1.8 | +0.25 |
-| b1_h1_nosell_5yr | 335 | +49.1 | 1.46 | 13.2 | -5.0 | +54.1 | 0 | +0.0 | +16.7 | +0.28 |
-| b1_h1_nosell_oos | 78 | +16.7 | 1.74 | 3.6 | +7.3 | +9.4 | 0 | +0.0 | +11.5 | +0.61 |
-| b1_m15_sellB_12mo | 341 | +74.6 | 1.62 | 10.8 | +68.4 | +6.2 | 82 | +9.1 | +0.7 | +0.09 |
-| b1_m15_sellB_5yr | 1378 | +119.1 | 1.22 | 41.4 | -21.3 | +140.3 | 335 | +3.7 | -8.9 | +0.02 |
-| b1_m15_sellB_oos | 272 | +47.1 | 1.51 | 13.8 | +12.3 | +34.8 | 59 | -3.4 | +8.3 | +0.15 |
-| b1_m5_selltrans_12mo | 557 | +112.2 | 1.56 | 9.9 | +55.9 | +56.4 | 75 | +14.8 | +7.9 | +0.06 |
-| b1_m5_selltrans_5yr | 889 | +133.7 | 1.41 | 12.8 | -8.3 | +142.0 | 88 | +9.0 | +5.3 | +0.03 |
-| b1_m5_selltrans_oos | 86 | +9.4 | 1.33 | 8.2 | -1.4 | +10.9 | 4 | +1.5 | +0.0 | +0.00 |
-| b2_h1_nodiv_12mo | 149 | +28.6 | 1.62 | 6.5 | +23.6 | +5.0 | 49 | +5.5 | +4.6 | +0.07 |
-| b2_h1_nodiv_5yr | 567 | +40.5 | 1.20 | 14.1 | -5.2 | +45.6 | 203 | -18.2 | +8.1 | +0.02 |
-| b2_h1_nodiv_oos | 119 | +7.6 | 1.17 | 7.7 | -1.9 | +9.5 | 31 | -14.3 | +2.4 | +0.04 |
-| b2_m15_nodiv_12mo | 439 | +81.3 | 1.51 | 13.1 | +72.3 | +8.9 | 147 | +7.0 | +7.4 | -0.02 |
-| b2_m15_nodiv_5yr | 1771 | +139.9 | 1.20 | 47.4 | -16.2 | +156.1 | 578 | +11.7 | +11.9 | -0.01 |
-| b2_m15_nodiv_oos | 341 | +52.3 | 1.43 | 17.1 | +15.6 | +36.7 | 93 | -10.0 | +13.5 | +0.07 |
-| b3_h1_bb0_12mo | 154 | +31.2 | 1.64 | 9.4 | +22.8 | +8.4 | 52 | +6.7 | +7.3 | +0.09 |
-| b3_h1_bb0_5yr | 583 | +37.3 | 1.18 | 18.3 | -7.4 | +44.7 | 206 | -20.4 | +4.9 | +0.00 |
-| b3_h1_bb0_oos | 122 | +5.6 | 1.12 | 8.7 | -2.0 | +7.6 | 32 | -13.8 | +0.4 | -0.01 |
-| b3_m15_bb0_12mo | 439 | +49.9 | 1.29 | 19.4 | +61.6 | -11.8 | 151 | +0.3 | -24.0 | -0.23 |
-| b3_m15_bb0_5yr | 1790 | +126.6 | 1.18 | 42.3 | -1.8 | +128.5 | 592 | +17.4 | -1.3 | -0.03 |
-| b3_m15_bb0_oos | 345 | +55.5 | 1.47 | 12.4 | +18.2 | +37.3 | 94 | -8.4 | +16.7 | +0.11 |
-| b3_m5_bb0_12mo | 621 | +84.0 | 1.35 | 16.6 | +49.3 | +34.7 | 104 | +9.7 | -20.4 | -0.14 |
-| b3_m5_bb0_5yr | 987 | +113.0 | 1.31 | 15.3 | -7.7 | +120.6 | 134 | +12.0 | -15.5 | -0.08 |
-| b3_m5_bb0_oos | 93 | +10.6 | 1.35 | 5.6 | -2.9 | +13.5 | 9 | +3.7 | +1.2 | +0.01 |
-| b12_h1_nosell_nodiv_12mo | 100 | +23.1 | 1.75 | 4.6 | +21.8 | +1.3 | 0 | +0.0 | -0.8 | +0.20 |
-| b12_h1_nosell_nodiv_5yr | 364 | +58.7 | 1.52 | 12.9 | -4.1 | +62.8 | 0 | +0.0 | +26.3 | +0.34 |
-| b12_h1_nosell_nodiv_oos | 88 | +21.9 | 1.92 | 4.8 | +9.4 | +12.5 | 0 | +0.0 | +16.8 | +0.79 |
-| base_m5_12mo | 575 | +104.3 | 1.50 | 10.7 | +50.8 | +53.5 | 93 | +6.9 | — | — |
-| ctl_h1_12mo | 138 | +24.0 | 1.55 | 8.0 | +19.4 | +4.6 | 47 | +1.8 | — | — |
-| ctl_h1_5yr | 515 | +32.4 | 1.18 | 15.2 | -5.9 | +38.3 | 180 | -16.7 | — | — |
-| ctl_h1_oos | 104 | +5.1 | 1.13 | 7.5 | -0.9 | +6.1 | 26 | -11.5 | — | — |
-| ctl_m15_12mo | 397 | +73.9 | 1.53 | 10.8 | +66.7 | +7.2 | 138 | +8.5 | — | — |
-| ctl_m15_5yr | 1578 | +128.0 | 1.21 | 35.8 | -3.4 | +131.4 | 535 | +12.6 | — | — |
-| ctl_m15_oos | 299 | +38.8 | 1.36 | 15.5 | +5.9 | +32.9 | 86 | -11.7 | — | — |
-| ctl_m5_5yr | 911 | +128.4 | 1.39 | 12.8 | -8.3 | +136.7 | 110 | +3.8 | — | — |
-| ctl_m5_oos | 86 | +9.4 | 1.33 | 8.2 | -1.4 | +10.9 | 4 | +1.5 | — | — |
-
-Raw table (n / R / PF / WR% / DD / H1 / H2 / PF-half-1 / PF-half-2 /
-sellN / sellR / ΔR / ΔPF / Δn vs control; control and `base_m5_12mo`
-rows carry no delta):
-
-```
-book                           n        R    PF   WR%    DD      H1      H2   PF1   PF2 sellN   sellR |      ΔR    ΔPF    Δn
-b1_h1_nosell_12mo             91    +22.2  1.79  62.6   4.1   +20.6    +1.6  2.10  1.18     0    +0.0 |    -1.8  +0.25  -47
-b1_h1_nosell_5yr             335    +49.1  1.46  58.2  13.2    -5.0   +54.1  0.87  1.78     0    +0.0 |   +16.7  +0.28 -180
-b1_h1_nosell_oos              78    +16.7  1.74  62.8   3.6    +7.3    +9.4  1.67  1.81     0    +0.0 |   +11.5  +0.61  -26
-b1_m15_sellB_12mo            341    +74.6  1.62  62.8  10.8   +68.4    +6.2  2.15  1.10    82    +9.1 |    +0.7  +0.09  -56
-b1_m15_sellB_5yr            1378   +119.1  1.22  57.0  41.4   -21.3  +140.3  0.91  1.46   335    +3.7 |    -8.9  +0.02 -200
-b1_m15_sellB_oos             272    +47.1  1.51  60.7  13.8   +12.3   +34.8  1.24  1.85    59    -3.4 |    +8.3  +0.15  -27
-b1_m5_selltrans_12mo         557   +112.2  1.56  53.7   9.9   +55.9   +56.4  1.54  1.58    75   +14.8 |    +7.9  +0.06  -18
-b1_m5_selltrans_5yr          889   +133.7  1.41  51.5  12.8    -8.3  +142.0  0.84  1.52    88    +9.0 |    +5.3  +0.03  -22
-b1_m5_selltrans_oos           86     +9.4  1.33  53.5   8.2    -1.4   +10.9  0.91  1.95     4    +1.5 |    +0.0  +0.00   +0
-b2_h1_nodiv_12mo             149    +28.6  1.62  62.4   6.5   +23.6    +5.0  1.93  1.24    49    +5.5 |    +4.6  +0.07  +11
-b2_h1_nodiv_5yr              567    +40.5  1.20  56.3  14.1    -5.2   +45.6  0.93  1.37   203   -18.2 |    +8.1  +0.02  +52
-b2_h1_nodiv_oos              119     +7.6  1.17  56.3   7.7    -1.9    +9.5  0.91  1.44    31   -14.3 |    +2.4  +0.04  +15
-b2_m15_nodiv_12mo            439    +81.3  1.51  60.8  13.1   +72.3    +8.9  1.90  1.11   147    +7.0 |    +7.4  -0.02  +42
-b2_m15_nodiv_5yr            1771   +139.9  1.20  55.8  47.4   -16.2  +156.1  0.94  1.39   578   +11.7 |   +11.9  -0.01 +193
-b2_m15_nodiv_oos             341    +52.3  1.43  59.2  17.1   +15.6   +36.7  1.23  1.68    93   -10.0 |   +13.5  +0.07  +42
-b3_h1_bb0_12mo               154    +31.2  1.64  60.4   9.4   +22.8    +8.4  1.89  1.36    52    +6.7 |    +7.3  +0.09  +16
-b3_h1_bb0_5yr                583    +37.3  1.18  54.5  18.3    -7.4   +44.7  0.90  1.35   206   -20.4 |    +4.9  +0.00  +68
-b3_h1_bb0_oos                122     +5.6  1.12  55.7   8.7    -2.0    +7.6  0.91  1.33    32   -13.8 |    +0.4  -0.01  +18
-b3_m15_bb0_12mo              439    +49.9  1.29  59.0  19.4   +61.6   -11.8  1.77  0.87   151    +0.3 |   -24.0  -0.23  +42
-b3_m15_bb0_5yr              1790   +126.6  1.18  56.1  42.3    -1.8  +128.5  0.99  1.31   592   +17.4 |    -1.3  -0.03 +212
-b3_m15_bb0_oos               345    +55.5  1.47  61.2  12.4   +18.2   +37.3  1.29  1.66    94    -8.4 |   +16.7  +0.11  +46
-b3_m5_bb0_12mo               621    +84.0  1.35  51.4  16.6   +49.3   +34.7  1.43  1.28   104    +9.7 |   -20.4  -0.14  +46
-b3_m5_bb0_5yr                987   +113.0  1.31  50.7  15.3    -7.7  +120.6  0.85  1.38   134   +12.0 |   -15.5  -0.08  +76
-b3_m5_bb0_oos                 93    +10.6  1.35  54.8   5.6    -2.9   +13.5  0.85  2.17     9    +3.7 |    +1.2  +0.01   +7
-b12_h1_nosell_nodiv_12mo     100    +23.1  1.75  63.0   4.6   +21.8    +1.3  2.00  1.15     0    +0.0 |    -0.8  +0.20  -38
-b12_h1_nosell_nodiv_5yr      364    +58.7  1.52  59.6  12.9    -4.1   +62.8  0.90  1.87     0    +0.0 |   +26.3  +0.34 -151
-b12_h1_nosell_nodiv_oos       88    +21.9  1.92  63.6   4.8    +9.4   +12.5  1.93  1.92     0    +0.0 |   +16.8  +0.79  -16
-base_m5_12mo                 575   +104.3  1.50  52.9  10.7   +50.8   +53.5  1.46  1.53    93    +6.9
-ctl_h1_12mo                  138    +24.0  1.55  60.9   8.0   +19.4    +4.6  1.84  1.22    47    +1.8
-ctl_h1_5yr                   515    +32.4  1.18  55.7  15.2    -5.9   +38.3  0.91  1.34   180   -16.7
-ctl_h1_oos                   104     +5.1  1.13  56.7   7.5    -0.9    +6.1  0.95  1.32    26   -11.5
-ctl_m15_12mo                 397    +73.9  1.53  62.2  10.8   +66.7    +7.2  1.95  1.10   138    +8.5
-ctl_m15_5yr                 1578   +128.0  1.21  56.5  35.8    -3.4  +131.4  0.99  1.37   535   +12.6
-ctl_m15_oos                  299    +38.8  1.36  59.2  15.5    +5.9   +32.9  1.10  1.66    86   -11.7
-ctl_m5_5yr                   911   +128.4  1.39  51.3  12.8    -8.3  +136.7  0.84  1.48   110    +3.8
-ctl_m5_oos                    86     +9.4  1.33  53.5   8.2    -1.4   +10.9  0.91  1.95     4    +1.5
-```
-
-## Decisions
-
-- **SHIPPED** — M5 `xau_strategy.sell_blocked_regimes=["transitioning"]`
-  (b1_m5_selltrans): 12mo +7.9R, 5yr +5.3R, OOS unchanged (no such trades
-  in that window); positive in both 12mo halves on every window. H1
-  `xau_strategy.sell_min_quality=[]` (b1_h1_nosell, all H1 shorts blocked):
-  5yr +16.7R and OOS +11.5R more than pay for the 12mo cost of -1.8R; H1
-  shorts were -16.7R/5yr and -11.5R/OOS in the control. Both land in
-  `pair_signals/xauusd.py`.
-- **REJECTED** — M15 `sell_min_quality=["B"]` (b1_m15_sellB): 5yr -8.9R
-  with max DD widening 35.8 -> 41.4R for a marginal +0.7R on 12mo; fails
-  the drawdown clause. All three B3 Bollinger-weight-0-in-trending-regimes
-  runs (b3_h1_bb0, b3_m15_bb0, b3_m5_bb0): negative on M5 (-20.4R/12mo,
-  -15.5R/5yr) and M15 (-24.0R/12mo, PF 1.29 -> still below the 12mo H2
-  going negative at -11.8R); H1's B3 5yr gain (+4.9R) doesn't clear the
-  bar once M5/M15 fail outright — Bollinger's constant-oppose vote stays
-  a load-bearing quality bar, consistent with the 2026-08-04 audit.
-- **DECLINED** — B2-M15 (`technical.disabled_indicators=["divergence"]`
-  on M15, b2_m15_nodiv): R improves on every window (+7.4/+11.9/+13.5R)
-  but 5yr max DD widens 35.8 -> 47.4R while PF stays flat (1.21 -> 1.20) —
-  more risk for no risk-adjusted gain; owner call.
-- **PARKED** — B2-H1 (`technical.disabled_indicators=["divergence"]` on
-  H1, b2_h1_nodiv, standalone: +4.6/+8.1/+2.4R, PF 1.62/1.20/1.17). The
-  combined run with the H1 no-short override (b12_h1_nosell_nodiv) passes
-  rule 1 and beats the no-shorts override alone on both 5yr and OOS: 12mo
-  100/+23.1R PF 1.75 (halves +21.8/+1.3), 5yr 364/+58.7R PF 1.52, OOS
-  88/+21.9R PF 1.92. Not shipped anyway: it would route through
-  `technical.disabled_indicators`, which CLAUDE.md rule 8 reserves for
-  experiments, not production paths. Owner decision; recommended vehicle
-  is an H1-only `regime_weights_override` with divergence weight 0 plus a
-  fresh validation run under that mechanism.

backtest/research/signal_log_eval_2026-08-18.md

@@ -1,113 +0,0 @@
-# Signal-log live performance evaluation — 2026-08-18
-
-**Tool:** `analyze_signal_log.py` (new, repo root). Read-only on
-`logs/signals.db` (sqlite `immutable=1`); replays every logged trade
-setup through the backtester's own `simulate_trades` on the RAW Oanda
-price path (next-open fill, SL/TP distances preserved, SL-first ties,
-gap-at-open fills, close-only trail ratchet with the per-setup stamped
-trail params). Reproduce:
-
-```bash
-python analyze_signal_log.py --csv backtest/signal_log_eval_20260818.csv
-```
-
-## Window & sample
-
-- Log window: **2026-08-13 14:31 → 2026-08-18 14:13 UTC (5 days)** —
-  the entire signal_log history to date.
-- 2,654 rows (telegram 2,180 / dashboard 474); M5 1,308, M15 1,189,
-  H1 119, H4 38.
-- 39 rows carry a `trade_setup` → **17 unique signal events** after
-  dedup (cross-source double-logs + persisting-signal re-fires collapse
-  on (tf, direction, entry, SL, strategy) with a 60-min gap).
-- **No real executions exist** (`oanda_state.json` empty, no
-  `oanda_daemon` rows) — every outcome below is a hypothetical replayed
-  fill, not a broker fill.
-- Cost: dual convention — **$0.65 rt headline**, $0.30 comparative
-  (derived analytically, R is linear in cost).
-
-## Trade replay results
-
-| First logged | TF | Signal | Grade | Strategy | Fill | Exit | Bars | R@0.65 |
-|---|---|---|---|---|---|---|---|---|
-| 08-13 15:30 | 5min | SELL | B | GENERIC | 4365.39 | TRAIL | 19 | −0.20 |
-| 08-13 18:00 | 15min | SELL | B | GENERIC | 4356.05 | TRAIL | 44 | +0.91 |
-| 08-14 07:45 | 5min | BUY | B | XAU_CP | 4339.72 | TRAIL | 60 | +2.56 |
-| 08-14 12:15 | 15min | BUY | B | GENERIC | 4371.91 | TRAIL | 27 | +0.06 |
-| 08-14 12:25 | 5min | STRONG_BUY | A | GENERIC | 4379.12 | TRAIL | 277 | +0.20 |
-| 08-14 12:30 | 5min | STRONG_BUY | A | GENERIC | 4379.87 | TRAIL | 276 | +0.16 |
-| 08-14 12:30 | 15min | BUY | B | GENERIC | 4379.87 | TRAIL | 63 | +0.18 |
-| 08-17 08:00 | 1h | BUY | C | GENERIC | 4405.19 | TRAIL | 18 | −0.13 |
-| 08-17 14:15 | 5min | STRONG_BUY | A | GENERIC | 4412.35 | TRAIL | 46 | −0.10 |
-| 08-17 14:20 | 5min | STRONG_BUY | A | GENERIC | 4418.55 | TRAIL | 125 | −0.23 |
-| 08-17 14:25 | 5min | STRONG_BUY | A | GENERIC | 4424.10 | SL | 151 | −1.02 |
-| 08-17 14:30 | 15min | BUY | A | GENERIC | 4418.23 | SL | 17 | −1.03 |
-| 08-17 14:45 | 15min | STRONG_BUY | A | GENERIC | 4417.49 | SL | 16 | −1.03 |
-| 08-17 15:00 | 15min | STRONG_BUY | A | GENERIC | 4417.48 | SL | 15 | −1.03 |
-| 08-17 15:00 | 5min | NEUTRAL | C | GENERIC | 4417.48 | TRAIL | 117 | −0.16 |
-| 08-17 15:15 | 15min | STRONG_BUY | A | GENERIC | 4421.30 | SL | 14 | −1.04 |
-| 08-17 15:30 | 15min | BUY | B | GENERIC | 4425.70 | SL | 85 | −1.02 |
-
-**Aggregate: 17 closed, 6 wins (35%), total −2.91R @0.65 / −2.67R
-@0.30, PF 0.58.** By grade: A n=9 → −5.12R; B n=6 → +2.50R; C n=2 →
-−0.30R.
-
-Reading: the book is two regimes. Everything through the 08-14 rally
-made money (+4.7R across 8 events, incl. the XAU_CP +2.56R); the 08-17
-afternoon BUY/STRONG_BUY cluster chased the spike to ~4425 and got
-stopped as price faded — 9 events, −7.6R combined. One bad afternoon
-dominates the 5-day sample. Notable: a NEUTRAL-graded row carried a
-setup (post-gate CP/PB-style path) and lost small.
-
-## Advisory panel scorecard
-
-Only **7 verdicts** logged so far (2 NEUTRAL, 5 OPPOSE, 0 AGREE) — the
-panel's min-quality gate + daily budget keep it dark on most scans.
-
-| Row | When | Signal | Verdict | Outcome basis | Impact |
-|---|---|---|---|---|---|
-| #1358 | 08-17 07:40 M5 | BUY B | NEUTRAL @55% | +4h move | −$8.19 in dir (pass was right) |
-| #1361 | 08-17 07:45 M5 | BUY A | NEUTRAL @55% | +4h move | −$12.43 in dir (pass was right) |
-| #1550 | 08-17 14:15 M5 | STRONG_BUY A | OPPOSE @80% | replay | saved 0.10R |
-| #1553 | 08-17 14:25 M5 | STRONG_BUY A | OPPOSE @80% | replay | saved 1.02R |
-| #1555 | 08-17 14:30 M15 | BUY A | OPPOSE @75% | replay | saved 1.03R |
-| #1561 | 08-17 14:45 M15 | STRONG_BUY A | OPPOSE @75% | replay | saved 1.03R |
-| #1588 | 08-17 15:30 M15 | BUY B | OPPOSE @75% | replay | saved 1.02R |
-
-**Every OPPOSE was correct: ≈ +4.2R would have been lost taking those
-five trades.** The panel's rationale ("price climbed too fast, profit
-too small") correctly identified the 08-17 exhaustion spike that the
-composite scored A. Both NEUTRAL passes also preceded adverse moves.
-Caveat: n=7, one market episode, zero AGREE verdicts — the panel has
-only proven itself as a veto so far, never as a confirmation.
-
-## Directional hit-rate (non-setup signals)
-
-27 deduped non-setup BUY/SELL rows, forward close-to-close on M5:
-
-| Horizon | Hit rate | Mean move in direction |
-|---|---|---|
-| +1h | 48% | −$0.72 |
-| +4h | 44% | −$2.30 |
-| +24h | 7% | −$27.45 |
-
-The 24h figure is dominated by the 08-17 buy-cluster reversal; not
-independently meaningful at n=27.
-
-## Method notes / caveats
-
-- **Small sample**: 5 days, 17 events, one trending episode + one
-  reversal. Directional read only; do not tune anything on this.
-- Entry model = next bar open after the scan boundary (live scans run
-  at boundary+10s), SL/TP distances preserved from the logged setup —
-  identical to the backtester replay loop.
-- Cross-source dedup verified: telegram+dashboard double-logs and
-  5-min re-fires of a persisting signal collapse to one event each.
-- The stray repo-root `signals.db` (828 rows, Aug 13–14, pre-panel
-  schema) is an early duplicate subset of `logs/signals.db` — excluded.
-- Tests: Module 25 in `test_all.py` was only added on 2026-09-03 (the
-  2026-08-18 claim of 7 tests was never true at HEAD). It now covers
-  boundary flooring, clustering (shared `signal_log.setup_key`), trade
-  building, the stored-candle replay books, surface parity, WAL-visible
-  loading (`mode=ro`; `immutable=1` missed the newest rows) and hit-rate
-  signing.

backtest/research/signal_parity_2026-08-14.md

@@ -1,59 +0,0 @@
-# Signal-Level Replay Parity — Oanda vs Tiingo (Phase 2)
-
-**Date:** 2026-08-14
-**Tool:** `scripts/signal_parity_study.py` (script removed 2026-09-06, in git history) (replays real `analyze_from_dataframes`, news off)
-**Window:** 2025-08-14 → 2026-08-14 (12 months); M15 stride 8, M5 stride 24 → ~2,900 sampled decisions each
-**Preceding:** Phase 1 price-diff study (`feed_divergence_2026-08-14.md`)
-
-## Headline
-
-On the ~94–96% of bars the two feeds share, **Oanda and Tiingo produce near-identical signals**:
-
-| Metric | M15 | M5 |
-|---|---|---|
-| same direction | 98.32% | 98.16% |
-| same grade | 94.13% | 95.23% |
-| same setup presence | 98.83% | 99.49% |
-| same direction + grade + setup | 93.41% | 94.21% |
-| opposite-sign flips (BUY↔SELL) | **0** | **0** |
-| direction flips on actionable bars | 6.6% | 10.9% |
-| grade changed on actionable bars | 14.2% | 7.8% |
-| mean \|Δ final_score\| | 1.76 | 1.78 |
-| mean \|Δ primary_score\| | 1.73 | 1.77 |
-| same strategy tag | 100% | 100% |
-
-Every direction flip is **actionable↔NEUTRAL**, never a sign reversal: the feeds never disagree on buy-vs-sell, only on *whether* a marginal bar crosses the action threshold. Flipping bars cluster at the Friday-close/Sunday-reopen boundary (UTC hours 22–23) and in the Asian hours (03–04).
-
-## Setup levels diverge even when the signal agrees
-
-On bars where **both** feeds emit a setup with the same tag:
-
-| | M15 (n=119) | M5 (n=39) |
-|---|---|---|
-| entry mean \|Δ\| | $0.12 | $0.10 |
-| SL mean \|Δ\| | $1.83 | $2.79 |
-| TP1 mean \|Δ\| | $2.47 | $1.52 |
-| TP2 mean \|Δ\| | $3.74 | $2.46 |
-| TP1 p95 \|Δ\| | $17.4 | $2.3 |
-
-Entry price is the current close (≈ identical across feeds), but **SL/TP are recomputed from ATR and recent swing extremes, which differ more** — mean SL drift ~$2–3 (p95 up to $10–26 on M15), i.e. 20–40% of ATR. So even an identical signal places stops/targets at meaningfully different prices on a different feed.
-
-## Phase 2 pass/fail vs plan thresholds
-
-| Criterion | Plan threshold | M15 / M5 | Verdict |
-|---|---|---|---|
-| identical direction+grade | ≥ 97% | 93.4% / 94.2% | **marginal** (below 97%, above 93%) |
-| opposite-sign flips | (none specified) | 0 / 0 | PASS |
-| flips clustered at marginal composites | no session/regime clustering | yes (boundary hours) | PASS |
-
-The "identical direction+grade" metric lands slightly under the proposed 97% bar — but the shortfall is grade *letters* (A+ vs A vs B) changing on marginal composites, not direction. If the acceptance criterion is "no opposite-sign disagreement and ~98% direction identity", this passes comfortably.
-
-## Conclusion
-
-Oanda ↔ Tiingo is **signal-equivalent for direction** (98% identical, zero sign flips) over 12 months. The residual deltas are:
-
-1. **~6% of decisions differ in grade letter** on marginal composites (A+↔A↔B near thresholds).
-2. **SL/TP levels drift ~$2–4 mean** (up to ~$20 p95 on M15) even for identical signals — a real P&L difference at the trade level.
-3. **Tiingo's extra Sunday bars** (Phase 1) are gated by the aggregator's NY-anchored weekend filter, so they do not inject phantom signals, but they do slightly shift indicator history at Monday opens.
-
-Net: switching Oanda→Tiingo changes a handful of marginal decisions and moves stop/target levels, but does not change *what the system is doing* (direction, strategy, setup presence). Oanda↔Twelve Data was not replayed — Phase 1 already showed 81% M5 bar-direction disagreement and a ~$3/$1 systematic gap, so a TD replay would trivially fail (that is the documented re-pin motivator).

backtest/research/sizing_campaign_2026-09.md

@@ -1,348 +0,0 @@
-# Stake-sizing campaign — 2026-09-05 (3-year Oanda window, $0.65)
-
-## Purpose and owner decisions
-Owner asked (2026-09-05): predict direction and choose order size better; protect against
-trend exhaustion at signal time; map hours/days where institutions move gold violently;
-backtests ≤ 3 years. Decisions taken in planning:
-- Order size = a per-signal RELATIVE stake ("Stake: 0.7× normal"), not account-based units.
-- Exhaustion protection = RISK LESS, DON'T SKIP (every hard exhaustion block tested since
-  2026-07 removed net-positive trades — the winners are the stretched runs).
-- Model only M5 and M15; H1/H4 measured descriptively.
-- scikit-learn / lightgbm allowed in ./venv for research only; the shipped artefact is
-  numpy-only JSON. Neither library enters requirements.txt.
-- No tuned threshold, gate or exit changes (rule 1). Sizing is a post-decision overlay computed
-  by `position_sizing.build_features` / `stake_multiplier` INSIDE `analyze_from_dataframes`, so
-  live and backtest use the same call on the same objects (rule 4).
-
-## Harness
-`backtest/gpu_backtest.py --pair XAUUSD --start 2023-09-05 --end 2026-09-05 --spread 0.65
---workers 12`, pair-factory config (no --set/--quality/--strategies/--confirm-tfs), NumPy CPU
-backend (CuPy cannot locate CUDA_PATH on this box; the 2026-09-03 pinned books were also CPU).
-Cache: `oanda_XAU_USD_{5min,15min,1h,4h}_20230705_20260905.parquet`, `…_1d_20221110_20260905`.
-Feature books add `--export-features` (65 `f_*` columns + `stake_multiplier` + `r_sized`; the
-default export stays the pinned 63-column shape). Outputs in `backtest/research/sizing_2026-09/`
-(gitignored).
-
-## Baseline (control books, unmodified backtester, 2026-09-05 23:36–23:41 UTC)
-| TF | trades | total R | WR | PF | max DD |
-|---|---|---|---|---|---|
-| M5 | 857 | +161.32 | 53.7% | 1.53 | 9.85 |
-| M15 | 1086 | +148.69 | 59.1% | 1.37 | 15.47 |
-| H1 | 255 | +70.52 | 64.3% | 1.98 | 4.43 |
-| H4 | 113 | +22.68 | 58.4% | 1.63 | 5.39 |
-
-### Behaviour-neutrality checks (2026-09-05 23:57 UTC)
-- 12-month M5 canonical window (2025-08-14 → 2026-08-14, $0.65) re-run on the modified tree:
-  `book_diff` vs `backtest/oanda_m5_12mo_065_nopb_20260903.csv` = ΔR +0.0, Δtrades +0, and the
-  two CSVs have the SAME MD5 (`a28fed4becd056466c26d6a947e1a0e5`). Record-keeping fields,
-  `SizingConfig`, and the `--export-features` hook change nothing when the flag is off.
-- Feature books: the 63 shared columns are value-identical to the control books on every TF
-  (`assert_frame_equal`). A column-ORDER quirk was found and fixed the same day: pandas orders
-  export columns by first appearance, so when the first trade of a book lacks an indicator
-  (the 3-year M5 book's first trade had no candlestick pattern) that indicator's columns were
-  appended after the extras. `trades_to_dataframe` now forces the extras to the tail; test
-  extended with that exact case (435 tests green).
-
-## Pre-registered acceptance rule (written 2026-09-05, BEFORE any model fit)
-A per-timeframe stake-multiplier model may ship only if, evaluated on OUT-OF-SAMPLE quarters
-only (all calendar quarters after a 12-month burn-in, each predicted by a model trained
-exclusively on trades whose entry time is ≥ 300 primary bars before that quarter's start,
-expanding window, standardisation fit on train only), ALL of the following hold:
-1. Sized total R > flat total R in BOTH halves of the OOS period (split at the median OOS
-   entry time).
-2. Sized total R > flat total R over the full OOS period.
-3. Sized max drawdown (0-prepended cumulative sized-R curve, as `compute_metrics`) ≤ flat max
-   drawdown × 1.10.
-4. Sized OOS book has PF ≥ 1.3 and positive total R in both halves (the rule-1 floor).
-5. Real OOS lift > the 95th percentile of the lift distribution from 50 label-shuffle refits
-   (R permuted within training windows only; test labels and walk-forward geometry unchanged).
-6. Mean multiplier over OOS trades within 1.00 ± 0.05; every multiplier in [0.5, 1.5].
-Plus the mechanical check: the `--sizing` full-window replay reproduces the flat book's
-entry/exit/`r_multiple` columns exactly (sizing changes no entries).
-Model family and size map (fixed): primary = ridge regression on winsorised R (clip to
-[−1.5, +4]); secondary = L2 logistic on R>0 (ships only if the primary fails and it passes);
-simplicity baseline = one-feature model on `f_is_buy` (the shipped model must beat its OOS
-lift, else the simpler one ships if it passes); upper bound = lightgbm on all `f_*` (never
-ships; if its OOS lift is inside the shuffle null, stop); ablation = exhaustion-block-only model.
-Map: `m = 0.5 + 1.0 × clip((z − q10)/(q90 − q10), 0, 1)` with q10/q90 from TRAIN predictions,
-times a normaliser making the mean train multiplier 1.0, clipped to [0.5, 1.5], rounded to 0.05.
-`SHIPPED_FEATURES` (≤ 12 per TF, ≥ 2 from the exhaustion block) are chosen from the Stage-B
-sign-stable screen on the 3-year feature books and recorded below BEFORE the first fit; any
-later change restarts the shuffle control. If any clause fails on a timeframe, that timeframe
-ships nothing. No widening the clip, changing the split, dropping the purge, re-picking
-features or adding quarters to rescue a result.
-
-## Pre-registered feature lists
-Derived MECHANICALLY (script in the session log, saved as `sizing_2026-09/preregistered_features.json`):
-sign-stable `f_*` features from the Stage-B screen (top-minus-bottom quintile mean-R gap has the
-same sign in both halves, both halves ≥ 10 trades per bucket), zero-variance columns removed,
-ranked by min(|gap_h1|, |gap_h2|), top 12; the ≥ 2-from-the-exhaustion-block clause was already
-satisfied on both TFs. Written 2026-09-06 00:05 UTC, before any Stage-D fit.
-- **M5 (29 sign-stable candidates):** f_sl_atr, f_confidence, f_rsi, f_mtf_c1, f_swing_dist_atr,
-  f_ret_48_atr, f_tag_no_pullback, f_rsi_dir, f_first_friday, f_mtf_agreement,
-  f_exhaustion_count, f_ret_24_atr. Exhaustion-block members: f_swing_dist_atr,
-  f_tag_no_pullback, f_exhaustion_count (all three run POSITIVE in both halves).
-- **M15 (37 sign-stable candidates):** f_rsi, f_bb_pos, f_day_of_month, f_tag_ema200_near,
-  f_atr_pct, f_is_buy, f_confidence, f_close_pos_in_bar, f_tag_weak_pullback, f_tag_exhaustion,
-  f_penalty_total, f_ret_6_atr. Exhaustion-block members: f_tag_exhaustion, f_penalty_total
-  (both NEGATIVE in both halves; f_tag_weak_pullback and f_tag_ema200_near also negative).
-Exhaustion-only ablation list (fixed by the plan): f_swing_dist_atr, f_ema21_dist_atr,
-f_ema200_dist_atr, f_exhaustion_count, f_penalty_total, f_tag_bb_overextended,
-f_tag_rsi_extended, f_tag_no_pullback, f_tag_exhaustion.
-
-## Stage B — book breakdown (3-year feature books)
-Tool: `backtest/research/book_report.py` (outputs in `sizing_2026-09/report_{5min,15min,1h,4h}/`).
-Halves = median TRADE split (M5 cut 2025-12-12 because trade density doubled in the last year;
-M15 cut 2025-04-25; H1 2025-04-09).
-
-**M5 (857 trades, +161.3R, PF 1.53, DD 9.85; halves +51.0R PF 1.33 / +110.3R PF 1.74; max 11
-wins / 8 losses in a row; longest flat stretch 88 trades).** Mix: GENERIC 469, XAU_CP 388, BUY
-765 / SELL 92 (SELLs +11.8R, positive both halves). Grade: A +0.26R/trade (227) > B +0.17 (608)
-> C −0.01 (21). Session (decision hour): NY 17–22 best (+0.31R/trade, PF 1.88), overlap 13–16
-weakest (+0.08, PF 1.22) — both stable. Weekday: Wed and Fri best and stable; Mon/Tue flip sign.
-Feature screen: 29 sign-stable `f_*` features. **Every stretch measure is POSITIVE**: wider
-stop in ATR (+0.22/+0.23 gap), higher RSI (+0.20/+0.19), farther from the last swing
-(+0.18/+0.27), 48-bar run-up (+0.18/+0.20), the `no_pullback` tag (+0.17/+0.28), more exhaustion
-tags (+0.12/+0.24). Only ATR-relative-to-500-bar-mean is negative (−0.11/−0.12: entries during
-a volatility spike do worse). "Risk less when stretched" has NO support on M5 — a model that
-honours the data will size stretched M5 trades UP, not down.
-
-**M15 (1086 trades, +148.7R, PF 1.37, DD 15.47; halves +65.8R / +82.9R; max 12 wins / 8 losses;
-flat stretch 131 trades).** All GENERIC; BUY 739 (+158.4R, PF 1.62) vs SELL 347 (−9.7R, PF 0.93;
-−19.2R in half 1, +9.4R in half 2). Grade NOT monotone: C +0.45R/trade (82, PF 2.41) > A +0.14
-(481) > B +0.08 (516) — stable in both halves (matches the 2026-08-04 quant review). Session:
-Asian and NY ≈ +0.18–0.20R/trade, London and overlap ≈ +0.09–0.10 (all stable). Weekday:
-Wednesday flat/negative in both halves (229 trades, −0.6R); Thu/Fri flip sign between halves.
-Feature screen: 37 sign-stable features. Here the exhaustion block runs NEGATIVE: `exhaustion`
-tag (−0.19/−0.10), `weak_pullback` tag (−0.24/−0.11), `ema200_near` tag (−0.17/−0.32), penalty
-total (−0.16/−0.10), ATR percentile (−0.20/−0.15); while raw RSI (+0.31/+0.24) and Bollinger
-position (+0.36/+0.22) are positive. So on M15 "stretched by the aggregator's own penalty
-logic" is bad while "strong momentum reading" is good — the sizing model gets both.
-
-**H1 (255 trades, +70.5R, PF 1.98, DD 4.43; halves +28.7R PF 1.79 / +41.9R PF 2.17)** and
-**H4 (113 trades, +22.7R, PF 1.63, DD 5.39)** are descriptive only. H1 is all-BUY by
-construction (shorts blocked 2026-09-03); Mon/Thu strongest, Friday the only losing weekday
-(−1.4R/50, flips between halves); 03:00 UTC decision hour is the best single hour (+20.0R/40,
-stable) and 04:00–05:00 negative.
-
-**Labelling quirk (pre-existing, not fixed):** 33 M5 trades carry the backtester `strategy`
-tag GENERIC while their setup strategy is XAU_ATR (`f_strat_atr = 1`); the CSV tag and the
-setup's own string disagree for the ATR path. Labels only — R is unaffected.
-
-**Stop-trigger check: PASSED** (both TFs have many sign-stable features) → proceed to Stage D.
-
-## Stage C — institutional burst / time map
-Tool: `backtest/research/burst_map.py` on the 3-year raw bars (`burst_5min/`, `burst_15min/`
-under `sizing_2026-09/`). Burst = bar range ≥ K × prior Wilder ATR(14). Base rate at K=3:
-0.86 % of M5 bars (1840), 1.19 % of M15 bars (846).
-
-**Where the violent bars cluster (bar-CLOSE hour, UTC, K=3 burst % of bars):**
-- 12:00–13:00 (US data window): M5 2.4 % / 2.3 %, M15 3.7 % / 5.0 % — 3–4× the base rate;
-  Friday 12:00–13:00 is the densest weekday cell on M5 (3.4 % / 3.1 %). Largest mean 1-hour
-  dollar moves are at close-hours 13–15 (M5: $7.8 / $9.4 / $8.5 vs ≈$5 elsewhere).
-- 01:00 (Asia open): M5 2.9 %, M15 3.3 %. Also 05:00 (Tokyo/London hand-over) ≈1.2 % on M5.
-- 22:00–23:00 (post-settlement thin book): ≈1.3–1.4 % on M5; Sunday reopen 22–23 h is the most
-  burst-prone slot of all (2.4–3.0 % of bars) but the calendar already blocks Sunday entries
-  before 18:00 NY.
-- Quietest hours: 03:00–04:00 and 16:00–17:00 (0.14–0.3 %).
-
-**Post-burst drift (in the burst's own direction, ATR units):** ≈ 0 at every horizon on both
-TFs (M5 6/12/24/48-bar means −0.01/−0.01/+0.09/+0.05; M15 −0.08/+0.14/+0.18/+0.17; % positive
-46–51 %). London-session bursts on M5 show a mild short-horizon reversal (42 % positive at 6
-bars). No continuation or reversal edge worth a rule.
-
-**Do bursts hurt the system's entries? No.** Trades DECIDED within 12 bars after a K=3 burst:
-M5 149 trades, mean +0.199R, PF 1.62 vs +0.186R / PF 1.52 for the rest; M15 242 trades,
-+0.160R / PF 1.47 vs +0.130R / PF 1.34. At K=4/5 the after-burst group is better still
-(M5 +0.276R PF 1.98; M15 +0.259R PF 1.80). This re-confirms the 2026-07-30 big-candle-gate
-rejection with 3 years of data: expansion bars are part of the entry signal, not a danger.
-(Group (b) "burst while open" is survivorship-shaped — winners live longer — and is not evidence.)
-
-**The 40 largest 1-hour moves** are almost all one episode (2026-01-28 → 2026-02-06, incl. a
-−$394 hour at 15:45 UTC on 2026-01-29, and 2026-03-23), not recurring clock slots; only 3 of 40
-fall in the 12:30 data slot and 1 in the Wednesday 18:00 central-bank slot (slot labels only —
-no economic calendar file exists in the repo). Violent moves are episodic regime events; the
-recurring, predictable clustering is the 12:00–15:00 UTC US-data window and the 01:00 Asia open.
-
-**Use in this campaign:** `f_bars_since_burst` and the hour/session features are already in the
-feature set, so Stage D can weigh them as a stake input; no entry gate is proposed (rule 1).
-
-## Stage D — walk-forward results (2026-09-06 00:20 UTC)
-Driver: `backtest/research/sizing_model.py` (numpy closed-form ridge + Newton logistic; lightgbm
-4.7 + scikit-learn 1.9 installed in ./venv for the upper bound only; NOT in requirements.txt).
-Outputs: `sizing_2026-09/sm_{m5,m15}_{ridge,logistic}/` (summary.txt, oos_trades.csv,
-quarters.csv, coefs_by_quarter.csv, shuffle_lifts.csv). Validation of the driver: planted
-synthetic effect recovered (lift +116.7R, shuffle rank 100 %, all clauses PASS); pure-noise
-book correctly FAILS (rank 32 %); ridge matches `lstsq` to 2e-16; logistic gradient < 3e-15.
-
-**M5 (5 OOS quarters 2025Q3→2026Q3, 619 trades — 2024Q4–2025Q2 skipped by the pre-registered
-min-train rule because only 96/119/142 trades precede them; 2026Q1 alone is +92R of the +125R
-flat OOS):**
-| model | flat R | sized R | lift | PF | DD | mean m | shuffle p95 | rank | verdict |
-|---|---|---|---|---|---|---|---|---|---|
-| ridge (primary) | +125.41 | +121.98 | −3.43 | 1.63 | 10.69 | 0.891 | +13.69 | 48 % | FAIL (1,2,5,6) |
-| logistic | +125.41 | +128.40 | +2.99 | 1.66 | 8.70 | 0.900 | +14.39 | 62 % | FAIL |
-| isbuy baseline | +125.41 | +123.22 | −2.20 | 1.55 | 11.92 | 1.000 | +7.05 | 4 % | FAIL |
-| exhaustion-only | +125.41 | +118.03 | −7.38 | 1.59 | 10.88 | 0.918 | +14.36 | 28 % | FAIL |
-| lightgbm (all 65) | +125.41 | +116.05 | −9.36 | 1.51 | 11.64 | 1.007 | +7.60 | 10 % | FAIL |
-Ridge weights (standardised, 5 quarters): `f_tag_no_pullback` +0.018 sign-stable 5/5 — the
-"no pullback" flag INCREASES the stake; `f_swing_dist_atr` and `f_exhaustion_count` ≈ 0 and
-sign-unstable. Exhaustion-only sizing LOSES 7.4R. **A clean null: nothing in the 65 features
-predicts M5 trade outcomes beyond noise, linear or non-linear.**
-
-**M15 (8 OOS quarters 2024Q4→2026Q3, 762 trades; flat +115.93R PF 1.42 DD 13.73):**
-| model | sized R | lift | PF | DD | mean m | shuffle p95 | rank | clauses |
-|---|---|---|---|---|---|---|---|---|
-| ridge (primary) | +142.04 | +26.11 | 1.53 | 14.46 | 0.980 | +28.50 | 88 % | 1,2,3,4,6 PASS · **5 FAIL** |
-| logistic | +144.78 | +28.84 | 1.53 | 13.97 | 1.002 | +31.89 | 92 % | 1,2,3,4,6 PASS · **5 FAIL** |
-| isbuy baseline | +139.38 | +23.45 | 1.50 | 16.44 | 1.007 | +24.47 | 58 % | FAIL (3, 5) |
-| exhaustion-only | +123.83 | +7.90 | 1.45 | 14.85 | 1.010 | +19.77 | 70 % | FAIL |
-| lightgbm (all 65) | +123.87 | +7.94 | 1.44 | 14.74 | 0.996 | +17.66 | 60 % | FAIL |
-Ridge weights sign-stable in 8/8 quarters: `f_tag_exhaustion` −0.022, `f_tag_ema200_near`
-−0.033 (both SHRINK the stake — the "risk less when stretched" direction), `f_rsi` +0.028,
-`f_bb_pos` +0.030, `f_is_buy` +0.029, `f_close_pos_in_bar` +0.022, `f_ret_6_atr` +0.022;
-λ = 1000 every quarter (heavy shrinkage — the score is weak and the map is essentially a rank
-transform). Sized R beat flat in 6 of 8 quarters and in both halves, BUT the label-shuffle
-null has p95 = +28.5R: a random re-weighting of these 12 pre-screened features produces the
-same lift 12 % of the time, and the one-feature BUY/SELL baseline already delivers +23.5R of
-the +26.1R. **The model's information beyond "buys are better than sells" (≈ +2.7R over 762
-trades) is indistinguishable from noise.**
-
-## Verdict (pre-registered rule applied, no rescue attempted)
-**FAIL on both timeframes → no stake multiplier ships.** No artifact, no `sizing.enabled`, no
-alert line, no `--sizing` backtester flag. What remains in the tree is behaviour-neutral
-research plumbing: `position_sizing.build_features` (65 decision-time features; the multiplier
-path returns 1.0 and no artifact exists), `SignalReport.penalty_tags/penalty_total/
-momentum_fading` (record-keeping), `TradeSetup.stake_multiplier` (None; reserved), the
-`--export-features` flag, `book_report.py`, `burst_map.py`, `sizing_model.py`, and Module 29
-tests. `SizingConfig` was REMOVED again before merge (rule 8: nothing reads it).
-
-**What the owner should take from this (plain language):**
-1. On the 5-minute chart there is nothing to size on. Stretched trades are the good ones; the
-   system's exhaustion flags carry no usable information about the outcome; a flexible model
-   given all 65 facts does worse than flat sizing out of sample.
-2. On the 15-minute chart the only robust sizing signal is BUY vs SELL, which the gates already
-   act on. The exhaustion flag does point the "risk less" way in every one of 8 unseen
-   quarters, but the money it would make (≈ +3R per year) is inside noise at 1,086 trades.
-3. "Predict direction better" with a learned second opinion is closed at this sample size:
-   the strongest non-linear model (all features, gradient boosting) fails on both charts.
-
-**Open avenues (not rescues — each needs a NEW pre-registration):**
-- Re-run the M15 exhaustion-tag sizing when the book has ≈ 2× the trades (≈ 2028) or on a
-  sealed forward sample; the sign is stable, the magnitude is not proven.
-- The shuffle null is wide because the size map rank-stretches a weak score to 0.5–1.5×; a
-  narrower map (e.g. 0.75–1.25×) would shrink both lift and null — pre-register before testing.
-- Session and weekday structure (M5: NY best / overlap weakest; M15: Wednesday flat) is stable
-  across halves but was rejected as a gate repeatedly; as a *size* input it was inside the
-  12-feature model and did not survive the null.
-
-## Addendum 2026-09-06 — violent bars: signs before them, and protecting OPEN trades
-Owner follow-up: "we might have an order running for hours before a violent move; we want
-protection." Tools: `backtest/research/burst_precursors.py` (signs before a burst) and
-`backtest/research/open_trade_bursts.py` (exact replay of every book trade on raw bars with the
-backtester's exit model — reproduces `r_multiple` on 100 % of trades — plus protection variants).
-Outputs: `sizing_2026-09/burst_precursors_{m5,m15}.txt`, `open_trade_bursts_{m5,m15}.txt`.
-
-**Signs before a burst (both TFs, both halves stable):** the clock (close-hours 01, 12, 13 UTC
-hold ~40 % of bursts in 13 % of bars); tick activity in the last 3 bars vs 50-bar mean (top
-quintile ≈ 2× rate); the last bars already wider than normal (top quintile ≈ 2.2×; the
-tightest quintile ≈ 0.3–0.5× — NOT calm-before-storm at bar scale); a move already under way
-over the past hour (≈ 1.6×); a burst minutes ago (≈ 1.4×). Out of sample (fit half 1, score
-half 2) a logistic model reaches AUC 0.75–0.77 for the next bar; the top 5 % of moments carry a
-3.4–5.5 % burst chance vs 0.8–1.1 % base, and 24 % vs 9 % (M5) for "within the next hour".
-Burst direction continues the prior hour's drift 61 % of the time. Signs say "ripe", not "now".
-
-**Damage to open trades (base exits):** M5 — 164 trades (19 %) meet an opposing burst while
-open: mean open R +0.42 just before → −0.06 just after → final +0.01; 43 % are stopped on that
-bar; 54 % end as losers. M15 — 246 trades (23 %): +0.27 → −0.20 → final −0.12; 39 % stopped on
-the bar. Trades already in profit before the burst still finish positive on average (M5 +0.53,
-M15 +0.27). Bursts WITH the trade (n 176 / 246) finish at +1.11 / +0.83R — they are where the
-winners come from.
-
-**Protection variants — ALL negative on both TFs, both halves:**
-| variant | M5 ΔR (h1/h2) | M15 ΔR (h1/h2) |
-|---|---|---|
-| flatten before US-data hours (12–13 UTC) | −31.9 (−13.6/−18.3) | −103.0 (−67.4/−35.6) |
-| flatten before Asian open (00–01 UTC) | −38.7 (−22.0/−16.7) | −73.0 (−42.0/−30.9) |
-| protective 0.5R stop through the data hours | −29.6 (−12.3/−17.3) | −82.3 (−55.5/−26.8) |
-| protective 0.5R stop through the Asian open | −19.2 (−9.5/−9.7) | −35.1 (−17.3/−17.8) |
-| exit next open after an opposing burst | −19.1 (−6.5/−12.5) | −6.9 (−11.9/+5.0) |
-| 0.5R protective stop after an opposing burst | −9.0 (−3.0/−6.0) | −8.3 (−15.5/+7.3) |
-The existing stop/trail already contains the opposing-burst damage (≈ 0.4–0.5R per hit); every
-pre-emptive rule removes far more from the with-trade bursts than it saves. **Verdict: no
-open-trade protection rule ships; do not re-propose flatten/tighten-before-window or
-react-to-burst exits without a new mechanism.** Display-only "violent-move conditions" note on
-alerts is the only harmless use of the precursor signs.
-
-## Addendum 2026-09-06 — calendar events, indicator states and time slots before violent bars
-Tools: `backtest/research/calendar_impact.py` (needs `backtest/cache/calendar.parquet`, scraped
-2023-09-01→2026-09-10 with `ff_calendar_scraper.py`: 15,124 events, 1,055 high-impact USD in
-window) and `backtest/research/burst_indicator_map.py` (states from the backtester's own
-`precompute_indicators`, taken at t−1). Outputs `sizing_2026-09/calendar_impact_{m5,m15}.txt`,
-`burst_indicator_map_{m5,m15}.txt`. Burst = range ≥ 3× prior ATR14; lifts quoted are stable in
-BOTH halves on both TFs unless noted.
-
-**Calendar.** The M5 bar containing a high-impact USD release averages 4.4× normal range; 54 %
-of such releases produce a burst within the hour (base 9 %). By event: Non-Farm Payrolls (with
-Unemployment Rate / Avg Hourly Earnings) 97 %, 9.3× bar, $11.9 mean 1-h move; CPI 91 %, 8.3×,
-$9.0; FOMC decision/statement 87.5 %, 7.1×, $18.2 (press conference a further $20 hour); FOMC
-projections 92 %; Retail Sales 64 %; PPI 54 %; Core PCE 50 %; Unemployment Claims 44 %; GDP
-45–58 %; ISM PMIs 26 %; Powell speaks 33 %; Trump speaks 21 %; UoM sentiment ≈ 5 %. 20 % (M5)
-/ 31 % (M15) of ALL bursts fall within ±30 min of a high-impact USD release (3.7–4.2 % of time);
-in the 30 min AFTER one, next-bar burst odds are 8–17 % (7–16× base); 15 min before, 2–3 %.
-Open trades that held through a high-impact release did BETTER (M5 +0.41R vs +0.16R; M15 +0.27
-vs +0.07), even when the first 15-min move went against them (M5 +0.09, M15 +0.22). Flattening
-5 min before a release: −5.4R (M5) / −59.3R (M15); 0.5R protective stop through the hour:
-−2.4R / −43.1R. **Events are the biggest single cause of violent bars and still not a reason
-to touch an open trade.**
-
-**Indicator states before a burst (next-bar lift, M5 / M15).** Price OUTSIDE the Bollinger
-bands: below 2.9× / 1.8–2.9×, above 2.1× / 2.2×, middle 0.2–0.8 band 0.7×. RSI < 30: 2.6–2.9× /
-1.8–2.2×; RSI ≥ 70: 1.9–2.1× / 1.6–1.9×; RSI 40–60: 0.7× (calmest). Close outside the Keltner
-channel: 1.4–2.2×; inside 0.76×. ADX RISING > 3: 1.4–1.5× / 1.2×; ADX FALLING < −3: 0.5–0.6×
-(safest state); the ADX level itself carries nothing. MACD histogram negative and still falling
-1.3× / 1.2–1.4×; histogram fading (either sign) 0.8×. Fully bearish 4-EMA stack 1.3–1.4× /
-1.3×; "mostly bull" 0.8× on M5. Regime: transitioning 1.4–1.6× / 1.5–1.6×, ranging 0.88×.
-Volatility regime: extreme 1.5–1.6× / 1.4–1.9×, low 0.6–0.8×. ROC beyond ±0.3 %: 1.2–1.9×
-(>1 %: 1.3–10×, n small). Bollinger-width squeeze: 0.59× on M5 (LESS likely next bar — narrow
-bands are not a warning); TTM squeeze (bands inside Keltner) 1.6× on M15 only. EMA200 distance,
-day of month, first-Friday-as-a-day: nothing stable (first Friday matters only through its
-12:30 slot).
-
-**Time slots (M5 next-bar burst %, base 0.86 %):** Fri 01:xx 3.4 %, Fri 12:xx 3.4 %, Tue/Wed
-01:xx 3.2 %, Thu 12:xx 3.1 %, Fri 13:xx 3.1 %, Sun 22:xx reopen 3.0 %, Thu 01:xx 2.7 %, Wed
-12:xx 2.6 %, Sun 23:xx 2.4 %, Tue 00:xx 2.3 % (all 2.5–3.9× base). Quietest: 03–04 and 16–17.
-
-**Direction of the burst (share going UP):** fully-bull stack 62 % (M5) / 50 % (M15); fully-bear
-stack 26 % / 29 %; RSI < 30 → 22 % / 30 % up (i.e. 70–78 % DOWN); RSI ≥ 70 → 64 % / 52 % up;
-MACD neg & falling → 30 % / 34 % up; trending_down regime → 26 % / 26 % up; trending_up → 60 % /
-46 % up. Bearish states foretell a DOWN burst reliably on both TFs; bullish states foretell an
-UP burst on M5 only.
-
-**Reading:** violent bars arrive when price is already at an extreme with momentum accelerating
-(outside the bands, RSI extreme, ADX rising, MACD still falling, transitioning regime, extreme
-volatility) and/or a release is minutes away — "the storm is already brewing", not "calm before
-the storm". Calm mid-band bars with fading momentum and falling ADX are the safest. None of it
-changes trade handling (protection tests above); it is display material for a warning line.
-
-## Addendum 2026-09-06 — violent-move WATCH rule (display-only feature, `violent_watch.py`)
-Owner asked for a warning. Delivery: a line on signal alerts + a standalone heads-up to the owner
-(once per hour per trigger set, daily cap), dark by default (`violent_watch` config block).
-The first draft rule ("hot" = calendar | ≥3 indicator readings | clock & ≥2 readings) was
-evaluated bar-by-bar on 3 years (`backtest/research/watch_rule_eval.py`, scratch variant
-sweep): it flagged 27 % of bars hot at only ≈2× odds and would have sent ≈13 heads-ups/day on
-M5 — rejected. Sweep of 11 variants (M5 / M15, both halves):
-| hot definition | bars hot | next-bar lift h1/h2 | next-hour rate | hot hours/day |
-|---|---|---|---|---|
-| calendar only (hi-impact USD ±30 min) | 3.4 % | 6.7×/4.4× · 9.1×/6.4× | 31 % · 28 % | 2.6 |
-| clock slot only | 18.8 % | 3.1×/2.0× · 3.6×/2.6× | 16 % · 13 % | 5.3 |
-| ≥2 price-extreme readings | 14.6 % | 2.3×/2.2× · 1.8×/2.0× | 11 % · 6 % | 10.7 |
-| **calendar ∨ (clock ∧ ≥1 price-extreme reading)** | **8.6 %** | **4.9×/3.1× · 5.9×/4.2×** | **21 % · 17 %** | **4.1 · 3.5** |
-| draft (cal ∨ readings≥3 ∨ clock∧readings≥2) | 26.9 % | 2.4×/2.0× · 2.4×/2.1× | 13 % · 8 % | 13.0 |
-**Shipped rule:** price-extreme readings = outside the Bollinger bands, RSI < 30 or ≥ 70, close
-outside the Keltner channel (the neutral readings — ADX rising, extreme volatility, transitioning
-— are dropped from the level logic; they are common and only ≈1.5×). `hot` = high-impact USD
-release within 30 min or 30 min after, OR a hot clock slot with ≥ 1 price-extreme reading; `watch`
-= clock slot alone or ≥ 2 price-extreme readings (≈2× odds). Direction sentence from the bear/bull
-polarity of the readings plus MACD acceleration and trend regime ("DOWN" on both TFs, "UP" on M5
-only). Heads-up only at `hot`. Expected ≈4 hot hours/day → ≤4 heads-ups/day after per-hour dedup.

chart_patterns.py

@@ -97,7 +97,7 @@ def _first_close_beyond(closes: np.ndarray, start: int, level: float,
 
 # ── Double bottom / top ──────────────────────────────────────────────
 
-def _detect_double(h, l, c, atr, cfg, bullish: bool) -> Optional[Dict]:
+def _detect_double(o, h, l, c, atr, cfg, bullish: bool) -> Optional[Dict]:
     w = cfg.cp_swing_window
     ext = l if bullish else h
     idx = _swing_indices(ext, w, find_high=not bullish)
@@ -147,7 +147,7 @@ def _detect_double(h, l, c, atr, cfg, bullish: bool) -> Optional[Dict]:
 
 # ── Head & shoulders (top and inverse) ───────────────────────────────
 
-def _detect_hs(h, l, c, atr, cfg, bullish: bool) -> Optional[Dict]:
+def _detect_hs(o, h, l, c, atr, cfg, bullish: bool) -> Optional[Dict]:
     """bullish=True → inverse H&S (three swing LOWS, head deepest)."""
     w = cfg.cp_swing_window
     ext = l if bullish else h
@@ -210,7 +210,7 @@ def _detect_hs(h, l, c, atr, cfg, bullish: bool) -> Optional[Dict]:
 
 # ── Flags (bull and bear) ────────────────────────────────────────────
 
-def _detect_flag(h, l, c, atr, cfg, bullish: bool) -> Optional[Dict]:
+def _detect_flag(o, h, l, c, atr, cfg, bullish: bool) -> Optional[Dict]:
     n = len(c)
     lo_r, hi_r = cfg.cp_flag_retrace
     if bullish:
@@ -257,12 +257,12 @@ def _detect_flag(h, l, c, atr, cfg, bullish: bool) -> Optional[Dict]:
 # Priority order: most structured / rarest first — if two patterns
 # complete on the same bar, the more specific one names the trade.
 PATTERN_DETECTORS = {
-    "inv_hs":        lambda h, l, c, atr, cfg: _detect_hs(h, l, c, atr, cfg, True),
-    "hs_top":        lambda h, l, c, atr, cfg: _detect_hs(h, l, c, atr, cfg, False),
-    "double_bottom": lambda h, l, c, atr, cfg: _detect_double(h, l, c, atr, cfg, True),
-    "double_top":    lambda h, l, c, atr, cfg: _detect_double(h, l, c, atr, cfg, False),
-    "bull_flag":     lambda h, l, c, atr, cfg: _detect_flag(h, l, c, atr, cfg, True),
-    "bear_flag":     lambda h, l, c, atr, cfg: _detect_flag(h, l, c, atr, cfg, False),
+    "inv_hs":        lambda o, h, l, c, atr, cfg: _detect_hs(o, h, l, c, atr, cfg, True),
+    "hs_top":        lambda o, h, l, c, atr, cfg: _detect_hs(o, h, l, c, atr, cfg, False),
+    "double_bottom": lambda o, h, l, c, atr, cfg: _detect_double(o, h, l, c, atr, cfg, True),
+    "double_top":    lambda o, h, l, c, atr, cfg: _detect_double(o, h, l, c, atr, cfg, False),
+    "bull_flag":     lambda o, h, l, c, atr, cfg: _detect_flag(o, h, l, c, atr, cfg, True),
+    "bear_flag":     lambda o, h, l, c, atr, cfg: _detect_flag(o, h, l, c, atr, cfg, False),
 }
 
 
@@ -276,6 +276,7 @@ def detect_chart_pattern(df: pd.DataFrame, atr: float, cfg) -> Optional[Dict]:
     if atr <= 0 or len(df) < cfg.cp_lookback + 1:
         return None
     tail = df.iloc[-(cfg.cp_lookback + 1):]
+    o = tail["Open"].to_numpy(dtype=np.float64)
     h = tail["High"].to_numpy(dtype=np.float64)
     l = tail["Low"].to_numpy(dtype=np.float64)
     c = tail["Close"].to_numpy(dtype=np.float64)
@@ -283,7 +284,7 @@ def detect_chart_pattern(df: pd.DataFrame, atr: float, cfg) -> Optional[Dict]:
     for name, det in PATTERN_DETECTORS.items():
         if name not in enabled:
             continue
-        hit = det(h, l, c, atr, cfg)
+        hit = det(o, h, l, c, atr, cfg)
         if hit is not None:
             return hit
     return None

config.json.example

@@ -26,8 +26,12 @@
     }
   },
   "oanda": {
+    "account_id": "REPLACE_WITH_ACCOUNT_ID",
     "api_token": "REPLACE_WITH_API_TOKEN",
-    "base_url": "https://api-fxpractice.oanda.com"
+    "base_url": "https://api-fxpractice.oanda.com",
+    "lot_size": 1,
+    "quality_gate": "C",
+    "max_hold_bars": 300
   },
   "deepseek": {
     "api_key": "REPLACE_WITH_YOUR_DEEPSEEK_API_KEY_OR_EMPTY",
@@ -36,39 +40,5 @@
     "timeout_seconds": 12,
     "daily_request_cap": 500
   },
-  "agent_panel": {
-    "enabled": false,
-    "telegram": true,
-    "dashboard": false,
-    "min_quality": "B",
-    "max_panels_per_day": 20,
-    "reuse_ttl_seconds": 3600,
-    "candles_enabled": true,
-    "candle_bars_primary": 20,
-    "candle_bars_confirm": 10,
-    "verdict_source": "macro"
-  },
-  "market_outlook": {
-    "enabled": false,
-    "telegram": false,
-    "max_per_day": 12,
-    "candles_bars": 20
-  },
-  "violent_watch": {
-    "enabled": false,
-    "alert_line": true,
-    "headsup": true,
-    "max_headsup_per_day": 12,
-    "headsup_min_level": "hot"
-  },
-  "signal_log": {
-    "enabled": true,
-    "path": "logs/signals.db",
-    "log_blocked_scans": true
-  },
-  "news_archive": {
-    "enabled": true,
-    "path": "logs/newsdata.db"
-  },
   "log_level": "INFO"
 }

config.py

@@ -31,10 +31,8 @@ class SignalStrength(Enum):
 XAUUSD = "XAU/USD"
 
 # price_precision: decimals for SL/TP rounding and display.
-# default_spread: $0.30 round-trip cost retained for TD-era cross-book
-# comparability and used when the backtester receives no --spread override.
-# Oanda's measured round trip is $0.65-$0.72; research reports headline it
-# separately with --spread 0.65.
+# default_spread: round-trip cost per trade in price units, charged by
+# the backtester when --spread is not given.
 PAIR_INFO = {
     XAUUSD: {"display": "XAU/USD", "pip_value": 0.10,
              "price_precision": 2, "default_spread": 0.30},
@@ -112,10 +110,7 @@ class SignalConfig:
     tp_tightening_factor: float = 1.0
 
     # ── Trade geometry ────────────────────────────────────────────
-    # Deliberately $0.30 for TD-era cross-book comparability. Oanda's measured
-    # round trip is $0.65-$0.72, reported separately in re-pin campaigns.
-    # This is also a LIVE gate input: increasing it to $0.65 raises the TP1
-    # floor from about $9 to $19.50, so it requires rule-1 revalidation.
+    # Round-trip transaction cost estimate in price units (XAU ~$0.30).
     round_trip_cost: float = 0.30
     # Spread gate: reject setups whose TP1 distance is below
     # round_trip_cost × this multiple (caps cost at ~3% of target).
@@ -171,13 +166,6 @@ class NewsConfig:
     # Must stay below pre_event_buffer_minutes so an event can't cross
     # into the danger window while a stale cache says it's clear.
     context_cache_ttl_seconds: int = 600
-    # Maximum age of RSS headlines accepted for scoring. Older headlines
-    # (and headlines with no parsable publish time) are discarded before
-    # keyword analysis and LLM enrichment. 12 h → 4 h on 2026-09-04: the
-    # 3-week signal log showed the 12 h mean was a trailing echo of the
-    # previous day's price move (see backtest/research/
-    # news_sentiment_value_2026-09-04.md). Display/warning-only either way.
-    headline_max_age_seconds: int = 14400  # 4 hours
 
     rss_feeds: List[str] = field(default_factory=lambda: [
         "https://investinglive.com/feed/",
@@ -332,6 +320,7 @@ class PlatformConfig:
 
     symbol: str = XAUUSD
     display_name: str = "XAU/USD"
+    pip_value: float = 0.10
 
     primary_timeframe: Timeframe = Timeframe.H1
     confirmation_timeframes: List[Timeframe] = field(

dashboard_publisher.py

@@ -3,18 +3,17 @@
 XAU/USD static dashboard publisher.
 
 Standalone daemon that scans XAU/USD at candle closes and atomically
-writes `snapshot.json` + `news.json` (plus copies of the static
-`dashboard.html` / `.css` / `.js` and `news_summary.php`) into an
-Apache-served directory. Apache does
+writes `snapshot.json` + `news.json` (plus a copy of
+`static/dashboard.html`) into an Apache-served directory. Apache does
 all web serving — this process never opens a port, so a failure here
 can only ever mean a stale page.
 
 Independence rules (owner requirement — the telegram bot must be
 immune to anything on the web side):
   * Runs as its own process; never imported by, and never imports,
-    `telegram_bot` (telegram_bot resolves credentials at import). The
-    confirmation map and boundary math are therefore duplicated here
-    as local constants.
+    `telegram_bot` / `oanda_daemon` (telegram_bot resolves credentials
+    at import). The confirmation map and boundary math are therefore
+    duplicated here as local constants.
   * The only shared-code dependency beyond the standard signal path is
     the additive `NewsEngine.get_week_events` accessor, which no
     production signal path calls.
@@ -50,14 +49,11 @@ from datetime import datetime, timezone
 from logging.handlers import TimedRotatingFileHandler
 from typing import Dict, List, Optional
 
-from agent_panel import attach_panel
-from app_config import get_app_config, violent_watch_enabled
 from config import PlatformConfig, Timeframe, XAUUSD
 from main import report_to_dict
 from news_engine import NewsEngine
 from pair_signals import produce_signal
 from providers import make_data_provider
-from violent_watch import attach_watch
 import providers.base as _providers_base
 
 logger = logging.getLogger("dashboard")
@@ -85,7 +81,7 @@ SUMMARY_PHP_FILE = "news_summary.php"
 # Repo files copied into the out dir at daemon start (git pull + restart
 # updates them). The PHP endpoint reads news.json in place, so it lives
 # next to the page.
-STATIC_FILES = [PAGE_FILE, "dashboard.css", "dashboard.js", SUMMARY_PHP_FILE]
+STATIC_FILES = [PAGE_FILE, SUMMARY_PHP_FILE]
 
 
 # ══════════════════════════════════════════════════════════════════
@@ -172,8 +168,8 @@ def report_to_webapp_dict(report) -> dict:
 
 
 def event_to_dict(ev) -> dict:
-    """EconomicEvent → dict. surprise is a property, so asdict() would
-    skip it — added explicitly.
+    """EconomicEvent → dict. surprise/minutes_until/is_high_impact are
+    properties, so asdict() would skip them — added explicitly.
     actual/forecast/previous stay strings as published ('228K', '0.3%')."""
     return {
         "time": ev.time.astimezone(timezone.utc).isoformat(),
@@ -184,6 +180,8 @@ def event_to_dict(ev) -> dict:
         "previous": ev.previous,
         "actual": ev.actual,
         "surprise": ev.surprise,
+        "minutes_until": _safe_num(ev.minutes_until),
+        "is_high_impact": ev.is_high_impact,
     }
 
 
@@ -237,8 +235,8 @@ def due_timeframes(wake_dt: datetime) -> List[Timeframe]:
 
 
 def seconds_until_boundary(interval_minutes: int, buffer: int = 10) -> float:
-    # Same math as telegram_bot.py — kept local so the dashboard never
-    # imports bot modules (independence rules above).
+    # Same math as telegram_bot.py / oanda_daemon.py — kept local so the
+    # dashboard never imports bot modules (independence rules above).
     now = datetime.now(timezone.utc)
     total_minutes = now.hour * 60 + now.minute
     elapsed_minutes = total_minutes % interval_minutes
@@ -323,7 +321,6 @@ class DashboardPublisher:
     def __init__(self, out_dir: str):
         self.out_dir = out_dir
         self.snapshots: Dict[str, dict] = {}
-        self._reports = {}
         self.news_payload: Optional[dict] = None
         self._dp = None
         self._news_engine: Optional[NewsEngine] = None
@@ -360,12 +357,6 @@ class DashboardPublisher:
         try:
             report = produce_signal(XAUUSD, tf, self._provider(),
                                     CONFIRMATIONS[tf])
-            # Display-only violent-move watch (dark by default). Never
-            # raises and never touches the signal — only watch_line.
-            if violent_watch_enabled():
-                attach_watch(report, tf,
-                             get_app_config().violent_watch_settings)
-            self._reports[tf.value] = report
             self.snapshots[tf.value] = {
                 "status": "ok",
                 "scanned_at": datetime.now(timezone.utc).isoformat(),
@@ -377,41 +368,16 @@ class DashboardPublisher:
             logger.info(f"{tf.value}: {report.final_signal.name} "
                         f"q={report.signal_quality or '-'} "
                         f"({time.time() - started:.1f}s)")
-        except Exception:
+        except Exception as e:
             # Keep the last good report; the page shows it with an
             # error badge instead of going blank.
             logger.exception(f"{tf.value} scan failed")
-            self._reports.pop(tf.value, None)
             entry = self.snapshots.get(tf.value) or _empty_tf_entry()
             entry["status"] = "error"
-            # The public snapshot carries a stable message; exception
-            # detail remains in dashboard.log for the operator.
-            entry["error"] = "Scan unavailable"
+            entry["error"] = f"{type(e).__name__}: {e}"
             entry["error_at"] = datetime.now(timezone.utc).isoformat()
             self.snapshots[tf.value] = entry
 
-    def attach_panels(self, tfs: List[Timeframe]) -> bool:
-        """Add advisory panels after publishing the technical snapshot.
-
-        A panel is display-only and can take tens of seconds on a cold LLM
-        request. Keeping its report object locally lets the page receive the
-        candle-close signal first, then a second atomic snapshot only when a
-        panel produced a result.
-        """
-        updated = False
-        for tf in tfs:
-            report = self._reports.get(tf.value)
-            if report is None:
-                continue
-            if not attach_panel(report, surface="dashboard"):
-                continue
-            entry = self.snapshots.get(tf.value)
-            if entry is None or entry.get("status") != "ok":
-                continue
-            entry["report"] = report_to_webapp_dict(report)
-            updated = True
-        return updated
-
     def refresh_news(self) -> None:
         try:
             ne = self._news()
@@ -431,14 +397,18 @@ class DashboardPublisher:
         if self.news_payload is not None:
             publish(self.news_payload, os.path.join(self.out_dir, NEWS_FILE))
 
+    def publish_all(self) -> None:
+        self.publish_snapshot()
+        self.publish_news()
+
     def run_cycle(self, tfs: List[Timeframe]) -> None:
         for tf in tfs:  # sequential — see module docstring (cache dedup)
             self.scan_tf(tf)
-        # Technical signals land before any optional slow work. News and the
-        # advisory panel are display-only and may take 20-30 seconds cold.
+        # Snapshot lands BEFORE the news refresh: when the news TTL has
+        # expired, refresh_news does FF + 4 RSS fetches (and possibly
+        # DeepSeek calls) — 20-30 s that must not keep fresh signals off
+        # the page.
         self.publish_snapshot()
-        if self.attach_panels(tfs):
-            self.publish_snapshot()
         self.refresh_news()
         self.publish_news()
 

deploy/README-dashboard.md

@@ -24,9 +24,8 @@ No new Python dependencies are required.
 ## 2. Make the web folder writable
 
 The dashboard publishes straight into the portal docroot `/var/www/trade/`
-(owner decision 2026-08-03) — it adds exactly six files there:
-`dashboard.html`, `dashboard.css`, `dashboard.js`, `news_summary.php`,
-`snapshot.json`, `news.json` (plus the
+(owner decision 2026-08-03) — it adds exactly four files there:
+`dashboard.html`, `news_summary.php`, `snapshot.json`, `news.json` (plus the
 apache-owned `ai/` cache dir from §6). The daemon user must be able to write
 them:
 
@@ -52,7 +51,7 @@ sudo restorecon -R /var/www/trade
 
 ```bash
 venv/bin/python dashboard_publisher.py --once
-ls /var/www/trade    # dashboard.html/.css/.js news_summary.php snapshot.json news.json added next to the portal
+ls /var/www/trade    # dashboard.html snapshot.json news.json added next to the portal
 ```
 
 Then open `http://<server>/dashboard.html` (path relative to however the
@@ -72,8 +71,7 @@ tail -f logs/dashboard.log
 
 The daemon scans all four timeframes at boot, then wakes at every M5
 boundary (+10 s) and rescans whichever timeframes just closed. Logs rotate
-every 3 h (240 backups); the bot's own `telegram.log` rotates daily (30
-backups) since 2026-09-08.
+every 3 h (240 backups), same scheme as the bot.
 
 ## 5. Optional Apache polish
 
@@ -102,11 +100,6 @@ the public page, accepted with these guards baked into the PHP — the id must
 match a headline in the published news.json (arbitrary text can never reach
 the LLM), each headline's summary is cached forever in `ai/` (≤1 API call per
 headline, ever), and a file-based daily cap fails **closed** on I/O errors.
-Since 2026-09-04 the engine publishes only headlines up to 4 h old whose
-score leans one way (neutral items are dropped), so a headline leaves the
-whitelist sooner: a dialog left open past that point returns `unknown_id`
-("This headline has left the live feed"); summaries already cached in `ai/`
-stay reachable.
 
 Requirements: `php` + the curl extension (`php -m | grep curl`).
 
@@ -143,8 +136,8 @@ curl -s "http://<server>/news_summary.php?id=$ID"   # instant, from ai/<id>.json
 ## Operational notes
 
 - **Updating**: `git pull` + `systemctl restart gold-dashboard`. The daemon
-  re-copies `static/dashboard.html`, `dashboard.css`, `dashboard.js` and
-  `news_summary.php` into the web folder at startup.
+  re-copies `static/dashboard.html` and `static/news_summary.php` into the
+  web folder at startup.
 - **If the daemon dies**, systemd restarts it in 30 s; worst case the page
   shows stale data with its own "updated N min ago" evidence. Apache and the
   telegram bot are unaffected.
@@ -157,5 +150,8 @@ curl -s "http://<server>/news_summary.php?id=$ID"   # instant, from ai/<id>.json
   DeepSeek-side spend alerting is the true backstop. Kept separate on
   purpose (independence rule: the PHP never imports/reads bot code or
   config.json).
+- **Restart window**: for the first minutes after a daemon restart the new
+  page can meet a pre-upgrade news.json without headline ids — those rows
+  simply render non-clickable until the first news publish.
 - **Credentials**: uses the same env/.env/config.json resolution as the rest
   of the platform (Oanda candles feed; news needs no key; DeepSeek optional).

docs/architecture_diagram.py

@@ -0,0 +1,199 @@
+"""ForexBot2 architecture diagram → docs/architecture.png.
+
+Regenerate after architecture changes:  venv/bin/python docs/architecture_diagram.py
+(matplotlib only — no graphviz on the dev box)."""
+import os
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+from matplotlib.patches import FancyBboxPatch, FancyArrowPatch
+
+W, H = 100.0, 100.0
+fig, ax = plt.subplots(figsize=(16, 11.5))
+ax.set_xlim(0, W); ax.set_ylim(0, H); ax.axis("off")
+
+INK = "#263238"; MUT = "#607D8B"; ARR = "#455A64"; HAUL = "#7B8FA1"
+
+def box(x, y, w, h, title, lines, fc, ec, title_fs=8.2, body_fs=6.6, lw=1.3):
+    ax.add_patch(FancyBboxPatch((x, y), w, h,
+                 boxstyle="round,pad=0.35,rounding_size=0.9",
+                 facecolor=fc, edgecolor=ec, linewidth=lw, zorder=2))
+    cy = y + h - 1.9
+    ax.text(x + w / 2, cy, title, ha="center", va="center", fontsize=title_fs,
+            fontweight="bold", color=INK, zorder=3)
+    cy -= 2.3
+    for ln in lines:
+        ax.text(x + w / 2, cy, ln, ha="center", va="center", fontsize=body_fs,
+                color=INK, zorder=3)
+        cy -= 1.85
+
+def band(x, y, w, h, label, color):
+    ax.add_patch(FancyBboxPatch((x, y), w, h,
+                 boxstyle="round,pad=0.3,rounding_size=1.2",
+                 facecolor="none", edgecolor=color, linewidth=1.0,
+                 linestyle=(0, (4, 3)), zorder=1))
+    ax.text(x + 0.9, y + h - 1.4, label, ha="left", va="center",
+            fontsize=7.2, fontweight="bold", color=color, zorder=3)
+
+def arrow(p1, p2, color=ARR, lw=1.5, style="-|>", conn="arc3,rad=0.0",
+          label=None, lx=0, ly=0, ls="solid", fs=6.4, rot=0):
+    ax.add_patch(FancyArrowPatch(p1, p2, arrowstyle=style, mutation_scale=11,
+                 color=color, linewidth=lw, linestyle=ls,
+                 connectionstyle=conn, zorder=4))
+    if label:
+        mx, my = (p1[0] + p2[0]) / 2 + lx, (p1[1] + p2[1]) / 2 + ly
+        ax.text(mx, my, label, fontsize=fs, color=MUT, ha="center",
+                va="center", zorder=5, rotation=rot,
+                bbox=dict(boxstyle="round,pad=0.15", fc="white", ec="none", alpha=0.9))
+
+# ── title ────────────────────────────────────────────────────────────
+ax.text(2, 98.4, "ForexBot2 — XAU/USD Signal-Intelligence Platform",
+        fontsize=15, fontweight="bold", color=INK, ha="left")
+ax.text(2, 96.0, "one pipeline, four delivery surfaces · closed bars only · live == backtest parity",
+        fontsize=8.5, color=MUT, ha="left")
+
+EXT_FC, EXT_EC = "#ECEFF1", "#90A4AE"
+DAT_FC, DAT_EC = "#E3F2FD", "#5C9BD1"
+NWS_FC, NWS_EC = "#E0F2F1", "#3D9C8F"
+COR_FC, COR_EC = "#FFFDF5", "#C9A227"
+REP_FC, REP_EC = "#FFE9A8", "#B8860B"
+DEL_FC, DEL_EC = "#E8F5E9", "#5FAF63"
+BTS_FC, BTS_EC = "#EDE7F6", "#8E6BC6"
+
+# ── external services (top band) ─────────────────────────────────────
+band(1, 85.0, 98, 9.4, "EXTERNAL SERVICES", "#78909C")
+box(2.5, 86.2, 15.5, 6.4, "Oanda v20 API",
+    ["candles (live feed)", "practice orders"], EXT_FC, EXT_EC)
+box(19.5, 86.2, 15.5, 6.4, "Twelve Data · Tiingo",
+    ["registered fallbacks", "(Tiingo: tick stream)"], EXT_FC, EXT_EC)
+box(36.5, 86.2, 15.5, 6.4, "Forex Factory",
+    ["economic calendar", "MyFXBook RSS fallback"], EXT_FC, EXT_EC)
+box(53.5, 86.2, 12.5, 6.4, "News RSS ×4",
+    ["gold / macro", "headlines"], EXT_FC, EXT_EC)
+box(67.5, 86.2, 13.5, 6.4, "DeepSeek API",
+    ["V4-Flash LLM", "thinking: disabled"], EXT_FC, EXT_EC)
+box(82.5, 86.2, 15, 6.4, "Telegram API",
+    ["owner bot +", "subscriber channel"], EXT_FC, EXT_EC)
+
+# ── data & context layer ─────────────────────────────────────────────
+box(2.5, 69.8, 33, 11.0, "providers/  (BaseDataProvider)",
+    ["single-flight fetch · TTL+boundary cache",
+     "forming-candle DROP · validate/clean",
+     "paginated fetch_range · optional tick stream",
+     "active feed knob: data_providers.active = oanda"],
+    DAT_FC, DAT_EC)
+box(38.5, 69.8, 33, 11.0, "news_engine.py  +  llm_client.py",
+    ["calendar events + keyword sentiment (gold-inverse)",
+     "DeepSeek enrichment: batch headline scores,",
+     "event gold-direction, market blurb — display-only,",
+     "keyword fallback, daily request cap"],
+    NWS_FC, NWS_EC)
+box(74.5, 69.8, 23, 11.0, "app_config.py",
+    ["credentials & deployment:",
+     "env → config.json → refuse",
+     "(no hardcoded fallbacks)",
+     ".env loaded at import"],
+    "#FAFAFA", "#9E9E9E")
+
+# ── signal core container ────────────────────────────────────────────
+ax.add_patch(FancyBboxPatch((2.5, 40.5), 95, 26.5,
+             boxstyle="round,pad=0.35,rounding_size=1.2",
+             facecolor=COR_FC, edgecolor=COR_EC, linewidth=1.6, zorder=1))
+ax.text(50, 65.2, "SIGNAL CORE — runs identically in live scans and backtests (closed bars only)",
+        ha="center", fontsize=8.4, fontweight="bold", color="#8a6d1a", zorder=3)
+
+box(4.5, 51.8, 28, 12.0, "engine.py — TechnicalEngine",
+    ["9 scored indicators + ATR context:",
+     "EMA · RSI · MACD · BB · ADX · ROC ·",
+     "Keltner · divergence · candlestick",
+     "regime → adaptive weights → composite"],
+    "white", COR_EC)
+box(4.5, 42.0, 28, 8.4, "pair_signals/xauusd.py — factory",
+    ["ALWAYS used: config.json overlay +",
+     "validated per-TF deltas (hour blocks, trail…)"],
+    "white", COR_EC)
+box(35.5, 42.0, 60, 20.7, "signal_aggregator.py — analyze_pair → analyze_from_dataframes",
+    ["multi-TF agreement  (M5→[M15,H1] · M15→[M5,H1] · H1→[M15,H4] · H4→[H1,D1])",
+     "→ news context (warn-only by default) → diminishing-decay penalties",
+     "→ exhaustion hard block → quality grade A+…F",
+     "→ ordered post-grade gate chain (session · XAU SELL vol/hour · regime · M5 noise)",
+     "→ setup generators: XAU ATR (A+/A) · generic swing · XAU_PB · XAU_CP",
+     "→ entry · SL · TP1/TP2 · trailing plan"],
+    "white", COR_EC, body_fs=6.9)
+
+# ── SignalReport hub ─────────────────────────────────────────────────
+box(31, 33.2, 38, 5.2, "SignalReport",
+    ["signal (−2…+2) · grade · trade setup · reasons · news summary"],
+    REP_FC, REP_EC, title_fs=8.8)
+
+# ── delivery surfaces ────────────────────────────────────────────────
+band(1, 13.2, 98, 17.2, "DELIVERY SURFACES", "#4C8F50")
+box(2.5, 15.0, 12, 12.6, "main.py",
+    ["CLI report", "--detail", "--json"], DEL_FC, DEL_EC)
+box(16, 15.0, 24, 12.6, "oanda_daemon.py + executor",
+    ["practice auto-execution",
+     "book-parity rules: no broker TP,",
+     "5-bar cooldown, once-per-bar",
+     "trail, OPEN_ONLY fills,",
+     "boot reconcile · orders → Oanda"],
+    DEL_FC, DEL_EC, title_fs=7.8)
+box(42.5, 15.0, 31, 12.6, "dashboard_publisher.py  →  Apache",
+    ["independent daemon · scans at candle closes",
+     "atomic publish to /var/www/trade:",
+     "dashboard.html · snapshot.json · news.json",
+     "+ news_summary.php — on-demand AI",
+     "gold-impact summary (whitelist · cache · cap)"],
+    DEL_FC, DEL_EC)
+box(76, 15.0, 21.5, 12.6, "telegram_bot.py",
+    ["candle-boundary auto-scans", "quality threshold + dedup",
+     "alerts + LLM context line", "WS level-breach monitor"],
+    DEL_FC, DEL_EC)
+
+# ── backtest / QA ────────────────────────────────────────────────────
+box(2.5, 3.2, 71.8, 8.0, "backtest/gpu_backtest.py — bar-by-bar replay through the SAME analyze_from_dataframes",
+    ["realism (results-critical): next-open entries · raw-path resolution · spread charged · SL-first ties · close-only trail",
+     "CuPy GPU / NumPy fallback · parquet cache · fed by providers.fetch_range (news off)"],
+    BTS_FC, BTS_EC)
+box(77.5, 3.2, 20, 8.0, "test_all.py",
+    ["359 tests · 22 modules", "no network (mocked)"], "#FAFAFA", "#9E9E9E")
+
+# ── arrows: short structural hops (solid dark) ───────────────────────
+arrow((10, 86.2), (14, 81.2), label="OHLCV", lx=-3.4, ly=0.2)          # oanda → providers
+arrow((27, 86.2), (24, 81.2), ls=(0, (3, 2)), lw=1.1)                  # fallbacks → providers
+arrow((44, 86.2), (48, 81.2))                                          # FF → news
+arrow((59.5, 86.2), (56, 81.2))                                        # RSS → news
+arrow((72, 86.2), (62, 81.2), label="LLM", lx=3.4, ly=0.6)             # deepseek → news
+arrow((19, 69.8), (19, 64.2), label="DataFrames (UTC, closed bars)", lx=0.2, ly=0.9)
+arrow((33.5, 57.5), (35.5, 55.5))                                      # engine → aggregator
+arrow((32.5, 46.0), (35.5, 48.0))                                      # factory → aggregator
+arrow((55, 69.8), (55, 63.1), label="warn-only context", lx=0, ly=1.0)
+arrow((86, 69.8), (86, 67.3), ls=(0, (3, 2)), lw=1.1)                  # config → core
+arrow((50, 40.5), (50, 38.8))                                          # core → report
+arrow((36, 34.5), (8.5, 27.95), conn="arc3,rad=0.12")                  # report → cli
+arrow((40, 33.2), (28, 27.95), conn="arc3,rad=0.06")                   # report → daemon
+arrow((60, 33.2), (58, 27.95))                                         # report → dashboard
+arrow((66, 34.5), (86.5, 27.95), conn="arc3,rad=-0.12")                # report → telegram
+
+# ── long-haul edges (thin dashed, routed through gutters) ────────────
+# telegram bot ↔ Telegram API — right-margin arc, clear of app_config
+arrow((95, 86.2), (95, 27.95), conn="arc3,rad=0.15", color=HAUL,
+      lw=1.2, ls=(0, (4, 2)))
+ax.text(99.2, 57, "alerts / commands", fontsize=6.4, color=MUT,
+        rotation=90, ha="center", va="center")
+# browser click → news_summary.php → DeepSeek
+# (oanda_daemon's order flow is noted inside its box — no long arc)
+arrow((66, 27.95), (73, 86.2), conn="arc3,rad=-0.18", color=HAUL,
+      lw=1.2, ls=(0, (4, 2)),
+      label="on-demand AI summary\n(browser click → PHP)", lx=2.5, ly=11.6)
+# providers.fetch_range → backtester — far-left margin channel
+arrow((1.5, 69.8), (1.5, 11.0), color=BTS_EC, lw=1.1, ls=(0, (3, 2)),
+      label="fetch_range\n(history)", lx=3.6, ly=-1.2)
+# backtester replays the signal core — right gutter between boxes
+arrow((74.0, 11.6), (74.9, 40.3), color=HAUL, lw=1.3)
+ax.text(81.8, 38.9, "replays the exact signal code", fontsize=6.4,
+        color=MUT, ha="center", va="center", zorder=5,
+        bbox=dict(boxstyle="round,pad=0.15", fc="white", ec="none", alpha=0.9))
+
+out = os.path.join(os.path.dirname(os.path.abspath(__file__)), "architecture.png")
+fig.savefig(out, dpi=200, bbox_inches="tight", facecolor="white")
+print(f"saved {out}")

docs/beginner_guide.html

@@ -86,8 +86,6 @@
 <li><b>Grade (A+, A, B, C):</b> the bot's own mark for how clean the setup is. A+ is best. It never sends anything worse than C.</li>
 <li><b>Confidence and the indicator list:</b> extra detail for curious people. You don't need to understand it to trade the signal.</li>
 <li><b>⚠️ Warning lines:</b> things the bot noticed, like high volatility. They are "be aware," not "don't trade."</li>
-<li><b>🌪 Violent-move watch:</b> now and then the bot adds a line saying the next hour could produce very big candles — usually because an important American economic number is about to come out, or the market is in an hour that is historically wild, or the price has stretched a long way from normal. It is only a "hold on tight" note. It does not change the signal, and it is not a reason to close or move your stop loss early — the bot was tested doing exactly that and it lost money. Your stop loss is already your protection; leave it where it is.</li>
-<li><b>📰 News line:</b> a short plain-English note on what is moving the gold price right now and why, written from the last few hours of headlines. It is background information, not a reason to trade.</li>
 <li><b>The trade setup at the bottom is the only part you must act on.</b></li>
 </ul>
 
@@ -136,8 +134,6 @@ The message says: <i>"once price hits 4,159, trail SL $33 behind best price."</i
 <ul>
 <li><b>H1 is forgiving.</b> You have up to an hour to act on a message, and small delays barely matter. On the 5-minute signals, being two minutes late already costs real money — they are not for manual beginners.</li>
 <li><b>H1 trades are held longer</b> — often overnight, sometimes over days. That is normal. Your stop loss is working for you while you sleep.</li>
-<li><b>H1 is BUY-only.</b> All H1 SELL signals are blocked (since 2026-09-03) because they were a losing side on this timeframe — expect to see BUY signals here, never SELL.</li>
-<li><b>M5 also holds back some SELL signals</b> — specifically while the market is in a choppy "transitioning" phase, since those shorts tested poorly.</li>
 <li><b>Never take the same signal on two timeframes.</b> They are usually the same gold move seen at different zoom levels. Taking both doubles your risk, not your profit.</li>
 </ul>
 

docs/superpowers/specs/2026-07-21-centralized-config-design.md

@@ -0,0 +1,154 @@
+# Centralized Configuration (config.json) — Design
+
+**Date:** 2026-07-21
+**Status:** Approved (user confirmed all four scoping decisions)
+
+## Goal
+
+Move deployment settings — Telegram credentials, data-provider API keys and
+selection, scanned pairs, scan timeframes, per-pair base blocked hours, alert
+quality boot default, log level — into a single gitignored `config.json` at the
+repo root, loaded once at startup.
+
+**Hard constraint:** zero change to signal logic, indicator weights, thresholds,
+or gates. With an unedited `config.json` (or none at all), every produced
+`PlatformConfig` must be byte-identical to today's factory output. This is
+test-enforced (see Testing).
+
+## Scoping decisions (user-confirmed 2026-07-21)
+
+1. **Secrets live in config.json** (gitignored). A committed
+   `config.json.example` carries placeholders. Env vars still override.
+   The hardcoded credential fallbacks in `config.py` and `telegram_bot.py`
+   are **deleted** — on a machine with neither config.json nor env vars, the
+   bot refuses to start with an explicit error instead of silently using
+   committed credentials. (Recommend rotating the Telegram token and Twelve
+   Data key after merge; both are exposed in git history.)
+2. **Multi-provider is structure only**: a `data_providers` section with named
+   provider blocks and an `active` selector. Only Twelve Data is implemented.
+   No failover logic (nothing to fail over to yet).
+3. **Blocked hours: base list per pair only.** The validated per-timeframe
+   deltas (M5 appends hour 9; M15 SELL-block 15/17 UTC) stay in
+   `pair_signals/xauusd.py` next to their validation evidence. The overlay is
+   applied **before** those deltas and updates **both** mirrored copies
+   (`xau_strategy.blocked_hours_utc` and `backtest.blocked_hours_utc`) so they
+   cannot drift.
+4. **Extra keys included:** scan timeframes, alert min quality (boot default),
+   log level. Everything else (indicator params, RR geometry, M5 gates,
+   CP knobs, news config) deliberately stays code-only.
+
+## File schema
+
+`config.json` (gitignored; `config.json.example` committed with placeholders
+for secrets and today's tuned values for everything else):
+
+```json
+{
+  "telegram": {
+    "token": "<bot token>",
+    "chat_id": "<owner chat id>",
+    "channel_id": "<-100… or empty string to disable the channel feed>",
+    "alert_min_quality": "C"
+  },
+  "data_providers": {
+    "active": "twelve_data",
+    "twelve_data": { "api_key": "<key>" }
+  },
+  "scanning": {
+    "pairs": ["XAU/USD"],
+    "timeframes": ["M5", "M15", "H1", "H4"]
+  },
+  "pairs": {
+    "XAU/USD": { "blocked_hours_utc": [0, 1, 2, 10, 11, 13, 16, 21] }
+  },
+  "log_level": "INFO"
+}
+```
+
+Timeframe strings use the `Timeframe` enum names (`M5`, `M15`, `H1`, `H4`, …).
+Pair keys use canonical symbols (`XAU/USD`).
+
+## Architecture
+
+### New module: `app_config.py`
+
+Single responsibility: read + validate `config.json`, expose typed accessors,
+apply the pair-level overlay. No signal knowledge.
+
+- `AppConfig` dataclass: `telegram` (token, chat_id, channel_id,
+  alert_min_quality), `data_providers` (active name + per-provider dicts),
+  `scan_pairs`, `scan_timeframes`, `pair_overrides`
+  (symbol → {blocked_hours_utc}), `log_level`.
+- `load_app_config(path=REPO_ROOT/"config.json") -> AppConfig` — module-level
+  cached (loaded once per process). Missing file → `AppConfig` of pure
+  defaults + one warning log. Malformed JSON, unknown top-level/section keys,
+  unknown pair symbols, hours outside 0–23, unknown timeframe names, unknown
+  `active` provider → raise `AppConfigError` with a named message (fail fast
+  at boot).
+- Precedence, applied inside the accessors:
+  **env var → config.json → code default**. Secrets (`TELEGRAM_TOKEN`,
+  `TELEGRAM_CHAT_ID`, `TELEGRAM_CHANNEL_ID`, `TWELVE_DATA_API_KEY`) have **no
+  code default** after this change.
+- `apply_pair_overrides(cfg: PlatformConfig, app: AppConfig) -> PlatformConfig`
+  — sets the pair's base `blocked_hours_utc` on **both**
+  `cfg.xau_strategy.blocked_hours_utc` and `cfg.backtest.blocked_hours_utc`,
+  plus `cfg.scan_pairs` / `cfg.scan_timeframes`. Called by the pair factory
+  (see below). A no-op when the pair has no override block.
+
+### Wiring points (all consumers)
+
+| Consumer | Change |
+|---|---|
+| `pair_signals/xauusd.py` | `make_config` calls `apply_pair_overrides` **before** the per-TF deltas (so M5 still appends hour 9 to whatever base the JSON sets, and mirrors stay in sync). |
+| `data_provider.py` | API key resolution becomes `app_config` accessor for the **active** provider (env `TWELVE_DATA_API_KEY` still wins). `config.twelve_data_api_key` field and its hardcoded default are deleted. Missing key keeps today's behavior: provider constructed with `_td=None`, errors on first fetch. |
+| `telegram_bot.py` | `TOKEN`, `CHAT_ID`, `CHANNEL_ID` come from `app_config` (hardcoded fallbacks deleted; missing token/chat_id → explicit startup error). `_alert_min_quality` boot default and `logging` level come from `app_config`. `SCAN_PAIRS` (today built from a bare `PlatformConfig()`) reads `app_config.scan_pairs` directly; `apply_pair_overrides` also stamps `cfg.scan_pairs`/`cfg.scan_timeframes` so the dataclass field stays truthful for any other reader. |
+| `main.py` | No structural change — it uses the factory, which now applies the overlay. |
+| `backtest/gpu_backtest.py` | No structural change — it uses the factory too, so an edited base blocked-hours list flows into replays identically to live (parity by construction). Data fetches pick up the provider key via `data_provider.py`. |
+| `config.py` | `twelve_data_api_key` field removed. Everything else untouched. |
+
+### Multi-provider structure
+
+`data_provider.py` gains a minimal registry:
+`PROVIDER_REGISTRY = {"twelve_data": ForexDataProvider}`. `active` selects
+from it; an unknown name fails at load. Adding a provider later = one config
+block + one adapter class registered here. No shared base class extracted now
+(YAGNI — with one provider there is nothing to abstract over yet).
+
+## Error handling summary
+
+| Condition | Behavior |
+|---|---|
+| No config.json | Defaults + warning. Secrets must come from env or startup fails (bot) / fetch fails (provider). |
+| Malformed JSON / bad values / unknown keys | `AppConfigError` at load, process exits with named message. |
+| Missing Telegram token or chat_id at bot boot | Explicit startup error naming the missing key and both places it can be set. |
+| Missing data-provider key | Same as today: provider builds, first fetch errors. |
+
+## Migration
+
+- Create the local `config.json` populated with the **current live values**
+  (the ones being deleted from source) so the running bot keeps working
+  unchanged. `.gitignore` gains `config.json`.
+- Commit `config.json.example` with placeholders.
+- CLAUDE.md: update the env-var table, repository map, and config section.
+- Post-merge recommendation (user action): rotate the Telegram token and
+  Twelve Data key, since both are in git history.
+
+## Testing (`test_all.py`, new module)
+
+1. **Parity pin (the critical one):** a config.json containing the example
+   defaults, and no config.json at all, both produce factory output identical
+   to today's (full dataclass equality across all four TFs).
+2. Overlay mapping: JSON blocked hours land on **both** mirrored fields; M5
+   factory still appends hour 9 on top of a custom base.
+3. Precedence: env var beats config.json beats default, per key.
+4. Validation: malformed JSON, bad hour, unknown pair/timeframe/provider each
+   raise `AppConfigError`.
+5. Missing-file behavior: defaults + no crash.
+6. Existing factory-equivalence test stays green (equivalence is now defined
+   over factory-with-overlay vs inline-config-with-overlay).
+
+## Out of scope
+
+Hot reload (config is read once at boot), a second concrete provider,
+failover, moving any tuned strategy value beyond the base blocked-hours list,
+news/indicator/geometry settings in JSON.

docs/user_guide.html

@@ -69,8 +69,7 @@
 <tr><td><b>Score / confidence</b></td><td>The regime-weighted composite of all nine indicators. Bigger absolute score = stronger consensus.</td></tr>
 <tr><td><b>MTF aligned</b></td><td>The two confirmation timeframes agree with the trade direction. Signals without alignment are filtered out on M5.</td></tr>
 <tr><td><b>Warnings ⚠️</b></td><td>Context, not veto. Backtesting shows warned trades perform about the same — strong momentum often triggers warnings.</td></tr>
-<tr><td><b>Violent-move watch 🌪</b></td><td>An occasional extra line saying the next hour looks unusually likely to produce a very large candle — because a high-impact US release is due (or just landed), because the clock is in a historically wild slot, or because price is stretched outside its bands. It is a heads-up about <i>volatility</i>, not a direction call, and it <b>never</b> changes the signal, the grade or the trade setup — the same alert would be sent without it. Do not close or tighten an open trade because of it: every such protective action was tested over three years and lost money. Your stop is the protection.</td></tr>
-<tr><td><b>Market context 📰</b></td><td>An AI-written note in plain English on what is moving the gold price right now, why it matters, and the next report to watch. It is built only from headlines published in the last 4 hours that lean one way (neutral items are dropped) plus today's high-impact releases. Informational only — it never blocks or grades a signal.</td></tr>
+<tr><td><b>Market context 📰</b></td><td>An AI-written one-liner summarizing today's macro backdrop (headlines + high-impact releases) and its likely pull on gold. Informational only — it never blocks or grades a signal.</td></tr>
 <tr><td><b>SL / TP1 / RR</b></td><td>Stop loss sits behind the nearest swing level with an ATR buffer. TP1 is the nearest opposing level. <b>1R</b> = the entry-to-SL distance = the amount you risk.</td></tr>
 <tr><td><b>Trail plan 🔄</b></td><td>The real exit. Once price moves +0.6R in your favor, move the stop to trail 0.75R behind the best price and update it at each candle close. <b>TP1 is advisory — let winners run.</b></td></tr>
 </table>
@@ -85,27 +84,27 @@
 </ol>
 
 <h2>4 · Choosing your timeframe</h2>
-<p>Twelve-month Oanda-candle backtest (Aug 2025 – Aug 2026, $0.65 round-trip cost, entry at next candle open):</p>
+<p>Twelve-month backtest (Jul 2025 – Jul 2026, real spreads, entry at next candle open):</p>
 <table>
 <tr><th></th><th>M5</th><th>M15</th><th>H1</th></tr>
-<tr><td>Trades / week</td><td>~11.3</td><td>~7.6</td><td>~2.7</td></tr>
-<tr><td>Total result (12 mo)</td><td>+106.8R</td><td>+73.9R</td><td>+24.0R</td></tr>
-<tr><td>Win rate / profit factor</td><td>52.9% / 1.51</td><td>62.2% / 1.53</td><td>60.9% / 1.55</td></tr>
-<tr><td>Typical (median) hold</td><td>2.4 h</td><td>7 h</td><td>15.5 h</td></tr>
-<tr><td>90% of trades close within</td><td>10.5 h</td><td>29.8 h</td><td>3.2 days</td></tr>
-<tr><td>Typical stop distance</td><td>~$21</td><td>~$32</td><td>~$52</td></tr>
-<tr><td>Worst drawdown</td><td>10.7R</td><td>10.8R</td><td>8.0R</td></tr>
+<tr><td>Trades / week</td><td>~10</td><td>~9.5</td><td>~3</td></tr>
+<tr><td>Total result (12 mo)</td><td>+135.6R</td><td>+67.2R</td><td>+49.9R</td></tr>
+<tr><td>Win rate / profit factor</td><td>59.7% / 1.81</td><td>55.8% / 1.40</td><td>59.4% / 1.94</td></tr>
+<tr><td>Typical (median) hold</td><td>2.4 h</td><td>6 h</td><td>12.5 h</td></tr>
+<tr><td>90% of trades close within</td><td>16 h</td><td>40 h</td><td>3.7 days</td></tr>
+<tr><td>Typical stop distance</td><td>~$19</td><td>~$29</td><td>~$44</td></tr>
+<tr><td>Worst drawdown</td><td>11.2R</td><td>9.4R</td><td>9.5R</td></tr>
 </table>
 <ul>
-<li><b>M5 — intraday, execution-critical.</b> Highest total edge, but stops are tight: a couple of dollars of slippage or a late entry eats a meaningful share of every trade. Suitable only if you can act within a minute or two of the alert (or automate). Trades open and close within a session; you sleep flat. SELL signals are held back while the market regime reads "transitioning" — a validated filter, not a bug if you notice fewer shorts there.</li>
+<li><b>M5 — intraday, execution-critical.</b> Highest total edge, but stops are tight: a couple of dollars of slippage or a late entry eats a meaningful share of every trade. Suitable only if you can act within a minute or two of the alert (or automate). Trades open and close within a session; you sleep flat.</li>
 <li><b>M15 — the middle path.</b> Same frequency as M5, wider stops, up to 15 minutes to act. About half your trades will still be open overnight.</li>
-<li><b>H1 — swing trading around a normal life.</b> Best quality per trade and very forgiving of slow manual execution (you have up to an hour). The costs: positions held for days, through news and overnight sessions, and a <b>lumpy</b> equity curve — one quarter of the test year was negative and most of the profit came in a single quarter. Judge it on quarters, not weeks. H1 no longer issues SELL signals at all (blocked since 2026-09-03) — it is BUY-only on this timeframe.</li>
+<li><b>H1 — swing trading around a normal life.</b> Best quality per trade and very forgiving of slow manual execution (you have up to an hour). The costs: positions held for days, through news and overnight sessions, and a <b>lumpy</b> equity curve — one quarter of the test year was negative and most of the profit came in a single quarter. Judge it on quarters, not weeks.</li>
 </ul>
 <div class="callout"><b>If you trade manually from the Telegram alerts, H1 (or M15) is the honest choice.</b> M5's backtested edge assumes near-instant fills that a human copying signals rarely achieves.</div>
 
 <h2>5 · What to expect (and what not to)</h2>
 <ul>
-<li><b>You will lose roughly 38–47% of trades.</b> A 53–62% win rate means losing streaks of 5+ are routine over a year. The system's edge is the average across hundreds of trades, not any single one.</li>
+<li><b>You will lose ~40% of trades.</b> A 56–60% win rate means losing streaks of 5+ are routine over a year. The system's edge is the average across hundreds of trades, not any single one.</li>
 <li><b>Stops hit fast, winners take time.</b> Losing trades die in roughly half the time winners take. An early stop-out is the system doing its job.</li>
 <li><b>Drawdowns of ~10R are normal.</b> At 1% risk per trade, expect to be ~10% down from a peak at some point in a normal year. This is why risking more than 1% per trade is how accounts die.</li>
 <li><b>Backtest ≠ live.</b> Results above charge real spreads and realistic fills, but cannot model your personal latency, broker slippage, or requotes. Assume live results will be somewhat worse — especially on M5.</li>
@@ -131,7 +130,7 @@
 <tr><td><code>/scan</code></td><td>Run an immediate analysis of the primary timeframe.</td></tr>
 <tr><td><code>/autoscan</code></td><td>Toggle automatic scanning (M5 / M15 / H1 / H4 at each candle close).</td></tr>
 <tr><td><code>/status</code></td><td>Uptime, auto-scan state, WebSocket price-monitor state.</td></tr>
-<tr><td><code>/news</code></td><td>Net XAU/USD news score from the last 4 hours of headlines (neutral headlines are removed before averaging; informational only — never gates signals).</td></tr>
+<tr><td><code>/news</code></td><td>Net XAU/USD news sentiment score (informational only — never gates signals).</td></tr>
 <tr><td><code>/cache</code> · <code>/clearcache</code></td><td>Inspect / reset the market-data cache.</td></tr>
 <tr><td><code>/help</code></td><td>Command reference.</td></tr>
 </table>

engine.py

@@ -13,7 +13,7 @@ Design philosophy:
 
 import logging
 from dataclasses import dataclass, field
-from typing import Dict, List, Optional, Tuple
+from typing import Dict, List, Tuple
 
 import numpy as np
 import pandas as pd
@@ -32,13 +32,6 @@ class IndicatorSignal:
     confidence: float  # 0-100
     description: str
     weight: float = 1.0
-    # RECORD-KEEPING ONLY (stamped by _compute_composite; nothing reads
-    # them back). effective_weight = weight x regime multiplier actually
-    # used, weighted_score = its contribution to the composite. Both stay
-    # None for indicators the composite skipped (weight 0, `_`-prefixed
-    # private series) and for analyses that never reached _compute_composite.
-    effective_weight: Optional[float] = None
-    weighted_score: Optional[float] = None
 
 
 @dataclass
@@ -57,46 +50,6 @@ class TechnicalAnalysis:
     pattern_signals: List[str] = field(default_factory=list)
 
 
-# NOTE: ROC is directionally redundant with MACD/Keltner/EMA
-# (0.8-0.94 correlation), but zeroing its weight was backtested
-# 2026-07 and LOST 13R — the redundant vote effectively raises
-# the composite bar and the thresholds are calibrated with it.
-# Don't remove without retuning score thresholds.
-_TRENDING_WEIGHTS = {
-    "ema_trend": 2.0, "rsi": 1.0, "macd": 2.0,
-    "bollinger": 0.5, "keltner": 1.5, "roc": 1.5,
-    "candlestick": 0.8, "divergence": 1.0, "adx": 0.8,
-}
-_DEFAULT_REGIME_WEIGHTS = {
-    "trending_up": _TRENDING_WEIGHTS,
-    "trending_down": _TRENDING_WEIGHTS,
-    "trending": _TRENDING_WEIGHTS,
-    "ranging": {
-        "ema_trend": 0.3, "rsi": 1.5, "macd": 1.5,
-        "bollinger": 1.5, "keltner": 2.0, "roc": 0.5,
-        "candlestick": 1.0, "divergence": 2.5, "adx": 0.5,
-    },
-    "transitioning": {
-        "ema_trend": 1.5, "rsi": 1.0, "macd": 2.5,
-        "bollinger": 1.0, "keltner": 2.5, "roc": 2.0,
-        "candlestick": 1.0, "divergence": 2.0, "adx": 1.0,
-    },
-}
-
-
-def regime_weight_table(config: TechnicalConfig) -> Dict[str, Dict[str, float]]:
-    """The regime -> indicator weight multipliers _compute_composite runs
-    with (``regime_weights_override`` wins, else the gold-optimized
-    defaults above).
-
-    Pure and side-effect free: ONE source of truth shared by the composite
-    and by signal_aggregator.engine_config_fingerprint. Returns a copy, so
-    a caller can never mutate the shipped table."""
-    override = getattr(config, "regime_weights_override", None)
-    table = override if override else _DEFAULT_REGIME_WEIGHTS
-    return {regime: dict(weights) for regime, weights in table.items()}
-
-
 class TechnicalEngine:
     """
     Core technical analysis engine.
@@ -153,7 +106,7 @@ class TechnicalEngine:
         if "candlestick" not in disabled: self._detect_candlestick_patterns(df, result)
 
         # ── Classify market regime ─────────────────────────────────
-        self._classify_regime(result)
+        self._classify_regime(df, result)
 
         # ── Regime-aware RSI reinterpretation ──────────────────────
         if self.config.rsi_mode == "regime_aware" and "rsi" not in disabled:
@@ -793,6 +746,8 @@ class TechnicalEngine:
         result.key_levels["kc_upper"] = float(kc_upper_val)
         result.key_levels["kc_lower"] = float(kc_lower_val)
         result.key_levels["kc_mid"] = float(kc_mid_val)
+        result._kc_squeeze = is_squeeze  # type: ignore[attr-defined]
+        result._kc_squeeze_released = squeeze_released  # type: ignore[attr-defined]
 
     def _detect_divergences(self, df: pd.DataFrame, result: TechnicalAnalysis):
         """
@@ -873,7 +828,7 @@ class TechnicalEngine:
     def _compute_structure_sr(self, df: pd.DataFrame, result: TechnicalAnalysis):
         """Detect swing high/low structure levels for S/R and trade setup."""
         base_lookback = self.config.swing_sr_lookback
-        if result.timeframe == "5min":
+        if result.timeframe in ("1min", "5min"):
             base_lookback = max(base_lookback, 120)
         lookback = min(base_lookback, len(df) - 1)
         if lookback < 10:
@@ -1096,7 +1051,7 @@ class TechnicalEngine:
             confidence=conf, description=desc, weight=rsi_ind.weight,
         )
 
-    def _classify_regime(self, result: TechnicalAnalysis):
+    def _classify_regime(self, df: pd.DataFrame, result: TechnicalAnalysis):
         """Classify the market regime (trending vs ranging vs transitioning)."""
         adx_sig = result.indicators.get("adx")
         ema_sig = result.indicators.get("ema_trend")
@@ -1126,9 +1081,35 @@ class TechnicalEngine:
         total_weight = 0.0
         total_confidence = 0.0
 
-        # Override weights if provided, else gold-optimized defaults
-        # (regime_weight_table is the single source of truth).
-        regime_weights = regime_weight_table(self.config)
+        # Use override weights if provided, else gold-optimized defaults
+        if self.config.regime_weights_override:
+            regime_weights = self.config.regime_weights_override
+        else:
+            # NOTE: ROC is directionally redundant with MACD/Keltner/EMA
+            # (0.8-0.94 correlation), but zeroing its weight was backtested
+            # 2026-07 and LOST 13R — the redundant vote effectively raises
+            # the composite bar and the thresholds are calibrated with it.
+            # Don't remove without retuning score thresholds.
+            _trending = {
+                "ema_trend": 2.0, "rsi": 1.0, "macd": 2.0,
+                "bollinger": 0.5, "keltner": 1.5, "roc": 1.5,
+                "candlestick": 0.8, "divergence": 1.0, "adx": 0.8,
+            }
+            regime_weights = {
+                "trending_up": _trending,
+                "trending_down": _trending,
+                "trending": _trending,
+                "ranging": {
+                    "ema_trend": 0.3, "rsi": 1.5, "macd": 1.5,
+                    "bollinger": 1.5, "keltner": 2.0, "roc": 0.5,
+                    "candlestick": 1.0, "divergence": 2.5, "adx": 0.5,
+                },
+                "transitioning": {
+                    "ema_trend": 1.5, "rsi": 1.0, "macd": 2.5,
+                    "bollinger": 1.0, "keltner": 2.5, "roc": 2.0,
+                    "candlestick": 1.0, "divergence": 2.0, "adx": 1.0,
+                },
+            }
 
         active_regime_weights = regime_weights.get(result.market_regime, {})
 
@@ -1142,9 +1123,6 @@ class TechnicalEngine:
 
             signal_score = ind.signal.value
             weighted_score = signal_score * weight * (ind.confidence / 100.0)
-            # Record-keeping stamp (no maths depends on these two lines).
-            ind.effective_weight = float(weight)
-            ind.weighted_score = float(weighted_score)
 
             total_score += weighted_score
             total_weight += weight

llm_client.py

@@ -90,11 +90,7 @@ class DeepSeekClient:
             if self._count_day != today:
                 self._count_day = today
                 self._count = 0
-                # Clear ALL warn-once kinds, not just "cap": the network/
-                # http_*/parse flags used to be set once and never reset,
-                # so in a daemon that runs for weeks the second and every
-                # later outage logged at DEBUG only.
-                self._warned.clear()
+                self._warned.discard("cap")
             if self._count >= self._daily_cap:
                 self._warn_once(
                     "cap", "DeepSeek daily request cap (%d) exhausted — "
@@ -153,14 +149,6 @@ class DeepSeekClient:
                             "expected JSON object — ignoring")
         else:
             self.last_ok = time.time()
-            # A recovered call re-arms the failure warnings so the NEXT
-            # outage is visible at WARNING again. "cap" keeps its own
-            # UTC-day reset in _under_cap. (Taken under the lock; safe
-            # here because chat_json holds no lock — _under_cap has
-            # already returned.)
-            if self._warned - {"cap"}:
-                with self._lock:
-                    self._warned = {k for k in self._warned if k == "cap"}
         return parsed
 
     @staticmethod
@@ -198,3 +186,10 @@ def get_llm_client() -> DeepSeekClient:
                 enabled=settings.get("enabled", True) is not False,
             )
         return _client
+
+
+def reset_llm_client() -> None:
+    """Drop the singleton (tests / config reloads)."""
+    global _client
+    with _client_lock:
+        _client = None

main.py

@@ -13,7 +13,6 @@ import json
 import logging
 import sys
 from datetime import datetime, timezone
-from typing import Optional
 
 # Force UTF-8 output on Windows so box-drawing chars render
 if hasattr(sys.stdout, "reconfigure"):
@@ -21,13 +20,9 @@ if hasattr(sys.stdout, "reconfigure"):
 if hasattr(sys.stderr, "reconfigure"):
     sys.stderr.reconfigure(encoding="utf-8")
 
-from agent_panel import attach_panel
-from app_config import get_app_config, violent_watch_enabled
 from config import PlatformConfig, Timeframe, SignalStrength, XAUUSD
-from market_outlook import OutlookResult, maybe_run_outlook
 from pair_signals import PAIR_PRODUCERS
 from signal_aggregator import SignalAggregator, SignalReport
-from violent_watch import attach_watch
 
 
 # ══════════════════════════════════════════════════════════════════
@@ -103,8 +98,8 @@ def _compact_reason(reason: str) -> str:
 
 
 def _fmt_price(price: float) -> str:
-    """Gold-scale price, 2 dp."""
-    return f"{price:.2f}"
+    """2dp for gold-scale prices, 5dp for sub-10 FX-scale prices."""
+    return f"{price:.2f}" if price >= 10 else f"{price:.5f}"
 
 
 def print_signal_report(report: SignalReport, detail: bool = False):
@@ -162,23 +157,12 @@ def print_signal_report(report: SignalReport, detail: bool = False):
     if report.warnings:
         for w in report.warnings:
             print(f"  {Colors.YELLOW}⚠{Colors.RESET}  {w}")
-    if report.watch_line:
-        print(f"  {Colors.YELLOW}🌪{Colors.RESET} {report.watch_line}")
     if report.event_description and report.event_risk != "clear":
         print(f"  {Colors.YELLOW}📅{Colors.RESET} {report.event_description}")
     if report.news_description and report.news_description != "Neutral news sentiment":
         print(f"  {Colors.MAGENTA}📰{Colors.RESET} {report.news_description}")
     if report.news_summary:
         print(f"  {Colors.MAGENTA}🧠{Colors.RESET} {report.news_summary}")
-    if report.panel_verdict:
-        panel_label = ("AI macro view"
-                      if report.panel_verdict_source == "macro" else "AI Panel")
-        print(f"  {Colors.CYAN}🤖{Colors.RESET} {panel_label}: "
-              f"{Colors.BOLD}{report.panel_verdict}{Colors.RESET} "
-              f"({report.panel_confidence:.0f}%) — {report.panel_rationale}")
-        for t in report.panel_takes:
-            print(f"     {Colors.GRAY}{t['persona']:<11} {t['stance']:<8} "
-                  f"{t['note']}{Colors.RESET}")
 
     # ── 6. DETAIL (only with --detail) ─────────────────────────────
     if detail and report.primary_analysis:
@@ -205,24 +189,6 @@ def print_signal_report(report: SignalReport, detail: bool = False):
                     print(f"    {label:>20}  {color}{val:.3f}{Colors.RESET}")
 
 
-def print_market_outlook(outlook: Optional[OutlookResult]) -> None:
-    """Render the independent advisory outlook for explicit CLI use."""
-    print(f"\n  {Colors.BOLD}AI Market Outlook{Colors.RESET}")
-    if outlook is None:
-        print(f"  {Colors.GRAY}Unavailable (disabled, market closed, or no LLM response).{Colors.RESET}")
-        return
-    forming = {
-        "buy": "possible buy forming",
-        "sell": "possible sell forming",
-        "none": "no clear setup forming",
-    }[outlook.setup_forming]
-    print(f"  {outlook.bias.title()} · {forming} · {outlook.confidence:.0f}%")
-    if outlook.watch:
-        print(f"  Watch: {outlook.watch}")
-    if outlook.note:
-        print(f"  {Colors.GRAY}{outlook.note}{Colors.RESET}")
-
-
 def report_to_dict(report: SignalReport) -> dict:
     d = {
         "symbol": report.symbol,
@@ -243,14 +209,6 @@ def report_to_dict(report: SignalReport) -> dict:
         "news_sentiment": round(report.news_sentiment, 3),
         "news_description": report.news_description,
         "news_summary": report.news_summary,
-        "panel_verdict": report.panel_verdict,
-        "panel_confidence": round(report.panel_confidence, 1),
-        "panel_rationale": report.panel_rationale,
-        "panel_takes": report.panel_takes,
-        "panel_created_at": report.panel_created_at,
-        "panel_cache_status": report.panel_cache_status,
-        "panel_payload_hash": report.panel_payload_hash,
-        "watch_line": report.watch_line,
         "bullish_reasons": report.bullish_reasons,
         "bearish_reasons": report.bearish_reasons,
         "warnings": report.warnings,
@@ -267,8 +225,6 @@ def report_to_dict(report: SignalReport) -> dict:
             "risk_pips": round(ts.risk_pips, 1),
             "reward_pips": round(ts.reward_pips, 1),
             "strategy": ts.strategy,
-            "setup_pattern": ts.setup_pattern,
-            "round_trip_cost": ts.round_trip_cost,
             "trail_arm_r": ts.trail_arm_r,
             "trail_dist_r": ts.trail_dist_r,
         }
@@ -308,16 +264,10 @@ Examples:
     parser.add_argument("--pair", "-p", default="XAUUSD",
                         help="Pair to analyze: XAUUSD (default: XAUUSD)")
     parser.add_argument("--timeframe", "-tf", default="H1",
-                        choices=list(_TF_BY_NAME),
+                        choices=["M5", "M15", "M30", "H1", "H4", "D1"],
                         help="Primary analysis timeframe (default: H1)")
     parser.add_argument("--detail", action="store_true",
                         help="Show full indicator breakdown")
-    parser.add_argument("--panel", action="store_true",
-                        help="Run the LLM agent panel on the result "
-                             "(advisory only; needs a DeepSeek key)")
-    parser.add_argument("--outlook", action="store_true",
-                        help="Run the hourly AI market outlook now "
-                             "(advisory only; needs a DeepSeek key)")
     parser.add_argument("--json", action="store_true",
                         help="Output as JSON")
     parser.add_argument("--verbose", "-v", action="store_true",
@@ -326,14 +276,6 @@ Examples:
     return parser.parse_args()
 
 
-# The four validated primary timeframes (the same set the bot and the
-# dashboard scan). M30/D1 were never validated as primaries.
-_TF_BY_NAME = {
-    "M5": Timeframe.M5, "M15": Timeframe.M15,
-    "H1": Timeframe.H1, "H4": Timeframe.H4,
-}
-
-
 def _resolve_pair(s: str) -> str:
     aliases = {
         "XAUUSD": XAUUSD, "XAU/USD": XAUUSD, "GOLD": XAUUSD,
@@ -345,57 +287,48 @@ def _resolve_pair(s: str) -> str:
 
 
 def build_config(args) -> PlatformConfig:
-    """Per-pair tuned config from the pair_signals factory.
-
-    The factory owns the validated confirmation map (M5->[M15,H1],
-    M15->[M5,H1], H1->[M15,H4], H4->[H1,D1]); passing None here keeps
-    the CLI on the same map as the bot and the dashboard. A CLI-local
-    table once paired M5 with the unvalidated [M1, M15].
-    """
     pair = _resolve_pair(args.pair)
-    return PAIR_PRODUCERS[pair](_TF_BY_NAME[args.timeframe], None)
+
+    tf_map = {
+        "M5": Timeframe.M5, "M15": Timeframe.M15, "M30": Timeframe.M30,
+        "H1": Timeframe.H1, "H4": Timeframe.H4, "D1": Timeframe.D1,
+    }
+    primary_tf = tf_map[args.timeframe]
+
+    tf_confirmations = {
+        "M5":  [Timeframe.M1,  Timeframe.M15],
+        "M15": [Timeframe.M5,  Timeframe.H1],
+        "M30": [Timeframe.M15, Timeframe.H1],
+        "H1":  [Timeframe.M15, Timeframe.H4],
+        "H4":  [Timeframe.H1,  Timeframe.D1],
+        "D1":  [Timeframe.H4,  Timeframe.W1],
+    }
+    confirmation_tfs = tf_confirmations.get(
+        args.timeframe, [Timeframe.M15, Timeframe.H4]
+    )
+
+    # Per-pair tuned configs from the pair_signals factories.
+    factory = PAIR_PRODUCERS.get(pair)
+    if factory is not None:
+        return factory(primary_tf, confirmation_tfs)
+
+    config = PlatformConfig()
+    config.primary_timeframe = primary_tf
+    config.confirmation_timeframes = confirmation_tfs
+    return config
 
 
 def run_live(args, config: PlatformConfig):
     print(f"{Colors.GRAY}  Fetching market data and running analysis...{Colors.RESET}\n")
     aggregator = SignalAggregator(config)
-    report = aggregator.analyze_pair(symbol=config.symbol, log_source="cli")
-
-    # Explicit user intent: force bypasses the enabled/surface/quality
-    # toggles, but the daily panel budget and LLM cap still apply.
-    if getattr(args, "panel", False):
-        attach_panel(report, surface="cli", force=True)
-
-    # Display-only violent-move watch (dark by default; never touches
-    # the signal, only report.watch_line).
-    if violent_watch_enabled():
-        attach_watch(report, config.primary_timeframe,
-                     get_app_config().violent_watch_settings)
-
-    outlook = None
-    if getattr(args, "outlook", False):
-        outlook = maybe_run_outlook(surface="cli", force=True)
+    report = aggregator.analyze_pair(symbol=config.symbol)
 
     if args.json:
-        payload = report_to_dict(report)
-        if getattr(args, "outlook", False):
-            payload["market_outlook"] = (
-                None if outlook is None else {
-                    "bias": outlook.bias,
-                    "setup_forming": outlook.setup_forming,
-                    "confidence": outlook.confidence,
-                    "watch": outlook.watch,
-                    "note": outlook.note,
-                    "created_ts": outlook.created_ts,
-                }
-            )
-        print(json.dumps(payload, indent=2))
+        print(json.dumps(report_to_dict(report), indent=2))
         return
 
     print_header(f"{config.display_name} SIGNAL INTELLIGENCE")
     print_signal_report(report, detail=args.detail)
-    if getattr(args, "outlook", False):
-        print_market_outlook(outlook)
     print(f"\n{Colors.CYAN}{'═' * 80}{Colors.RESET}")
     print(f"{Colors.GRAY}  Disclaimer: For educational/research purposes only.{Colors.RESET}")
     print(f"{Colors.CYAN}{'═' * 80}{Colors.RESET}\n")

market_outlook.py

@@ -1,439 +0,0 @@
-"""
-ForexBot2 — AI market outlook (advisory hourly assessment)
-===========================================================
-Builds a broad XAU/USD market brief and asks DeepSeek whether a good setup
-appears to be forming. The result is written to the signal-history database
-and may be displayed by consumers, but it is ADVISORY-ONLY: it never changes
-signal scoring, gates, grades, or trade setup generation.
-
-The outlook runs outside the replayable signal path. Individual candle/news
-failures produce a partial brief; an unavailable LLM, malformed reply, or
-locked database never reaches a scan caller as an exception.
-
-The static prompt is intentional: DeepSeek prices byte-identical prompt
-prefixes at its cache-hit rate, so timestamps and all other changing evidence
-live only in the JSON user payload.
-"""
-
-import datetime as _dt
-import logging
-import math
-import os
-import sqlite3
-import threading
-import time
-from dataclasses import dataclass
-from typing import Any, Dict, List, Optional
-
-from agent_panel import _normalize_candles
-from app_config import (get_app_config, market_outlook_enabled,
-                        signal_log_path)
-from config import PlatformConfig, Timeframe, XAUUSD
-from engine import TechnicalEngine
-from llm_client import get_llm_client
-from providers import base as _provider_base
-
-logger = logging.getLogger(__name__)
-
-DEFAULT_MAX_PER_DAY = 12
-DEFAULT_CANDLE_BARS = 20
-_OUTLOOK_TFS = (Timeframe.M15, Timeframe.H1, Timeframe.H4)
-_VALID_BIASES = ("bullish", "bearish", "neutral")
-_VALID_SETUPS = ("buy", "sell", "none")
-
-
-# STATIC prompt: repeated prompt prefixes are much cheaper with DeepSeek's
-# server-side cache. Variable market evidence belongs exclusively in payload.
-SYSTEM_PROMPT = """\
-You are writing an advisory market outlook for gold (XAU/USD). You receive a
-JSON market brief covering recent candle shape, indicator states, price
-levels, news, and recent technical signals. Assess what may be forming soon;
-you are not a signal generator and must not provide an entry price, stop, or
-target.
-
-Gold doctrine: hawkish Fed / rate hikes / rising real yields / strong
-dollar are gold-bearish; easing / rate cuts / safe-haven flows /
-geopolitical stress / weak dollar are gold-bullish.
-
-Indicator "value" units: rsi 0-100 (above 70 overbought, below 30
-oversold); adx 0-100 (above 25 = strong trend); macd = histogram in
-dollars (momentum); roc = percent; bollinger = position 0-1 (0 = lower
-band, 1 = upper band); keltner = dollar distance from the mid-line;
-ema_trend = 0-1 alignment score; divergence and candlestick = pattern
-counts. "candles" is per timeframe: {"ref_close", "atr", "bars"}; bars
-are oldest-first [open, high, low, close] in ATR units relative to the
-latest close. Read their shape, never quote them as prices.
-
-Hard rules: use ONLY the JSON provided. Do not invent numbers, events, or
-price levels. Your training data about gold, markets, and current events is
-stale and OFF-LIMITS: never use outside knowledge; if a fact is not in the
-JSON, treat it as unknown. If evidence is thin or missing, answer "neutral"
-with confidence below 30. Use plain everyday English for someone unfamiliar
-with markets: short simple sentences, no acronyms or jargon. Keep "watch"
-and "note" to at most 25 words each.
-
-Reply with ONLY JSON:
-{"bias": "bullish"|"bearish"|"neutral",
- "setup_forming": "buy"|"sell"|"none",
- "confidence": 0-100,
- "watch": "<what to watch for, max 25 words>",
- "note": "<plain-English summary, max 25 words>"}
-"""
-
-
-@dataclass
-class OutlookResult:
-    bias: str
-    setup_forming: str
-    confidence: float
-    watch: str
-    note: str
-    created_ts: float
-
-
-def _finite(x: Any) -> Optional[float]:
-    """float(x) or None for non-numeric / non-finite JSON values."""
-    try:
-        value = float(x)
-    except (TypeError, ValueError):
-        return None
-    return value if math.isfinite(value) else None
-
-
-def _round_finite(x: Any, digits: int) -> Optional[float]:
-    value = _finite(x)
-    return round(value, digits) if value is not None else None
-
-
-def _json_safe(value: Any) -> Any:
-    """Replace any accidental NaN/inf nested in provider data with null."""
-    if isinstance(value, dict):
-        return {str(key): _json_safe(item) for key, item in value.items()}
-    if isinstance(value, (list, tuple)):
-        return [_json_safe(item) for item in value]
-    if isinstance(value, bool) or value is None or isinstance(value, str):
-        return value
-    if isinstance(value, (int, float)):
-        return _finite(value)
-    return value
-
-
-def _clamp_confidence(raw: Any) -> Optional[float]:
-    value = _finite(raw)
-    return max(0.0, min(100.0, value)) if value is not None else None
-
-
-def _word_limit(text: Any, words: int = 25) -> str:
-    """Bound a response field by words as well as a practical char limit."""
-    clean = str(text or "").strip()
-    return " ".join(clean.split()[:words])[:200]
-
-
-def _validate_outlook(reply: Any) -> Optional[OutlookResult]:
-    """Validate only model replies that fully match the requested contract."""
-    if not isinstance(reply, dict):
-        return None
-    bias = reply.get("bias")
-    setup_forming = reply.get("setup_forming")
-    confidence = _clamp_confidence(reply.get("confidence"))
-    if (bias not in _VALID_BIASES or setup_forming not in _VALID_SETUPS
-            or confidence is None):
-        return None
-    return OutlookResult(
-        bias=bias,
-        setup_forming=setup_forming,
-        confidence=confidence,
-        watch=_word_limit(reply.get("watch")),
-        note=_word_limit(reply.get("note")),
-        created_ts=time.time(),
-    )
-
-
-# ── Data collection ──────────────────────────────────────────────────
-
-_dp: Any = None  # Patchable singleton, mirroring agent_panel._dp.
-
-
-def _outlook_provider() -> Any:
-    """Lazy provider singleton so its closed-bar cache stays warm hourly."""
-    global _dp
-    if _dp is None:
-        from providers import make_data_provider
-        _dp = make_data_provider(PlatformConfig())
-    return _dp
-
-
-def _indicator_payload(analysis: Any) -> Dict[str, Dict[str, Any]]:
-    """Visible scored indicators with finite, compact values only."""
-    out: Dict[str, Dict[str, Any]] = {}
-    for name, indicator in (getattr(analysis, "indicators", None) or {}).items():
-        if name.startswith("_") or getattr(indicator, "weight", 0) == 0:
-            continue
-        entry: Dict[str, Any] = {
-            "signal": getattr(getattr(indicator, "signal", None), "name", ""),
-            "confidence": _round_finite(
-                getattr(indicator, "confidence", None), 1),
-            "value": _round_finite(getattr(indicator, "value", None), 2),
-        }
-        out[name] = entry
-    return out
-
-
-def _analysis_payload(analyses: Dict[str, Any]) -> Dict[str, Any]:
-    """Pure formatter for the technical portion of the market brief."""
-    indicators: Dict[str, Any] = {}
-    market_regime: Dict[str, Optional[str]] = {}
-    volatility_regime: Dict[str, Optional[str]] = {}
-    key_levels: Dict[str, Dict[str, Optional[float]]] = {}
-    for tf in _OUTLOOK_TFS:
-        name = tf.value
-        analysis = analyses.get(name)
-        indicators[name] = _indicator_payload(analysis)
-        market_regime[name] = getattr(analysis, "market_regime", None)
-        volatility_regime[name] = getattr(analysis, "volatility_regime", None)
-        key_levels[name] = {
-            level: _round_finite(value, 2)
-            for level, value in (getattr(analysis, "key_levels", None) or {}).items()
-        }
-    return {
-        "indicators": indicators,
-        "market_regime": market_regime,
-        "volatility_regime": volatility_regime,
-        "key_levels": key_levels,
-    }
-
-
-def _empty_news_payload() -> Dict[str, Any]:
-    return {
-        "headlines": None,
-        "events": None,
-        "gold_sentiment": None,
-        "remaining_high_impact_events": None,
-    }
-
-
-def _news_payload() -> Dict[str, Any]:
-    """Collect current news without allowing any news failure to block us."""
-    try:
-        from news_engine import NewsEngine
-
-        news = NewsEngine(PlatformConfig().news)
-        context = news.get_fundamental_context()
-        headlines = [str(getattr(h, "headline", ""))
-                     for h in (context.news_headlines or [])[:10]
-                     if getattr(h, "headline", None)]
-        events = [
-            {
-                "event": getattr(event, "event", None),
-                "currency": getattr(event, "currency", None),
-                "actual": getattr(event, "actual", None),
-                "forecast": getattr(event, "forecast", None),
-                "released": not event.is_upcoming,
-            }
-            for event in (context.events_today or [])
-            if event.is_high_impact
-        ][:8]
-        gold_sentiment, _ = news.get_pair_sentiment(XAUUSD, context)
-        remaining = sum(
-            1 for event in news.get_week_events()
-            if event.is_high_impact and event.is_upcoming
-        )
-        return {
-            "headlines": headlines,
-            "events": events,
-            "gold_sentiment": _round_finite(gold_sentiment, 2),
-            "remaining_high_impact_events": remaining,
-        }
-    except Exception as exc:
-        logger.debug("market outlook: news context unavailable: %s", exc)
-        return _empty_news_payload()
-
-
-def _recent_signals() -> List[Dict[str, Any]]:
-    """Read the latest signal history through SQLite's immutable URI."""
-    try:
-        path = os.path.abspath(signal_log_path())
-        # mode=ro (not immutable=1): immutable ignores the WAL, so the
-        # newest rows written by the live loggers were invisible here.
-        uri = f"file:{path}?mode=ro"
-        con = sqlite3.connect(uri, uri=True)
-        try:
-            rows = con.execute(
-                "SELECT logged_at, timeframe, signal, quality, panel_verdict "
-                "FROM signals ORDER BY id DESC LIMIT 10").fetchall()
-        finally:
-            con.close()
-        return [
-            {
-                "logged_at": row[0],
-                "timeframe": row[1],
-                "signal": row[2],
-                "quality": row[3],
-                "panel_verdict": row[4],
-            }
-            for row in reversed(rows)
-        ]
-    except Exception as exc:
-        logger.debug("market outlook: signal history unavailable: %s", exc)
-        return []
-
-
-def _build_payload(
-    analyses: Dict[str, Any],
-    candles: Dict[str, Any],
-    news: Dict[str, Any],
-    recent_signals: List[Dict[str, Any]],
-    as_of: _dt.datetime,
-) -> Dict[str, Any]:
-    """Pure market-brief formatter. Every numeric field is finite or null."""
-    technical = _analysis_payload(analyses)
-    return _json_safe({
-        "as_of": as_of.astimezone(_dt.timezone.utc).isoformat(),
-        "candles": candles,
-        **technical,
-        "news": news,
-        "recent_signals": recent_signals,
-    })
-
-
-def _collect_payload(settings: Dict[str, Any]) -> Dict[str, Any]:
-    """Fetch closed bars and turn them into a structured market brief."""
-    analyses: Dict[str, Any] = {}
-    candles: Dict[str, Any] = {tf.value: None for tf in _OUTLOOK_TFS}
-    bars = int(settings.get("candles_bars", DEFAULT_CANDLE_BARS))
-    provider = _outlook_provider()
-    engine = TechnicalEngine(PlatformConfig().technical)
-
-    for tf in _OUTLOOK_TFS:
-        try:
-            df = provider.fetch_ohlcv(XAUUSD, tf)
-            analysis = engine.analyze(df, XAUUSD, tf.value)
-            analyses[tf.value] = analysis
-            atr = _finite(getattr(
-                (getattr(analysis, "indicators", None) or {}).get("atr"),
-                "value", None))
-            if atr is not None and atr > 0 and df is not None and len(df) >= 2:
-                candles[tf.value] = _normalize_candles(df, atr, bars)
-        except Exception as exc:
-            logger.debug("market outlook: %s data unavailable: %s", tf.value, exc)
-
-    return _build_payload(
-        analyses=analyses,
-        candles=candles,
-        news=_news_payload(),
-        recent_signals=_recent_signals(),
-        as_of=_dt.datetime.now(_dt.timezone.utc),
-    )
-
-
-# ── Orchestration, hourly dedup, and request budget ──────────────────
-
-_state_lock = threading.Lock()
-_last_run_hour: Optional[int] = None   # UTC hour bucket of the last attempt
-_outlook_day: Optional[_dt.date] = None
-_outlook_count = 0
-_warned_budget = False
-
-
-def _skip(reason: str) -> None:
-    """Heartbeat for every silent early return — a dark outlook feature
-    (zero rows for 15 days, 2026-08/09) must be visible in the log."""
-    logger.info("market outlook: skipped (%s)", reason)
-
-
-def reset_outlook_state() -> None:
-    """Drop hourly and daily state (tests / deployment config reloads)."""
-    global _last_run_hour, _outlook_day, _outlook_count
-    global _warned_budget
-    with _state_lock:
-        _last_run_hour = None
-        _outlook_day = None
-        _outlook_count = 0
-        _warned_budget = False
-
-
-def _begin_attempt(cap: int, force: bool) -> bool:
-    """Atomically apply hourly dedup, then reserve a daily attempt.
-
-    Hourly dedup is by UTC hour BUCKET, not ``now - last < 3600``: the
-    H1 loop calls this at boundary+10 s+scan time, so consecutive hours
-    were routinely 3,590 s apart and every other hour was dropped."""
-    global _last_run_hour, _outlook_day, _outlook_count
-    global _warned_budget
-    now = time.time()
-    hour_bucket = int(now // 3600)
-    today = _dt.datetime.now(_dt.timezone.utc).date()
-    with _state_lock:
-        if not force and _last_run_hour == hour_bucket:
-            _skip(f"hour bucket {hour_bucket} already attempted")
-            return False
-        if _outlook_day != today:
-            _outlook_day = today
-            _outlook_count = 0
-            _warned_budget = False
-        if _outlook_count >= cap:
-            if not _warned_budget:
-                _warned_budget = True
-                logger.warning(
-                    "market outlook: daily budget (%d) exhausted — no more "
-                    "outlooks until UTC midnight", cap)
-            else:
-                _skip("daily budget exhausted")
-            return False
-        _outlook_count += 1
-        _last_run_hour = hour_bucket
-        return True
-
-
-def run_outlook(settings: Dict[str, Any],
-                surface: str = "live") -> Optional[OutlookResult]:
-    """Build, ask, validate, and persist one advisory outlook."""
-    try:
-        payload = _collect_payload(settings)
-        reply = get_llm_client().chat_json(SYSTEM_PROMPT, payload)
-        result = _validate_outlook(reply)
-        if result is None:
-            logger.warning(
-                "market outlook: no outlook produced (%s)",
-                "LLM returned nothing — cap/timeout/disabled" if reply is None
-                else "reply failed validation")
-            return None
-        try:
-            from signal_log import log_outlook
-            log_outlook(result, surface=surface, payload=payload)
-        except Exception:
-            logger.debug("market outlook: persistence unavailable", exc_info=True)
-        return result
-    except Exception:
-        logger.warning("market outlook: run failed", exc_info=True)
-        return None
-
-
-def maybe_run_outlook(surface: str, force: bool = False) -> Optional[OutlookResult]:
-    """Best-effort public entry point. It NEVER raises to a scan caller.
-
-    ``force`` is for explicit CLI use: it bypasses enabled, surface, and
-    hourly gates, but not the independent outlook budget or shared LLM cap.
-    """
-    try:
-        settings = get_app_config().market_outlook_settings
-        if not force:
-            if not market_outlook_enabled():
-                _skip("disabled")
-                return None
-            if surface == "telegram" and not settings.get("telegram", False):
-                _skip("telegram surface off")
-                return None
-        if _provider_base._is_forex_weekend():
-            _skip("weekend")
-            return None
-        cap = int(settings.get("max_per_day", DEFAULT_MAX_PER_DAY))
-        if cap < 1:
-            _skip("max_per_day < 1")
-            return None
-        if not _begin_attempt(cap, force):
-            return None
-        return run_outlook(settings, surface=surface)
-    except Exception:
-        logger.warning("market outlook: unavailable", exc_info=True)
-        return None

news_archive.py

@@ -1,361 +0,0 @@
-"""
-ForexBot2 — News archive (durable headline corpus)
-==================================================
-
-Every headline the news engine fetches is written here, exactly as it
-arrived, together with the score that was attached to it and **which
-scorer produced that score**. One row per distinct headline, kept
-forever.
-
-Why this exists
----------------
-Headlines used to survive only as a side effect of signal logging, inside
-``logs/signals.db``. That database was rotated/rebuilt around 2026-09-04
-and three weeks of news — 4,054 distinct headlines, the corpus the
-``news_sentiment_value_2026-09-04.md`` study was written on — was lost.
-``logs/`` is gitignored, so git could not bring it back and nobody
-noticed until 2026-09-08.
-
-This module writes to its own database (``logs/newsdata.db`` by default)
-so signal logging can be wiped, rotated or rebuilt without touching the
-news corpus.
-
-What the corpus is for
-----------------------
-Training a small local scorer that reproduces DeepSeek's headline
-labelling without calling it. On the 2026-08-06 golden set DeepSeek
-scored 78/78 and the built-in keyword scorer it overrides scored 24/78
-(31%) — and the keyword score is what stands whenever DeepSeek is
-unavailable (no key, disabled, daily cap, timeout, HTTP/parse failure,
-or the >25-new-headlines-per-cycle batch overflow). ``score_source``
-records which one produced each row; training filters on ``'llm'``.
-
-Contract
---------
-* **Never raises.** ``archive_headlines`` swallows every exception. A
-  logging failure must not break a scan — the same contract
-  ``signal_log.log_signal`` holds.
-* **Upgrade-only scores.** A row already marked ``llm`` is never
-  overwritten by a later ``keyword`` score. A headline first seen under
-  the batch-overflow cap is keyword-scored, then gets its DeepSeek score
-  on a later fetch window; the better label wins and is never demoted.
-* **Dedup by headline text.** ``hash`` = sha256 of the headline. A
-  headline seen 50 times is one row with ``last_seen`` bumped.
-* **stdlib only** (``sqlite3``), like ``signal_log``.
-
-Known corpus limitation: a headline mentioning no tracked currency is
-never constructed by the news engine at all (``news_engine`` drops it
-before scoring), so DeepSeek never labels it either and it cannot appear
-here.
-"""
-
-from __future__ import annotations
-
-import hashlib
-import json
-import logging
-import os
-import sqlite3
-import threading
-import time
-from datetime import datetime, timezone
-from typing import Any, Dict, Iterable, Optional
-
-import app_config
-
-logger = logging.getLogger(__name__)
-
-
-_SCHEMA = """
-CREATE TABLE IF NOT EXISTS headlines (
-    hash TEXT PRIMARY KEY,
-    headline TEXT NOT NULL,
-    description TEXT,
-    link TEXT,
-    source TEXT,
-    publisher TEXT,
-    published_at TEXT,
-    first_seen TEXT NOT NULL,
-    last_seen TEXT NOT NULL,
-    seen_count INTEGER NOT NULL DEFAULT 1,
-    score REAL,
-    score_by_currency_json TEXT,
-    currencies_json TEXT,
-    keywords_json TEXT,
-    score_source TEXT NOT NULL DEFAULT 'keyword',
-    teacher_model TEXT,
-    teacher_prompt_sha1 TEXT,
-    origin TEXT NOT NULL DEFAULT 'live'
-);
-CREATE INDEX IF NOT EXISTS idx_headlines_source ON headlines (score_source);
-CREATE INDEX IF NOT EXISTS idx_headlines_first_seen ON headlines (first_seen);
-"""
-
-#: Rank of a score's provenance. A write only lands when its rank is >=
-#: the rank already stored, so a keyword score can never overwrite an LLM
-#: one (the batch-overflow case) but an LLM score always upgrades.
-_SOURCE_RANK = {"unknown": 0, "keyword": 1, "model": 2, "llm": 3}
-
-
-def _utc_now_iso() -> str:
-    return datetime.now(timezone.utc).isoformat()
-
-
-def headline_hash(headline: str) -> str:
-    """Primary key for a headline: sha256 of its exact text."""
-    return hashlib.sha256(headline.encode("utf-8")).hexdigest()
-
-
-def _json_or_none(value: Any) -> Optional[str]:
-    if value is None:
-        return None
-    try:
-        return json.dumps(value, default=str)
-    except Exception:
-        return None
-
-
-def _iso_or_none(value: Any) -> Optional[str]:
-    if value is None:
-        return None
-    if isinstance(value, datetime):
-        return value.isoformat()
-    return str(value)
-
-
-def _finite_score(value: Any) -> Optional[float]:
-    try:
-        out = float(value)
-    except (TypeError, ValueError):
-        return None
-    if out != out or out in (float("inf"), float("-inf")):
-        return None
-    return out
-
-
-class NewsArchive:
-    """Thread-safe SQLite writer for the headline corpus. One instance
-    per database path; safe to share across the bot's executor threads
-    and the dashboard daemon's pool."""
-
-    def __init__(self, path: str):
-        parent = os.path.dirname(path)
-        if parent:
-            os.makedirs(parent, exist_ok=True)
-        self._conn = sqlite3.connect(path, check_same_thread=False)
-        # busy_timeout BEFORE the journal-mode switch: the bot, the
-        # dashboard publisher and the tests open this file at once and
-        # the WAL switch needs a brief exclusive lock.
-        self._conn.execute("PRAGMA busy_timeout=5000")
-        self._lock = threading.Lock()
-        with self._lock:
-            # busy_timeout does not cover the WAL switch, which needs a
-            # brief exclusive lock: three processes creating this file at
-            # once will race it. Same retry signal_log uses.
-            self._with_lock_retry(
-                lambda: self._conn.execute("PRAGMA journal_mode=WAL"))
-            self._with_lock_retry(lambda: self._conn.executescript(_SCHEMA))
-            self._conn.commit()
-
-    @staticmethod
-    def _with_lock_retry(fn, attempts: int = 8, delay: float = 0.05):
-        """Run ``fn`` again on 'database is locked' (concurrent openers
-        racing the same schema step); other errors propagate at once."""
-        for attempt in range(attempts):
-            try:
-                return fn()
-            except sqlite3.OperationalError as exc:
-                if "locked" not in str(exc).lower() or attempt == attempts - 1:
-                    raise
-                time.sleep(delay * (attempt + 1))
-
-    def close(self) -> None:
-        with self._lock:
-            try:
-                self._conn.close()
-            except Exception:
-                pass
-
-    #: Rank of the score already stored, computed in SQL so the compare
-    #: happens inside the same atomic statement as the write.
-    _STORED_RANK = ("CASE headlines.score_source WHEN 'llm' THEN 3"
-                    " WHEN 'model' THEN 2 WHEN 'keyword' THEN 1 ELSE 0 END")
-
-    def store(self, rows: Iterable[Dict[str, Any]]) -> int:
-        """Insert-or-upgrade a batch of prepared rows. Returns the number
-        of rows written. Raises on a database failure — the module-level
-        wrapper is what guarantees a scan cannot break.
-
-        One atomic UPSERT per row. A read-then-write pair would lose an
-        increment whenever two processes (bot, dashboard publisher, tests)
-        first see the same headline at the same moment: both would find it
-        absent, one INSERT would win and the other's count would vanish.
-        """
-        rows = list(rows)
-        if not rows:
-            return 0
-        now = _utc_now_iso()
-        wins = f"? >= {self._STORED_RANK}"   # incoming rank vs stored rank
-        sql = (
-            "INSERT INTO headlines ("
-            " hash, headline, description, link, source, publisher,"
-            " published_at, first_seen, last_seen, seen_count,"
-            " score, score_by_currency_json, currencies_json,"
-            " keywords_json, score_source, teacher_model,"
-            " teacher_prompt_sha1, origin"
-            ") VALUES (?,?,?,?,?,?,?,?,?,1,?,?,?,?,?,?,?,?)"
-            " ON CONFLICT(hash) DO UPDATE SET"
-            # The backfill stamps historical timestamps, so clamp rather
-            # than assign: first_seen only ever moves earlier, last_seen
-            # only ever moves later.
-            "  first_seen = MIN(headlines.first_seen, excluded.first_seen),"
-            "  last_seen = MAX(headlines.last_seen, excluded.last_seen),"
-            "  seen_count = headlines.seen_count + 1,"
-            # Upgrade-only: a weaker scorer never overwrites a stronger
-            # one's label, but re-seeing a headline always counts.
-            f"  score = CASE WHEN {wins} THEN excluded.score"
-            "           ELSE headlines.score END,"
-            f"  score_by_currency_json = CASE WHEN {wins}"
-            "           THEN excluded.score_by_currency_json"
-            "           ELSE headlines.score_by_currency_json END,"
-            f"  currencies_json = CASE WHEN {wins} THEN excluded.currencies_json"
-            "           ELSE headlines.currencies_json END,"
-            f"  keywords_json = CASE WHEN {wins} THEN excluded.keywords_json"
-            "           ELSE headlines.keywords_json END,"
-            # COALESCE, not a bare overwrite: the backfill re-ingests
-            # llm-scored headlines with NO teacher identity (neither source
-            # DB recorded one), and rank >= rank is true, so a plain
-            # assignment blanks what the live hook already stored.
-            f"  teacher_model = CASE WHEN {wins}"
-            "           THEN COALESCE(excluded.teacher_model,"
-            "                         headlines.teacher_model)"
-            "           ELSE headlines.teacher_model END,"
-            f"  teacher_prompt_sha1 = CASE WHEN {wins}"
-            "           THEN COALESCE(excluded.teacher_prompt_sha1,"
-            "                         headlines.teacher_prompt_sha1)"
-            "           ELSE headlines.teacher_prompt_sha1 END,"
-            f"  score_source = CASE WHEN {wins} THEN excluded.score_source"
-            "           ELSE headlines.score_source END,"
-            # Enrichment fills gaps but never blanks what is already there
-            # (the legacy backfill shape carries no description or link).
-            "  description = COALESCE(excluded.description, headlines.description),"
-            "  link = COALESCE(excluded.link, headlines.link),"
-            "  publisher = COALESCE(excluded.publisher, headlines.publisher),"
-            "  published_at = COALESCE(excluded.published_at,"
-            "                          headlines.published_at)")
-
-        # The rank is bound once per CASE arm. Positional binding means a
-        # miscount silently shifts every parameter, so assert it instead of
-        # trusting the arithmetic: 17 INSERT values + one rank per arm.
-        n_rank = sql.count("? >= ")
-        n_values = 17
-        assert sql.count("?") == n_values + n_rank, (
-            f"placeholder count drifted: {sql.count('?')} "
-            f"!= {n_values} + {n_rank}")
-
-        params = []
-        for row in rows:
-            rank = _SOURCE_RANK.get(row["score_source"], 0)
-            params.append((
-                row["hash"], row["headline"], row["description"], row["link"],
-                row["source"], row["publisher"], row["published_at"],
-                row.get("first_seen") or now, row.get("last_seen") or now,
-                row["score"], row["score_by_currency_json"],
-                row["currencies_json"], row["keywords_json"],
-                row["score_source"], row["teacher_model"],
-                row["teacher_prompt_sha1"], row.get("origin", "live"),
-            ) + (rank,) * n_rank)
-
-        def _write() -> None:
-            # BEGIN IMMEDIATE takes the write lock up front, so contention
-            # fails here — before any row is applied — and the rollback
-            # makes the batch atomic if it fails later. Without this a
-            # partial executemany would be re-applied by the retry and
-            # double-count seen_count on the rows that already landed.
-            self._conn.execute("BEGIN IMMEDIATE")
-            try:
-                self._conn.executemany(sql, params)
-                self._conn.commit()
-            except Exception:
-                try:
-                    self._conn.rollback()
-                except Exception:
-                    pass
-                raise
-
-        with self._lock:
-            self._with_lock_retry(_write)
-        return len(rows)
-
-
-def build_row(headline: Any, *, teacher_model: Optional[str] = None,
-              teacher_prompt_sha1: Optional[str] = None,
-              origin: str = "live",
-              first_seen: Optional[str] = None,
-              last_seen: Optional[str] = None) -> Optional[Dict[str, Any]]:
-    """Flatten one ``NewsSentiment`` (or an equivalent object) into a row.
-
-    Returns None when the object carries no usable headline text."""
-    text = getattr(headline, "headline", None)
-    if not isinstance(text, str) or not text.strip():
-        return None
-    source = getattr(headline, "score_source", None) or "keyword"
-    return {
-        "hash": headline_hash(text),
-        "headline": text,
-        "description": getattr(headline, "description", None),
-        "link": getattr(headline, "link", None),
-        "source": getattr(headline, "source", None),
-        "publisher": getattr(headline, "publisher", None),
-        "published_at": _iso_or_none(getattr(headline, "timestamp", None)),
-        "first_seen": first_seen,
-        "last_seen": last_seen,
-        "score": _finite_score(getattr(headline, "sentiment_score", None)),
-        "score_by_currency_json": _json_or_none(
-            getattr(headline, "sentiment_by_currency", None)),
-        "currencies_json": _json_or_none(
-            getattr(headline, "currencies_mentioned", None)),
-        "keywords_json": _json_or_none(getattr(headline, "keywords", None)),
-        "score_source": source,
-        "teacher_model": teacher_model if source == "llm" else None,
-        "teacher_prompt_sha1": (
-            teacher_prompt_sha1 if source == "llm" else None),
-        "origin": origin,
-    }
-
-
-#: path -> NewsArchive
-_archives: Dict[str, NewsArchive] = {}
-_archives_lock = threading.Lock()
-
-
-def archive_headlines(headlines: Any, *, teacher_model: Optional[str] = None,
-                      teacher_prompt_sha1: Optional[str] = None,
-                      origin: str = "live") -> None:
-    """Persist a batch of scored headlines if the news_archive config
-    block enables it. NEVER raises — a logging failure must not break a
-    scan."""
-    try:
-        if not headlines:
-            return
-        if not app_config.news_archive_enabled():
-            return
-        rows = []
-        for headline in headlines:
-            row = build_row(headline, teacher_model=teacher_model,
-                            teacher_prompt_sha1=teacher_prompt_sha1,
-                            origin=origin)
-            if row is not None:
-                rows.append(row)
-        if not rows:
-            return
-        path = app_config.news_archive_path()
-        with _archives_lock:
-            archive = _archives.get(path)
-            if archive is None:
-                archive = NewsArchive(path)
-                _archives[path] = archive
-        archive.store(rows)
-    except Exception:
-        logger.warning("news_archive: failed to persist headlines",
-                       exc_info=True)

news_engine.py

@@ -11,7 +11,6 @@ A perfect technical setup can be destroyed by an unexpected NFP print.
 import hashlib
 import html
 import logging
-import math
 import re
 import threading
 import time
@@ -23,7 +22,6 @@ from typing import Dict, List, Optional, Tuple
 
 import requests
 
-import news_archive
 from config import NewsConfig
 from llm_client import get_llm_client
 
@@ -81,35 +79,16 @@ Reply with ONLY JSON:
 """
 
 _LLM_SUMMARY_SYSTEM = """\
-You write a short news note about the gold price for someone who has
-never traded. You receive JSON with recent "headlines" (newest first;
-each carries "gold": "up" or "down", the platform's own reading of how
-that story pushes the gold price — trust it) and today's high-impact
-"events" (released: true means the number is out).
-Say three things, in this order: what is happening to gold right now,
-why it matters for the gold price, and the next scheduled report to
-watch if one is listed. Use only the headlines and events given; if
-they are thin, say so plainly.
-Style: plain everyday English, short simple sentences, no acronyms, no
-abbreviations, no trader words. Write "the US central bank", not "the
-Fed"; "interest rates", not "yields" or "hawkish"; "a report on US
-prices", not "CPI" or "PCE"; "gold", never "XAU". No advice, no hedging
-boilerplate. 2-3 sentences, max 55 words.
+You are a forex/gold market-context writer. You receive JSON with recent
+"headlines" and today's high-impact "events" (released: true means the
+number is out). Write a 1-2 sentence plain-English summary of the
+current macro backdrop for a GOLD (XAU/USD) trader: the dominant theme
+and its likely pull on gold. No advice, no hedging boilerplate, max 45
+words.
 Reply with ONLY JSON: {"summary": "..."}
-
-Examples:
-{"summary": "Gold is falling today. The US central bank hinted it may raise interest rates, which makes the dollar stronger, and a stronger dollar usually pushes gold down. The next thing to watch is Thursday's report on US prices."}
-{"summary": "Gold is climbing. Fresh fighting in the Middle East has people moving money into gold as a safe place, and a weaker dollar is helping too. No major US reports are due today."}
 """
 
 
-#: Identity of the labelling prompt, stored on every archived LLM score.
-#: If the prompt changes, scores before and after are answers to different
-#: questions and must not be pooled into one training set.
-_LLM_SENTIMENT_PROMPT_SHA1 = hashlib.sha1(
-    _LLM_SENTIMENT_SYSTEM.encode("utf-8")).hexdigest()
-
-
 _RELEASE_VALUE_RE = re.compile(r"(-?\d+(?:\.\d+)?)\s*([kmbt])?", re.IGNORECASE)
 _RELEASE_MULTIPLIERS = {"k": 1e3, "m": 1e6, "b": 1e9, "t": 1e12}
 
@@ -220,12 +199,6 @@ class NewsSentiment:
     link: Optional[str] = None         # article URL (Google News = redirect)
     description: Optional[str] = None  # plain text, tags stripped, ≤500 chars
     publisher: Optional[str] = None    # Google News <source> text, else None
-    # Which scorer produced sentiment_score / sentiment_by_currency:
-    # "keyword" (the built-in scorer, ~31% on the 2026-08-06 golden set),
-    # "llm" (DeepSeek, 78/78 on the same set) or "model" (a trained local
-    # artifact). Without it the two are indistinguishable in the archive
-    # and a model trained on the mix inherits the keyword scorer's errors.
-    score_source: str = "keyword"
 
 
 @dataclass
@@ -237,52 +210,11 @@ class FundamentalContext:
     warning_message: str = ""
     news_sentiment: Dict[str, float] = field(default_factory=dict)  # currency -> sentiment
     news_headlines: List[NewsSentiment] = field(default_factory=list)
+    event_impact_on_pairs: Dict[str, str] = field(default_factory=dict)
     calendar_available: bool = False  # True if live calendar data was fetched
     llm_summary: str = ""  # DeepSeek market-context blurb (display-only)
 
 
-# Headlines whose every relevant score is inside ±NEUTRAL_HEADLINE_THRESHOLD
-# lean neither way: they carry no direction for display and only dilute the
-# per-currency mean. Same threshold agent_panel uses for its macro payload.
-NEUTRAL_HEADLINE_THRESHOLD = 0.1
-
-
-def _is_neutral_headline(h) -> bool:
-    """True when no per-currency score on the headline leans either way.
-
-    A headline whose overall score is 0.0 but whose gold-specific score is
-    -0.7 is NOT neutral. Missing, empty or non-finite scores count as
-    neutral (they cannot be shown as leaning one way)."""
-    by_currency = getattr(h, "sentiment_by_currency", None) or {}
-    overall = getattr(h, "sentiment_score", None)
-    values = []
-    for currency in (getattr(h, "currencies_mentioned", None) or []):
-        value = by_currency.get(currency, overall)
-        if isinstance(value, (int, float)) and math.isfinite(value):
-            values.append(abs(float(value)))
-    return not values or max(values) < NEUTRAL_HEADLINE_THRESHOLD
-
-
-def _gold_direction_label(h) -> str:
-    """"up" / "down" / "flat" for the market-summary payload — the same
-    XAU-first, USD-inverted reading agent_panel uses for its macro persona."""
-    by_currency = getattr(h, "sentiment_by_currency", None) or {}
-    xau = by_currency.get("XAU")
-    usd = by_currency.get("USD")
-    if isinstance(xau, (int, float)) and math.isfinite(xau):
-        score = float(xau)
-    elif isinstance(usd, (int, float)) and math.isfinite(usd):
-        score = -float(usd)
-    else:
-        raw = getattr(h, "sentiment_score", None)
-        score = float(raw) if isinstance(raw, (int, float)) and math.isfinite(raw) else 0.0
-    if score > 0:
-        return "up"
-    if score < 0:
-        return "down"
-    return "flat"
-
-
 class NewsEngine:
     """
     Economic calendar and news sentiment analyzer.
@@ -428,23 +360,21 @@ class NewsEngine:
                     )
                     break
 
+            # Map upcoming high-impact events to affected pairs
+            for event in context.events_upcoming_1h:
+                if event.is_high_impact:
+                    affected = self._get_affected_pairs(event.currency)
+                    for pair in affected:
+                        context.event_impact_on_pairs[pair] = (
+                            f"{event.currency} {event.event} in {event.minutes_until:.0f}min"
+                        )
+
         except Exception as e:
             logger.warning(f"Economic calendar fetch failed: {e}")
 
         # 2. News sentiment
         try:
             headlines = self._fetch_news_sentiment()
-            # Drop headlines that lean neither way (2026-09-04). Applied
-            # here rather than in the fetch so the class-level cache and
-            # the feed-health counts still describe what the feeds sent;
-            # every consumer (aggregate score, summary, dashboard, panel,
-            # outlook, signal log) reads context.news_headlines.
-            kept = [h for h in headlines if not _is_neutral_headline(h)]
-            if len(kept) != len(headlines):
-                logger.debug(
-                    f"News: dropped {len(headlines) - len(kept)} neutral "
-                    f"headlines, {len(kept)} kept")
-            headlines = kept
             context.news_headlines = headlines
 
             # Aggregate sentiment per currency (per-currency score where
@@ -471,10 +401,11 @@ class NewsEngine:
 
     @staticmethod
     def _parse_pair(pair_symbol: str) -> Tuple[str, str]:
-        base, sep, quote = pair_symbol.partition("/")
-        if not sep:
-            raise ValueError(f"pair symbol must be BASE/QUOTE, got {pair_symbol!r}")
-        return base, quote
+        if "/" in pair_symbol:
+            parts = pair_symbol.split("/")
+            return parts[0], parts[1]
+        clean = pair_symbol.replace("=X", "")
+        return clean[:3], clean[3:]
 
     def get_event_risk_level(self, pair_symbol: str, context: FundamentalContext) -> Tuple[str, str]:
         """
@@ -814,7 +745,6 @@ class NewsEngine:
         headlines = []
         seen_titles = set()
         ok_feeds = 0
-        undated = 0
 
         for feed_url in self.config.rss_feeds:
             host = feed_url.split("/")[2] if "//" in feed_url else feed_url
@@ -840,14 +770,8 @@ class NewsEngine:
                         pub_date_el.text if pub_date_el is not None else None
                     )
 
-                    # Only recent headlines (configurable: headline_max_age_seconds).
-                    # An item without a parsable pubDate cannot be proven
-                    # fresh, so it is dropped too — it used to be stamped
-                    # "now" and never aged out (2026-09-04).
-                    if pub_date is None:
-                        undated += 1
-                        continue
-                    if (datetime.now(timezone.utc) - pub_date).total_seconds() > self.config.headline_max_age_seconds:
+                    # Only recent headlines (last 24h)
+                    if pub_date and (datetime.now(timezone.utc) - pub_date).total_seconds() > 86400:
                         continue
 
                     # Dedup within and across feeds so a syndicated story
@@ -903,8 +827,6 @@ class NewsEngine:
                     f"rss:{host}", False, f"{type(e).__name__}: {e}")
                 logger.debug(f"Failed to fetch RSS from {feed_url}: {e}")
 
-        if undated:
-            logger.debug(f"RSS: dropped {undated} headlines without a publish time")
         self._record_source(
             "headlines", ok_feeds > 0,
             detail="all feeds failed" if ok_feeds == 0 else "",
@@ -916,23 +838,6 @@ class NewsEngine:
         except Exception as e:
             logger.debug(f"LLM sentiment failed, keyword scores stand: {e}")
 
-        # Archive BEFORE any downstream filtering. get_fundamental_context
-        # drops every headline inside ±NEUTRAL_HEADLINE_THRESHOLD, so a
-        # corpus built from what the context carries holds almost no
-        # genuinely neutral headlines and would teach a trained scorer
-        # that "neutral" barely exists. Never raises.
-        # Guarded at the call site as well as inside archive_headlines:
-        # arguments are evaluated BEFORE that function's own try/except, so
-        # a raising get_llm_client() would escape it. Archiving is a side
-        # effect — it must never cost us the headlines we just fetched.
-        try:
-            news_archive.archive_headlines(
-                headlines,
-                teacher_model=getattr(get_llm_client(), "_model", None),
-                teacher_prompt_sha1=_LLM_SENTIMENT_PROMPT_SHA1)
-        except Exception as e:
-            logger.debug(f"news archive skipped, headlines unaffected: {e}")
-
         return headlines
 
     def _analyze_headline_sentiment(
@@ -1034,7 +939,6 @@ class NewsEngine:
                 cur: (xau if cur == "XAU" else overall)
                 for cur in h.currencies_mentioned
             }
-            h.score_source = "llm"
 
     def _llm_event_direction(self, event: EconomicEvent) -> Optional[str]:
         """Gold direction for a released surprise, e.g. "bearish: strong
@@ -1082,11 +986,9 @@ class NewsEngine:
         return text or None
 
     def _llm_market_summary(self, context: FundamentalContext) -> str:
-        """2-3 sentence plain-language news note for alerts; "" when
-        unavailable. Headlines go newest-first with the platform's own
-        gold direction so the note agrees with the score badges. Cached
-        (success OR failure) for the context-cache TTL so at most one
-        call is made per window."""
+        """1-2 sentence macro blurb for alerts; "" when unavailable.
+        Cached (success OR failure) for the context-cache TTL so at most
+        one call is made per window."""
         with NewsEngine._cache_lock:
             ts, cached = NewsEngine._llm_summary_cache
             if time.time() - ts < self.config.context_cache_ttl_seconds:
@@ -1097,17 +999,8 @@ class NewsEngine:
         client = get_llm_client()
         if not client.available:
             return ""
-        oldest = datetime.min.replace(tzinfo=timezone.utc)
-        recent = sorted(
-            context.news_headlines,
-            key=lambda h: getattr(h, "timestamp", None) or oldest,
-            reverse=True,
-        )[:15]
         result = client.chat_json(_LLM_SUMMARY_SYSTEM, {
-            "headlines": [
-                {"headline": h.headline, "gold": _gold_direction_label(h)}
-                for h in recent
-            ],
+            "headlines": [h.headline for h in context.news_headlines[:15]],
             "events": [
                 {"event": e.event, "currency": e.currency,
                  "actual": e.actual, "forecast": e.forecast,
@@ -1117,7 +1010,7 @@ class NewsEngine:
         })
         summary = ""
         if result:
-            summary = str(result.get("summary", "")).strip()[:400]
+            summary = str(result.get("summary", "")).strip()[:300]
         with NewsEngine._cache_lock:
             NewsEngine._llm_summary_cache = (time.time(), summary)
         return summary
@@ -1170,3 +1063,11 @@ class NewsEngine:
             except ValueError:
                 continue
         return None
+
+    @staticmethod
+    def _get_affected_pairs(currency: str) -> List[str]:
+        _MAP = {
+            "USD": ["XAU/USD"],
+            "XAU": ["XAU/USD"],
+        }
+        return _MAP.get(currency, [])

oanda_daemon.py

@@ -0,0 +1,854 @@
+"""
+ForexBot2 — Oanda Automated Trading Daemon
+============================================
+Autonomous trading process for XAU/USD on an Oanda practice account.
+Runs candle-boundary scan loops (M5/M15/H1/H4), executes trades via
+the Oanda v20 API, and manages trailing stops at bar close — matching
+the backtested bar-close ratchet model.
+
+Book-parity notes (matching backtest/gpu_backtest.py):
+- No broker-side take-profit: in the trailing model TP1/TP2 are
+  advisory; the only exits are SL (broker), TRAIL (daemon ratchet),
+  and TIMEOUT (max_hold_bars). A hard TP would cap winners the trail
+  lets run and diverge the live book from the backtest.
+- Entry cooldown is 5 bars per timeframe (the backtester's only
+  sequential rule) — not a fingerprint block.
+- The trail ratchets at bar CLOSE only, and each closed bar is
+  processed exactly once (last_bar_ts guard), so weekends and feed
+  lag can't double-count bars or inflate the hold counter.
+
+Divergence from the backtest book (accepted for demo safety): one
+position total. While a trade is open, other signals are logged and
+skipped, so skipped entries can be compared against the book later.
+
+Usage:  python oanda_daemon.py
+Config: env vars OANDA_ACCOUNT_ID, OANDA_API_TOKEN, OANDA_BASE_URL
+        or config.json "oanda" section (env wins).
+"""
+
+import asyncio
+import json
+import logging
+import os
+import signal
+import sys
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime, timezone
+from logging.handlers import TimedRotatingFileHandler
+from typing import Dict, Optional
+
+import pandas as pd
+
+from app_config import (
+    get_app_config, oanda_account_id, oanda_api_token, oanda_base_url,
+)
+from config import PAIR_INFO, PlatformConfig, Timeframe, XAUUSD
+from signal_aggregator import SignalAggregator, SignalReport, SignalStrength
+from pair_signals import PAIR_PRODUCERS
+from providers.oanda import OandaDataProvider
+from oanda_executor import OandaExecutor, OandaAPIError
+
+logger = logging.getLogger(__name__)
+
+# ── Constants ──────────────────────────────────────────────────────────
+
+SYMBOL = XAUUSD
+INSTRUMENT = "XAU_USD"
+
+SCAN_CONFIRMATIONS = {
+    Timeframe.M5:  [Timeframe.M15, Timeframe.H1],
+    Timeframe.M15: [Timeframe.M5, Timeframe.H1],
+    Timeframe.H1:  [Timeframe.M15, Timeframe.H4],
+    Timeframe.H4:  [Timeframe.H1, Timeframe.D1],
+}
+
+TF_INTERVAL_MINUTES = {
+    Timeframe.M5: 5, Timeframe.M15: 15, Timeframe.M30: 30,
+    Timeframe.H1: 60, Timeframe.H4: 240, Timeframe.D1: 1440,
+}
+
+SCAN_TIMEFRAMES = [Timeframe.M5, Timeframe.M15, Timeframe.H1, Timeframe.H4]
+
+QUALITY_RANK = {"A+": 4, "A": 3, "B": 2, "C": 1, "D": 0, "F": -1}
+
+# Matches the backtester's --cooldown-bars default (the only sequential
+# rule in the replay): no new entry within 5 bars of the previous entry
+# on the same timeframe.
+COOLDOWN_BARS = 5
+
+# How often the heartbeat confirms the process is alive and the account
+# is still reachable — independent of the candle-boundary loops, so a
+# wedged scan or mid-run credential expiry still surfaces in the log.
+HEARTBEAT_SECONDS = 3600
+
+STATE_FILE = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "oanda_state.json")
+
+# ── Config resolution (env → config.json → default) ──────────────────
+
+def _oanda_setting_int(env_key: str, json_key: str, default: int) -> int:
+    env = os.environ.get(env_key)
+    if env:
+        try:
+            return int(env)
+        except ValueError:
+            raise SystemExit(
+                f"{env_key}={env!r} is not an integer — refusing to start")
+    try:
+        return int(get_app_config().oanda_settings.get(json_key, default))
+    except (TypeError, ValueError):
+        raise SystemExit(
+            f"config.json oanda.{json_key} is not an integer — "
+            f"refusing to start")
+
+
+def _oanda_setting_str(env_key: str, json_key: str, default: str) -> str:
+    return (os.environ.get(env_key)
+            or str(get_app_config().oanda_settings.get(json_key, default)))
+
+# ── Logging setup ──────────────────────────────────────────────────────
+
+def _setup_logging(level: str = "INFO") -> None:
+    log_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "logs")
+    os.makedirs(log_dir, exist_ok=True)
+    log_file = os.path.join(log_dir, "daemon.log")
+
+    fmt = logging.Formatter(
+        "%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S")
+
+    file_handler = TimedRotatingFileHandler(
+        log_file, when="h", interval=3, backupCount=240, utc=True)
+    file_handler.setFormatter(fmt)
+
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_handler.setFormatter(fmt)
+
+    logging.basicConfig(
+        level=getattr(logging, level.upper(), logging.INFO),
+        handlers=[file_handler, console_handler],
+        force=True)
+
+# ── Boundary timing (same as telegram_bot.py) ─────────────────────────
+
+def _seconds_until_boundary(interval_minutes: int, buffer: int = 10) -> float:
+    now = datetime.now(timezone.utc)
+    total_minutes = now.hour * 60 + now.minute
+    elapsed_minutes = total_minutes % interval_minutes
+    elapsed_seconds = elapsed_minutes * 60 + now.second + now.microsecond / 1e6
+    interval_seconds = interval_minutes * 60
+    remaining = interval_seconds - elapsed_seconds + buffer
+    # NOTE: remaining > buffer always (elapsed < interval by construction);
+    # a call at the exact boundary instant targets the NEXT boundary —
+    # steady-state loops wake at boundary+buffer, so only a process
+    # started in the boundary microsecond skips one scan (2026-08-04
+    # audit I42; the old `remaining <= buffer` guard was unreachable).
+    return remaining
+
+# ── Trade state persistence ───────────────────────────────────────────
+
+def _load_state() -> Dict:
+    if os.path.exists(STATE_FILE):
+        try:
+            with open(STATE_FILE, encoding="utf-8") as fh:
+                return json.load(fh)
+        except (OSError, json.JSONDecodeError):
+            logger.warning("Could not load state file, starting fresh")
+    return {"trades": {}, "last_entries": {}}
+
+
+def _save_state(state: Dict) -> None:
+    try:
+        tmp = STATE_FILE + ".tmp"
+        with open(tmp, "w", encoding="utf-8") as fh:
+            json.dump(state, fh, indent=2)
+        os.replace(tmp, STATE_FILE)
+    except OSError:
+        logger.exception("Failed to save state file")
+
+# ── Trading Daemon ─────────────────────────────────────────────────────
+
+class TradingDaemon:
+
+    def __init__(self):
+        account_id = oanda_account_id()
+        api_token = oanda_api_token()
+        base_url = oanda_base_url()
+        lot_size = _oanda_setting_int("OANDA_LOT_SIZE", "lot_size", 1)
+        self._quality_gate = _oanda_setting_str(
+            "OANDA_QUALITY_GATE", "quality_gate", "C").strip()
+        if self._quality_gate not in QUALITY_RANK:
+            # Fail CLOSED: an unrecognized gate must never silently fall
+            # back to the most permissive rank (QUALITY_RANK.get default).
+            raise SystemExit(
+                f"OANDA_QUALITY_GATE/quality_gate {self._quality_gate!r} "
+                f"is not one of {sorted(QUALITY_RANK)} — refusing to start")
+        self._max_hold_bars = _oanda_setting_int(
+            "OANDA_MAX_HOLD_BARS", "max_hold_bars", 300)
+        self._price_precision = PAIR_INFO[SYMBOL]["price_precision"]
+
+        if not account_id or not api_token:
+            msg = (
+                "Missing Oanda credentials. Set OANDA_ACCOUNT_ID and "
+                "OANDA_API_TOKEN env vars or config.json oanda section.")
+            logger.critical(msg)
+            raise SystemExit(msg)
+
+        self._provider = OandaDataProvider(
+            PlatformConfig(), api_token, base_url)
+        self._executor = OandaExecutor(
+            account_id, api_token, base_url, lot_size,
+            price_precision=self._price_precision)
+        self._thread_pool = ThreadPoolExecutor(max_workers=4)
+
+        self._state = _load_state()
+        self._state.setdefault("trades", {})
+        self._state.setdefault("last_entries", {})
+        self._state_lock = threading.Lock()
+        # Serializes the position-check + order-placement critical section
+        # across concurrent TF loops (all four fire at aligned boundaries).
+        self._entry_lock = threading.Lock()
+        self._running = True
+
+    # ── Main entry point ───────────────────────────────────────────
+
+    async def run(self) -> None:
+        loop = asyncio.get_running_loop()
+        loop.set_exception_handler(self._asyncio_exception_handler)
+        for sig in (signal.SIGINT, signal.SIGTERM):
+            loop.add_signal_handler(sig, self._request_shutdown)
+
+        logger.info("=== Oanda Trading Daemon starting ===")
+        logger.info(
+            "Quality gate: %s | Max hold: %d bars | Cooldown: %d bars | "
+            "Timeframes: %s",
+            self._quality_gate, self._max_hold_bars, COOLDOWN_BARS,
+            "/".join(tf.name for tf in SCAN_TIMEFRAMES))
+
+        # Credentials first: with a bad token every later call fails, so
+        # refuse to start before touching positions or spawning loops.
+        self._verify_account_access()
+
+        self._discover_existing_positions()
+
+        tasks = []
+        for tf in SCAN_TIMEFRAMES:
+            interval = TF_INTERVAL_MINUTES[tf]
+            tasks.append(asyncio.create_task(
+                self._candle_loop(tf, interval)))
+        tasks.append(asyncio.create_task(self._heartbeat_loop()))
+
+        logger.info("Candle-boundary loops started")
+        try:
+            await asyncio.gather(*tasks)
+        except asyncio.CancelledError:
+            pass
+        finally:
+            self._shutdown()
+
+    def _verify_account_access(self) -> None:
+        """Boot-time credential check. A 4xx here means the token/account
+        id is wrong — that never fixes itself, so refuse to start rather
+        than run a daemon that can't place or manage trades. A connection
+        failure or 5xx is a transient outage: log it and continue, since
+        broker-side stops already protect any open position and the
+        hourly heartbeat will keep reporting account health."""
+        try:
+            self._executor.get_account_summary()
+        except OandaAPIError as e:
+            if 400 <= e.status_code < 500:
+                logger.critical(
+                    "Oanda rejected the account credentials (HTTP %d): "
+                    "%s — refusing to start", e.status_code, e)
+                raise SystemExit(
+                    f"Oanda rejected the account credentials "
+                    f"(HTTP {e.status_code}): {e}")
+            logger.error(
+                "Failed to fetch account summary at boot (HTTP %d): %s",
+                e.status_code, e)
+
+    def _asyncio_exception_handler(self, loop, context) -> None:
+        exc = context.get("exception")
+        if exc is not None:
+            logger.critical(
+                "Uncaught asyncio exception: %s",
+                context.get("message", ""), exc_info=exc)
+        else:
+            logger.critical(
+                "Uncaught asyncio error: %s", context.get("message", ""))
+
+    def _request_shutdown(self) -> None:
+        logger.info("Shutdown requested")
+        self._running = False
+        for task in asyncio.all_tasks():
+            task.cancel()
+
+    def _shutdown(self) -> None:
+        with self._state_lock:
+            _save_state(self._state)
+        self._thread_pool.shutdown(wait=False)
+        logger.info("=== Oanda Trading Daemon stopped ===")
+
+    # ── Position discovery (restart recovery) ──────────────────────
+
+    def _discover_existing_positions(self) -> None:
+        self._reconcile_positions("boot discovery")
+
+    def _reconcile_positions(self, reason: str) -> None:
+        """Re-sync tracked state with the broker's open trades. Safe to
+        call mid-run (lock-guarded); used at boot and whenever a fill or
+        placement error suggests state and broker may have diverged."""
+        try:
+            open_trades = self._executor.get_open_trades(INSTRUMENT)
+        except OandaAPIError:
+            logger.exception(
+                "Failed to query open trades (%s) — reconcile skipped",
+                reason)
+            return
+
+        live_ids = {t["id"] for t in open_trades}
+
+        with self._state_lock:
+            saved = self._state.get("trades", {})
+
+            stale = [tid for tid in saved if tid not in live_ids]
+            for tid in stale:
+                logger.info(
+                    "Stale trade %s in state file (closed on Oanda), "
+                    "removing", tid)
+                del saved[tid]
+
+            for trade in open_trades:
+                tid = trade["id"]
+                if tid in saved:
+                    logger.info(
+                        "Recovered trade %s from state: %s %s @ %.2f, SL %.2f",
+                        tid, saved[tid].get("direction"), INSTRUMENT,
+                        saved[tid].get("entry"), saved[tid].get("current_sl"))
+                    continue
+
+                direction = ("BUY" if float(trade.get("currentUnits", 0)) > 0
+                             else "SELL")
+                entry = float(trade.get("price", 0))
+                sl_order = trade.get("stopLossOrder") or {}
+                sl_price = sl_order.get("price")
+                if sl_price is None:
+                    # No broker-side stop at all: risk is undefined, so the
+                    # trailing model cannot manage this trade. Scream — a
+                    # naked position needs a human (or a policy) to set a
+                    # stop; fabricating one here would invent trading
+                    # behavior the books never validated.
+                    logger.critical(
+                        "Adopted trade %s (%s @ %.2f) has NO STOP LOSS on "
+                        "the broker — it will NOT be trailed. Set a stop "
+                        "manually or close it.", tid, direction, entry)
+                    sl = entry
+                    risk = 0.0
+                else:
+                    sl = float(sl_price)
+                    risk = abs(entry - sl)
+
+                saved[tid] = {
+                    "tf": "H1",
+                    "direction": direction,
+                    "entry": entry,
+                    "sl_initial": sl,
+                    "current_sl": sl,
+                    "risk": risk,
+                    "peak": entry,
+                    "trail_armed": False,
+                    "trail_arm_r": 0.6,
+                    "trail_dist_r": 0.75,
+                    "bars_held": 0,
+                    "last_bar_ts": None,
+                    "signal_bar_ts": None,
+                }
+                logger.info(
+                    "Adopted unknown trade %s: %s @ %.2f, SL %.2f, risk %.2f "
+                    "(conservative defaults)",
+                    tid, direction, entry, sl, risk)
+
+            self._state["trades"] = saved
+            _save_state(self._state)
+        logger.info(
+            "Position reconcile (%s): %d active trade(s)", reason, len(saved))
+
+    # ── Candle-boundary loop ───────────────────────────────────────
+
+    async def _candle_loop(self, tf: Timeframe, interval_minutes: int) -> None:
+        label = tf.value
+        try:
+            while self._running:
+                wait = _seconds_until_boundary(interval_minutes, buffer=10)
+                logger.info("%s loop: next scan in %.0fs", label, wait)
+                await asyncio.sleep(wait)
+                if not self._running:
+                    break
+                logger.info("%s loop: firing scan", label)
+                try:
+                    await self._scan_and_manage(tf)
+                except Exception:
+                    logger.exception("%s loop: cycle failed", label)
+        except asyncio.CancelledError:
+            pass
+        finally:
+            logger.info("%s loop stopped", label)
+
+    async def _scan_and_manage(self, tf: Timeframe) -> None:
+        loop = asyncio.get_running_loop()
+
+        # Trade management FIRST: the SL ratchet must not wait on (or be
+        # starved by) the signal scan — news fetches can take seconds and
+        # a failed scan must never cost an open trade its trail update.
+        try:
+            await loop.run_in_executor(
+                self._thread_pool, self._manage_open_trades, tf)
+        except Exception:
+            logger.exception("%s: trade management failed", tf.value)
+
+        report = await loop.run_in_executor(
+            self._thread_pool, self._run_signal, tf)
+
+        if not self._should_execute(report, tf):
+            return
+
+        await loop.run_in_executor(
+            self._thread_pool, self._execute_trade, report, tf)
+
+    def _manage_open_trades(self, tf: Timeframe) -> None:
+        self._check_closed_trades()
+        self._manage_trailing(tf)
+
+    # ── Heartbeat ─────────────────────────────────────────────────
+
+    def _heartbeat_once(self) -> None:
+        with self._state_lock:
+            n = len(self._state.get("trades", {}))
+        try:
+            self._executor.get_account_summary()
+        except OandaAPIError as e:
+            logger.error("HEARTBEAT: account summary failed: %s", e)
+        # Logged unconditionally — even on a failed summary call the
+        # heartbeat line itself must reach the log, so a wedged process
+        # doesn't look identical to a healthy but silent one.
+        logger.info("HEARTBEAT: alive | tracked trades: %d", n)
+
+    async def _heartbeat_loop(self) -> None:
+        loop = asyncio.get_running_loop()
+        try:
+            while self._running:
+                await asyncio.sleep(HEARTBEAT_SECONDS)
+                if not self._running:
+                    break
+                try:
+                    await loop.run_in_executor(
+                        self._thread_pool, self._heartbeat_once)
+                except Exception:
+                    logger.exception("heartbeat loop: cycle failed")
+        except asyncio.CancelledError:
+            pass
+        finally:
+            logger.info("heartbeat loop stopped")
+
+    # ── Signal pipeline ────────────────────────────────────────────
+
+    def _run_signal(self, tf: Timeframe) -> SignalReport:
+        conf_tfs = SCAN_CONFIRMATIONS.get(
+            tf, [Timeframe.M15, Timeframe.H4])
+        factory = PAIR_PRODUCERS.get(SYMBOL)
+        if factory is not None:
+            config = factory(tf, conf_tfs)
+        else:
+            config = PlatformConfig()
+            config.primary_timeframe = tf
+            config.confirmation_timeframes = conf_tfs
+
+        aggregator = SignalAggregator(config)
+        aggregator.data_provider = self._provider
+        report = aggregator.analyze_pair(symbol=SYMBOL)
+
+        logger.info(
+            "%s scan: signal=%s score=%.1f conf=%.0f quality=%s "
+            "regime=%s vol=%s price=%.2f spread=%.4f",
+            tf.value,
+            report.final_signal.name, report.final_score,
+            report.final_confidence, report.signal_quality,
+            report.market_regime, report.volatility_regime,
+            report.current_price,
+            self._provider.get_spread(SYMBOL))
+
+        # Stale-data/suppression warnings must be visible at WARNING;
+        # routine session-filter notices go to DEBUG — the M5 loop fires
+        # 288x/day and would drown the log at WARNING otherwise.
+        for w in report.warnings:
+            if "STALE" in w.upper() or "FAIL" in w.upper():
+                logger.warning("%s scan: %s", tf.value, w)
+            else:
+                logger.debug("%s scan: %s", tf.value, w)
+
+        return report
+
+    # ── Execution decision ─────────────────────────────────────────
+
+    def _should_execute(self, report: SignalReport, tf: Timeframe) -> bool:
+        if report.final_signal == SignalStrength.NEUTRAL:
+            return False
+
+        if report.trade_setup is None:
+            return False
+
+        quality_rank = QUALITY_RANK.get(report.signal_quality, -1)
+        gate_rank = QUALITY_RANK.get(self._quality_gate, 1)
+        if quality_rank < gate_rank:
+            return False
+
+        if self._in_cooldown(tf):
+            logger.info(
+                "%s: actionable signal (%s %s) but inside %d-bar entry "
+                "cooldown, skipping",
+                tf.value, report.trade_setup.direction,
+                report.signal_quality, COOLDOWN_BARS)
+            return False
+
+        with self._state_lock:
+            if self._state.get("trades"):
+                logger.info(
+                    "%s: actionable signal (%s %s) but position already "
+                    "open, skipping",
+                    tf.value, report.trade_setup.direction,
+                    report.signal_quality)
+                return False
+
+        return True
+
+    def _in_cooldown(self, tf: Timeframe) -> bool:
+        """Backtest parity: no entry within COOLDOWN_BARS bars of the
+        previous entry on the same timeframe (wall-clock approximation)."""
+        with self._state_lock:
+            last = self._state.get("last_entries", {}).get(tf.value)
+        if not last:
+            return False
+        try:
+            last_ts = datetime.fromisoformat(last)
+        except ValueError:
+            return False
+        interval_s = TF_INTERVAL_MINUTES[tf] * 60
+        elapsed = (datetime.now(timezone.utc) - last_ts).total_seconds()
+        return elapsed < COOLDOWN_BARS * interval_s
+
+    # ── Trade execution ────────────────────────────────────────────
+
+    def _execute_trade(self, report: SignalReport, tf: Timeframe) -> None:
+        ts = report.trade_setup
+        if ts is None:
+            return
+
+        # All four TF loops can fire at an aligned boundary (e.g. :00) and
+        # pass _should_execute before any of them has recorded a fill —
+        # re-check under the entry lock so only one order goes out.
+        with self._entry_lock:
+            with self._state_lock:
+                if self._state.get("trades"):
+                    logger.info(
+                        "%s: position opened by a concurrent scan, skipping",
+                        tf.value)
+                    return
+
+            # NO take-profit on the order: the backtested trailing model
+            # exits via SL/TRAIL/TIMEOUT only — a hard TP would cap
+            # winners and diverge the live book from the backtest.
+            try:
+                result = self._executor.place_market_order(
+                    instrument=INSTRUMENT,
+                    direction=ts.direction,
+                    stop_loss=ts.stop_loss,
+                )
+            except OandaAPIError:
+                # The order is never retried (a timeout after a server-side
+                # fill would double-up on retry), but the server MAY have
+                # filled before the error reached us — reconcile so a live
+                # position can't stay untracked until the next restart.
+                logger.exception(
+                    "Failed to place %s order for %s %s — reconciling "
+                    "broker positions",
+                    ts.direction, SYMBOL, tf.value)
+                self._reconcile_positions("order placement error")
+                return
+
+            fill_tx = result.get("orderFillTransaction")
+            if not fill_tx:
+                logger.warning(
+                    "Order not filled for %s %s", SYMBOL, tf.value)
+                return
+
+            trade_opened = fill_tx.get("tradeOpened", {})
+            trade_id = trade_opened.get("tradeID")
+            if not trade_id:
+                # A fill with no tradeOpened means the order interacted
+                # with an existing broker position (tradesClosed /
+                # tradeReduced on a netting account) — state and broker
+                # have diverged. Surface it and re-sync instead of
+                # silently forgetting a real fill.
+                logger.critical(
+                    "Order FILLED but opened no trade (netting close/"
+                    "reduce?) — fill: closed=%s reduced=%s. Reconciling.",
+                    fill_tx.get("tradesClosed"), fill_tx.get("tradeReduced"))
+                self._reconcile_positions("fill without tradeOpened")
+                return
+
+            fill_price = float(fill_tx.get("price", ts.entry_price))
+            risk = abs(fill_price - ts.stop_loss)
+
+            trail_arm = ts.trail_arm_r if ts.trail_arm_r is not None else 0.6
+            trail_dist = ts.trail_dist_r if ts.trail_dist_r is not None else 0.75
+
+            now_iso = datetime.now(timezone.utc).isoformat()
+            trade_state = {
+                "tf": tf.value,
+                "direction": ts.direction,
+                "entry": fill_price,
+                "sl_initial": ts.stop_loss,
+                "current_sl": ts.stop_loss,
+                "tp1": ts.take_profit_1,
+                "tp2": ts.take_profit_2,
+                "risk": risk,
+                "peak": fill_price,
+                "trail_armed": False,
+                "trail_arm_r": trail_arm,
+                "trail_dist_r": trail_dist,
+                "bars_held": 0,
+                # The signal bar is already closed; the first bar this
+                # trade lives through is the one that just started. Guard
+                # so a lagging feed can't trail on the signal bar itself.
+                "signal_bar_ts": report.timestamp,
+                "last_bar_ts": None,
+                "strategy": ts.strategy,
+                "quality": report.signal_quality,
+                "rr": ts.risk_reward,
+                "opened_at": now_iso,
+            }
+
+            with self._state_lock:
+                self._state["trades"][trade_id] = trade_state
+                self._state["last_entries"][tf.value] = now_iso
+                _save_state(self._state)
+
+        logger.info(
+            "TRADE OPENED: %s %s @ %.2f | SL %.2f | TP1 %.2f (advisory) | "
+            "risk $%.2f | RR %.2f | quality %s | strategy %s | TF %s | "
+            "trail arm %.2fR dist %.2fR",
+            ts.direction, SYMBOL, fill_price, ts.stop_loss,
+            ts.take_profit_1,
+            risk, ts.risk_reward, report.signal_quality,
+            ts.strategy, tf.value, trail_arm, trail_dist)
+
+    # ── Trailing stop management ───────────────────────────────────
+
+    def _manage_trailing(self, tf: Timeframe) -> None:
+        # Copy the per-trade dicts too: the loop mutates info (peak,
+        # bars_held, last_bar_ts) and must not touch live state outside
+        # the lock — another thread may be serializing it in _save_state.
+        with self._state_lock:
+            trades = {tid: dict(info)
+                      for tid, info in self._state.get("trades", {}).items()}
+
+        for trade_id, info in trades.items():
+            if info.get("tf") != tf.value:
+                continue
+
+            try:
+                df = self._provider.fetch_ohlcv(SYMBOL, tf)
+            except Exception:
+                logger.exception(
+                    "Failed to fetch candle data for trail update")
+                continue
+
+            if df.empty:
+                logger.warning(
+                    "%s: no candle data for trail update of trade %s",
+                    tf.value, trade_id)
+                continue
+
+            bar_ts = df.index[-1].isoformat()
+
+            # Process each closed bar exactly once. Weekends and feed lag
+            # re-serve the same last bar; without this guard bars_held
+            # inflates (an M5 trade would "time out" over a weekend) and
+            # the peak gets double-counted.
+            if bar_ts == info.get("last_bar_ts"):
+                continue
+            sig_ts = info.get("signal_bar_ts")
+            if sig_ts:
+                try:
+                    if df.index[-1] <= pd.Timestamp(sig_ts):
+                        continue
+                except (ValueError, TypeError):
+                    pass
+            info["last_bar_ts"] = bar_ts
+
+            info["bars_held"] = info.get("bars_held", 0) + 1
+
+            if info["bars_held"] >= self._max_hold_bars:
+                logger.info(
+                    "TIMEOUT: trade %s held %d bars, closing",
+                    trade_id, info["bars_held"])
+                try:
+                    self._executor.close_trade(trade_id)
+                except OandaAPIError:
+                    # Keep tracking — the trade is still live on the
+                    # broker (incl. a 200 whose close order was cancelled,
+                    # e.g. MARKET_HALTED — close_trade raises on no-fill);
+                    # we retry the close at the next bar.
+                    logger.exception(
+                        "Failed to close timed-out trade %s — will retry",
+                        trade_id)
+                    with self._state_lock:
+                        if trade_id in self._state["trades"]:
+                            self._state["trades"][trade_id] = info
+                            _save_state(self._state)
+                    continue
+                with self._state_lock:
+                    self._state["trades"].pop(trade_id, None)
+                    _save_state(self._state)
+                continue
+
+            last_bar = df.iloc[-1]
+            bar_high = float(last_bar["High"])
+            bar_low = float(last_bar["Low"])
+
+            if info.get("risk", 0) <= 0:
+                # Adopted naked position (no broker SL → risk undefined):
+                # never manageable by the trailing model — keep it loud
+                # instead of silently skipping forever.
+                logger.warning(
+                    "Trade %s has risk<=0 (no stop loss) — trailing "
+                    "skipped; set a stop manually", trade_id)
+
+            new_sl = self._compute_trail(info, bar_high, bar_low)
+
+            if new_sl is not None:
+                # Round to instrument precision and track exactly what the
+                # broker holds, so tracked vs broker SL can never drift.
+                new_sl = round(new_sl, self._price_precision)
+                if new_sl != info["current_sl"]:
+                    try:
+                        self._executor.modify_trade_sl(trade_id, new_sl)
+                        info["current_sl"] = new_sl
+                    except OandaAPIError:
+                        logger.exception(
+                            "Failed to update trailing SL for trade %s",
+                            trade_id)
+
+            with self._state_lock:
+                # Write back ONLY if the trade is still tracked — another
+                # loop's _check_closed_trades may have popped it while we
+                # iterated (broker-side close). Re-inserting would create
+                # a ghost that blocks new entries.
+                if trade_id in self._state["trades"]:
+                    self._state["trades"][trade_id] = info
+                    _save_state(self._state)
+
+    def _compute_trail(
+        self, info: Dict, bar_high: float, bar_low: float
+    ) -> Optional[float]:
+        buy = info["direction"] == "BUY"
+        risk = info["risk"]
+        if risk <= 0:
+            return None
+        arm = info.get("trail_arm_r", 0.6)
+        dist = info.get("trail_dist_r", 0.75)
+        if arm <= 0:
+            return None
+
+        fav = bar_high if buy else bar_low
+        if buy:
+            info["peak"] = max(info.get("peak", info["entry"]), fav)
+        else:
+            info["peak"] = min(info.get("peak", info["entry"]), fav)
+
+        exc = ((info["peak"] - info["entry"]) if buy
+               else (info["entry"] - info["peak"])) / risk
+
+        if exc >= arm:
+            if buy:
+                lvl = info["peak"] - dist * risk
+                cur_sl = info.get("current_sl", info["sl_initial"])
+                if lvl > cur_sl:
+                    info["trail_armed"] = True
+                    return lvl
+            else:
+                lvl = info["peak"] + dist * risk
+                cur_sl = info.get("current_sl", info["sl_initial"])
+                if lvl < cur_sl:
+                    info["trail_armed"] = True
+                    return lvl
+
+        return None
+
+    # ── Closed-trade detection ─────────────────────────────────────
+
+    def _check_closed_trades(self) -> None:
+        with self._state_lock:
+            tracked = set(self._state.get("trades", {}).keys())
+
+        if not tracked:
+            return
+
+        try:
+            open_trades = self._executor.get_open_trades(INSTRUMENT)
+        except OandaAPIError:
+            logger.exception("Failed to query open trades")
+            return
+
+        live_ids = {t["id"] for t in open_trades}
+        closed = tracked - live_ids
+
+        for tid in closed:
+            with self._state_lock:
+                info = self._state["trades"].pop(tid, {})
+            entry = info.get("entry", 0)
+            direction = info.get("direction", "?")
+            risk = info.get("risk", 1)
+            sl = info.get("current_sl", 0)
+            was_trailed = info.get("trail_armed", False)
+
+            logger.info(
+                "TRADE CLOSED BY BROKER: %s %s %s | entry %.2f | "
+                "SL was %.2f | risk $%.2f | trailed=%s | bars_held=%d",
+                tid, direction, SYMBOL, entry, sl, risk,
+                was_trailed, info.get("bars_held", 0))
+
+        if closed:
+            with self._state_lock:
+                _save_state(self._state)
+
+# ── Uncaught exception handlers ───────────────────────────────────────
+
+def _excepthook(exc_type, exc_value, exc_tb):
+    if issubclass(exc_type, KeyboardInterrupt):
+        sys.__excepthook__(exc_type, exc_value, exc_tb)
+        return
+    logger.critical(
+        "Uncaught exception", exc_info=(exc_type, exc_value, exc_tb))
+
+def _threading_excepthook(args):
+    logger.critical(
+        "Uncaught exception in thread %s",
+        args.thread.name if args.thread else "?",
+        exc_info=(args.exc_type, args.exc_value, args.exc_traceback))
+
+# ── Entry point ────────────────────────────────────────────────────────
+
+def main() -> None:
+    _setup_logging(
+        os.environ.get("OANDA_LOG_LEVEL") or get_app_config().log_level)
+    sys.excepthook = _excepthook
+    threading.excepthook = _threading_excepthook
+
+    daemon = TradingDaemon()
+    asyncio.run(daemon.run())
+
+
+if __name__ == "__main__":
+    main()

oanda_executor.py

@@ -0,0 +1,230 @@
+"""
+ForexBot2 — Oanda v20 Order Executor
+======================================
+Handles all Oanda account operations: market orders, trade queries,
+SL/TP modification, and trade closure.  Uses raw ``requests`` against
+the Oanda v20 REST API — no extra dependencies.
+"""
+
+import logging
+import time
+from typing import Dict, List, Optional
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+class OandaAPIError(Exception):
+    """Raised when an Oanda API call fails."""
+
+    def __init__(self, message: str, status_code: int = 0, body: str = ""):
+        super().__init__(message)
+        self.status_code = status_code
+        self.body = body
+
+
+class OandaExecutor:
+    """Oanda v20 order execution and trade management."""
+
+    MAX_RETRIES = 1
+    RETRY_DELAY = 2.0
+
+    def __init__(
+        self,
+        account_id: str,
+        api_token: str,
+        base_url: str = "https://api-fxpractice.oanda.com",
+        lot_size: int = 1,
+        price_precision: int = 2,
+    ):
+        self._account_id = account_id
+        self._base_url = base_url.rstrip("/")
+        self._lot_size = lot_size
+        self._price_precision = price_precision
+        self._session = requests.Session()
+        self._session.headers.update({
+            "Authorization": f"Bearer {api_token}",
+            "Accept-Datetime-Format": "RFC3339",
+            "Content-Type": "application/json",
+        })
+
+    # ── Order Placement ────────────────────────────────────────────
+
+    def place_market_order(
+        self,
+        instrument: str,
+        direction: str,
+        stop_loss: float,
+        take_profit: Optional[float] = None,
+        units: Optional[int] = None,
+    ) -> Dict:
+        if units is None:
+            units = self._lot_size
+        signed_units = str(units) if direction.upper() == "BUY" else str(-units)
+
+        order_body = {
+            "order": {
+                "type": "MARKET",
+                "instrument": instrument,
+                "units": signed_units,
+                "timeInForce": "FOK",
+                # The book model is one position total; on a netting
+                # account an opposite-side order would silently CLOSE an
+                # existing (possibly untracked) position instead of opening
+                # the signal trade. OPEN_ONLY makes that case fail loudly
+                # (order cancelled) so the divergence surfaces instead of
+                # executing an unintended close.
+                "positionFill": "OPEN_ONLY",
+                "stopLossOnFill": {
+                    "price": self._fmt_price(stop_loss),
+                },
+            }
+        }
+        if take_profit is not None:
+            order_body["order"]["takeProfitOnFill"] = {
+                "price": self._fmt_price(take_profit),
+            }
+
+        url = f"{self._base_url}/v3/accounts/{self._account_id}/orders"
+        # Never retry order placement: a timeout after the server already
+        # filled would open a SECOND position on retry. Fail loudly instead;
+        # the next candle boundary re-evaluates.
+        resp = self._request_with_retry("POST", url, json=order_body, retries=0)
+        data = self._parse_json(resp)
+
+        fill_tx = data.get("orderFillTransaction")
+        if fill_tx:
+            trade_id = fill_tx.get("tradeOpened", {}).get("tradeID", "unknown")
+            fill_price = float(fill_tx.get("price", 0))
+            logger.info(
+                "ORDER FILLED: %s %s %s units @ %.2f — trade %s, SL %.2f%s",
+                direction.upper(), instrument, signed_units, fill_price,
+                trade_id, stop_loss,
+                f", TP {take_profit:.2f}" if take_profit else "")
+        else:
+            cancel_tx = data.get("orderCancelTransaction", {})
+            reason = cancel_tx.get("reason", "unknown")
+            logger.warning(
+                "ORDER NOT FILLED: %s %s — reason: %s",
+                direction.upper(), instrument, reason)
+
+        return data
+
+    # ── Trade Queries ──────────────────────────────────────────────
+
+    def get_open_trades(self, instrument: Optional[str] = None) -> List[Dict]:
+        url = f"{self._base_url}/v3/accounts/{self._account_id}/openTrades"
+        resp = self._request_with_retry("GET", url)
+        trades = self._parse_json(resp).get("trades", [])
+        if instrument:
+            trades = [t for t in trades if t.get("instrument") == instrument]
+        return trades
+
+    def get_trade(self, trade_id: str) -> Dict:
+        url = (f"{self._base_url}/v3/accounts/{self._account_id}"
+               f"/trades/{trade_id}")
+        resp = self._request_with_retry("GET", url)
+        return self._parse_json(resp).get("trade", {})
+
+    def get_account_summary(self) -> Dict:
+        url = f"{self._base_url}/v3/accounts/{self._account_id}/summary"
+        resp = self._request_with_retry("GET", url)
+        account = self._parse_json(resp).get("account", {})
+        logger.info(
+            "Account: balance=%.2f, NAV=%.2f, unrealizedPL=%.2f, openTrades=%s",
+            float(account.get("balance", 0)),
+            float(account.get("NAV", 0)),
+            float(account.get("unrealizedPL", 0)),
+            account.get("openTradeCount", 0))
+        return account
+
+    # ── Trade Modification ─────────────────────────────────────────
+
+    def modify_trade_sl(self, trade_id: str, new_sl: float) -> Dict:
+        url = (f"{self._base_url}/v3/accounts/{self._account_id}"
+               f"/trades/{trade_id}/orders")
+        body = {"stopLoss": {"price": self._fmt_price(new_sl)}}
+        resp = self._request_with_retry("PUT", url, json=body)
+        logger.info(
+            "TRAIL UPDATE: trade %s — new SL %.2f", trade_id, new_sl)
+        return self._parse_json(resp)
+
+    def modify_trade_tp(self, trade_id: str, new_tp: float) -> Dict:
+        url = (f"{self._base_url}/v3/accounts/{self._account_id}"
+               f"/trades/{trade_id}/orders")
+        body = {"takeProfit": {"price": self._fmt_price(new_tp)}}
+        resp = self._request_with_retry("PUT", url, json=body)
+        logger.info("TP UPDATE: trade %s — new TP %.2f", trade_id, new_tp)
+        return self._parse_json(resp)
+
+    def close_trade(self, trade_id: str) -> Dict:
+        url = (f"{self._base_url}/v3/accounts/{self._account_id}"
+               f"/trades/{trade_id}/close")
+        resp = self._request_with_retry("PUT", url, json={"units": "ALL"})
+        data = self._parse_json(resp)
+        close_tx = data.get("orderFillTransaction")
+        if not close_tx:
+            # A 200 whose close order was created but NOT filled (e.g.
+            # cancelled with MARKET_HALTED during the settlement pause)
+            # means the position is STILL OPEN. Raising keeps the caller
+            # tracking it and retrying — silently returning here would
+            # orphan a live position.
+            reason = data.get("orderCancelTransaction", {}).get(
+                "reason", "no orderFillTransaction in response")
+            raise OandaAPIError(
+                f"Close of trade {trade_id} did not fill: {reason}",
+                resp.status_code, str(data)[:300])
+        close_price = float(close_tx.get("price", 0))
+        pl = float(close_tx.get("pl", 0))
+        logger.info(
+            "TRADE CLOSED: trade %s @ %.2f — PL %.2f",
+            trade_id, close_price, pl)
+        return data
+
+    # ── Internals ──────────────────────────────────────────────────
+
+    @staticmethod
+    def _parse_json(resp: requests.Response) -> Dict:
+        try:
+            return resp.json()
+        except ValueError as e:
+            raise OandaAPIError(
+                f"Oanda returned a non-JSON response (HTTP {resp.status_code})",
+                resp.status_code, resp.text[:300]) from e
+
+    def _fmt_price(self, price: float) -> str:
+        # Oanda rejects prices with more decimals than the instrument's
+        # displayPrecision (XAU_USD: sub-cent precision is invalid).
+        return f"{price:.{self._price_precision}f}"
+
+    def _request_with_retry(
+        self, method: str, url: str, retries: Optional[int] = None, **kwargs
+    ) -> requests.Response:
+        if retries is None:
+            retries = self.MAX_RETRIES
+        last_err = None
+        for attempt in range(1 + retries):
+            try:
+                resp = self._session.request(method, url, timeout=15, **kwargs)
+                if resp.status_code >= 500:
+                    raise OandaAPIError(
+                        f"Oanda server error {resp.status_code}",
+                        resp.status_code, resp.text)
+                if resp.status_code >= 400:
+                    raise OandaAPIError(
+                        f"Oanda API error {resp.status_code}: {resp.text}",
+                        resp.status_code, resp.text)
+                return resp
+            except (requests.ConnectionError, requests.Timeout,
+                    OandaAPIError) as e:
+                last_err = e
+                if isinstance(e, OandaAPIError) and 400 <= e.status_code < 500:
+                    raise
+                if attempt < retries:
+                    logger.warning(
+                        "Oanda request failed (attempt %d/%d): %s — retrying in %.0fs",
+                        attempt + 1, 1 + retries, e,
+                        self.RETRY_DELAY)
+                    time.sleep(self.RETRY_DELAY)
+        raise OandaAPIError(f"Oanda request failed after retries: {last_err}")

pair_signals/__init__.py

@@ -10,7 +10,7 @@ then register it in this file.
 
 from typing import Optional, List, Dict, Callable
 
-from config import Timeframe
+from config import PlatformConfig, Timeframe
 from signal_aggregator import SignalAggregator, SignalReport
 from providers import BaseDataProvider
 
@@ -21,12 +21,15 @@ def register_pair(symbol: str, config_factory: Callable):
     PAIR_PRODUCERS[symbol] = config_factory
 
 
+def get_supported_pairs() -> List[str]:
+    return list(PAIR_PRODUCERS.keys())
+
+
 def produce_signal(
     symbol: str,
     primary_tf: Timeframe,
     data_provider: BaseDataProvider,
     confirmation_tfs: Optional[List[Timeframe]] = None,
-    log_source: str = "dashboard",
 ) -> SignalReport:
     config_factory = PAIR_PRODUCERS.get(symbol)
     if config_factory is None:
@@ -38,7 +41,7 @@ def produce_signal(
     config = config_factory(primary_tf, confirmation_tfs)
     aggregator = SignalAggregator(config)
     aggregator.data_provider = data_provider
-    return aggregator.analyze_pair(symbol=symbol, log_source=log_source)
+    return aggregator.analyze_pair(symbol=symbol)
 
 
 from pair_signals import xauusd as _xau_mod

pair_signals/xauusd.py

@@ -141,49 +141,6 @@ validated price of riding trends, exits confirmed NOT the loss
 driver: exit-management accounts for 1-2% of lost R vs 50%
 wrong-direction / 49% entry-timing).
 
-M5 override 6 — SELL block in transitioning regime (validated
-2026-09-03, signal-log campaign, `backtest/research/
-signal_log_campaign_2026-09.md`, $0.65 round trip, same CPU harness
-as its control): sell_blocked_regimes=["transitioning"]. Live
-evidence: M5 SELLs in transitioning were -7.9R/12mo PF 0.21 and
--5.2R/5yr PF 0.49 in the pinned books; the live signal log showed the
-same pocket -7.9R/PF 0.56 over 3 weeks. Blocking: 12mo (2025-08-14 ->
-2026-08-14) 557 trades +112.21R PF 1.56 (halves +55.9/+56.4, PF
-1.54/1.58) vs control 575/+104.34R PF 1.50; 5yr (2021-08-01 ->
-2026-08-04) 889/+133.7R PF 1.41 vs 911/+128.4R PF 1.39; OOS
-2024-06-01 -> 2025-03-30 unchanged (86/+9.45R PF 1.33 — no such
-trades in that window). Removed trades: 18 in 12mo (-7.9R), 22 in
-5yr (-5.3R).
-
-H1 override — all SELL entries blocked (validated 2026-09-03, same
-campaign): sell_min_quality=[] (gate 6e in signal_aggregator.py
-downgrades any actionable-grade SELL whose quality is not in this
-list to D, so an empty list downgrades every H1 short). H1 shorts
-were a structural loser: -16.7R over the 5yr control window and
--11.5R OOS. Blocking: 12mo (2025-08-04 -> 2026-08-04) 91/+22.2R
-PF 1.79 (halves +20.6/+1.6) vs control 138/+24.0R PF 1.55 (12mo cost
--1.8R); 5yr 335/+49.1R PF 1.46 vs 515/+32.4R PF 1.18; OOS 78/+16.7R
-PF 1.74 vs 104/+5.1R PF 1.13. New reference books:
-backtest/oanda_h1_{12mo,5yr,oos}_065_20260903.csv,
-backtest/oanda_m5_12mo_065_nopb_20260903.csv,
-backtest/oanda_m5_5yr_065_nopb_20260903.csv.
-
-Rejected/parked in the same campaign (see the memo for the full
-table): M15 sell_min_quality=["B"] (5yr -8.9R, max DD 35.8 -> 41.4R
-— REJECTED); Bollinger weight 0 in trending regimes on M5, M15, H1
-(M5 -20.4R/12mo, -15.5R/5yr; M15 -24.0R/12mo — REJECTED across the
-board); divergence disabled on M15 (R improves +7.4/+11.9/+13.5R
-across windows but 5yr max DD 35.8 -> 47.4R with PF flat — DECLINED,
-owner call); divergence disabled on H1 (+4.6/+8.1/+2.4R, PF
-1.62/1.20/1.17) — PARKED: it passes rule 1 in combination with the
-H1 no-short override (b12_h1_nosell_nodiv: 12mo 100/+23.1R PF 1.75,
-halves +21.8/+1.3; 5yr 364/+58.7R PF 1.52; OOS 88/+21.9R PF 1.92 —
-better than the no-shorts override alone on both 5yr and OOS), but
-shipping it would route through technical.disabled_indicators, which
-CLAUDE.md rule 8 reserves for experiments; owner decision, recommended
-vehicle is an H1-only regime_weights_override with divergence weight
-0 plus a fresh validation run.
-
 M5 override 5 — EMA partial-alignment squash (validated 2026-08-04,
 Oanda bars, $0.65 real spread): ema_partial_neutral True on M5 —
 partial 4-EMA ribbon alignment (2-3 of 4 checks) scores NEUTRAL
@@ -259,9 +216,6 @@ def make_config(
         # Wider trailing distance on M5 only — see module docstring.
         config.signals.trail_arm_r = 0.60
         config.signals.trail_dist_r = 0.85
-        # Block SELL entries while the regime is "transitioning" on M5
-        # only — see module docstring (M5 override 6).
-        config.xau_strategy.sell_blocked_regimes = ["transitioning"]
     if primary_tf == Timeframe.M15:
         # 1R minimum target on M15 only — see module docstring.
         config.signals.tp_floor_rr = 1.0
@@ -272,10 +226,4 @@ def make_config(
         # Later trailing-stop arm on M15 only — see module docstring.
         config.signals.trail_arm_r = 0.70
         config.signals.trail_dist_r = 0.75
-    if primary_tf == Timeframe.H1:
-        # Block every H1 SELL — the SELL quality gate (6e in
-        # signal_aggregator.py) downgrades any actionable-grade short
-        # to D when its grade is not in this list; an empty list means
-        # no grade qualifies. H1-only — see module docstring.
-        config.xau_strategy.sell_min_quality = []
     return config

position_sizing.py

@@ -1,611 +0,0 @@
-"""
-ForexBot2 — Position sizing (stake-multiplier overlay)
-======================================================
-
-Single source of truth for **decision-time features** and the resulting
-**stake multiplier**. Everything here is designed to be called from
-inside the signal path so that live and backtest see byte-identical
-inputs (rule 4: live == backtest parity).
-
-Design contract
----------------
-* **Bounded lookbacks.** No feature may read further back than the live
-  fetch depth of its timeframe (5000 bars on M5/M15, 1460 on H1, 380 on
-  H4). The primary frame is hard-capped at ``MAX_LOOKBACK_BARS`` rows,
-  confirmation frames at 60 rows, and the burst scan at 200 rows, so a
-  feature computed on a full backtest frame equals the same feature
-  computed on the shorter live tail.
-* **Closed bars only.** The frames handed in are exactly the frames
-  ``analyze_from_dataframes`` received: UTC tz-aware, oldest-first,
-  capitalized OHLCV columns, forming candle already dropped. The last
-  row is the decision bar; nothing peeks past it.
-* **Never raises into a scan.** ``build_features`` and
-  ``stake_multiplier`` swallow every exception (logged at DEBUG) and
-  fall back to neutral defaults. A sizing failure can never break a
-  signal.
-* **Advisory today.** ``stake_multiplier`` returns ``1.0`` in every case
-  at this stage — artifact evaluation lands in a later step. The guard
-  order is already implemented so the wiring is testable now.
-
-Dependencies are deliberately limited to numpy/pandas plus ``config``
-(the ``Timeframe`` enum): this module ships live and must not drag the
-provider/news stack into any process that only wants features.
-
-VERDICT 2026-09-06 (backtest/research/sizing_campaign_2026-09.md): the walk-forward
-stake-multiplier model FAILED the pre-registered acceptance rule on both M5 and
-M15 (M5 a clean null; M15 lift inside the label-shuffle null and explained by
-BUY-vs-SELL). NO artifact ships and nothing reads a multiplier live. This module
-is research plumbing: `build_features` feeds the backtester's --export-features
-flag; `stake_multiplier` returns 1.0 unless a validated artifact is ever added
-under a NEW pre-registration.
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import math
-from pathlib import Path
-from typing import Dict, Optional, Sequence, Tuple
-
-import numpy as np
-import pandas as pd
-
-from config import SignalStrength, Timeframe
-
-logger = logging.getLogger(__name__)
-
-
-#: Bump whenever FEATURE_ORDER or any feature's definition changes.
-#: A trained artifact carrying a different value is ignored.
-FEATURE_VERSION = 1
-
-#: Hard cap on any primary-timeframe window used in this module.
-MAX_LOOKBACK_BARS = 500
-
-#: Bars of each confirmation timeframe used by the cross-TF features.
-CONFIRM_LOOKBACK_BARS = 60
-
-#: Window for the "bars since the last volatility burst" feature. Also
-#: the sentinel returned when no burst is present in the window.
-BURST_LOOKBACK_BARS = 200
-
-#: Burst threshold: bar range >= this multiple of the PRIOR bar's ATR.
-BURST_ATR_MULTIPLE = 3.0
-
-#: Candle duration in seconds. Deliberate local copy of
-#: ``providers.base.BaseDataProvider.CANDLE_SECONDS`` — importing the
-#: providers package here would pull requests/twelvedata into every
-#: feature-only consumer. A test asserts the copies agree.
-CANDLE_SECONDS: Dict[Timeframe, int] = {
-    Timeframe.M5: 300,
-    Timeframe.M15: 900,
-    Timeframe.H1: 3600,
-    Timeframe.H4: 14400,
-    Timeframe.D1: 86400,
-}
-
-#: Grade -> ordinal used by ``f_quality_rank`` (D/F both fall to 0).
-_QUALITY_RANK = {"A+": 4.0, "A": 3.0, "B": 2.0, "C": 1.0}
-
-_BULLISH = (SignalStrength.BUY, SignalStrength.STRONG_BUY)
-_BEARISH = (SignalStrength.SELL, SignalStrength.STRONG_SELL)
-
-
-FEATURE_ORDER: Tuple[str, ...] = (
-    # ── Time ──────────────────────────────────────────────────────
-    "f_hour_utc",
-    "f_hour_sin",
-    "f_hour_cos",
-    "f_weekday",
-    "f_sess_asian",
-    "f_sess_london",
-    "f_sess_overlap",
-    "f_sess_ny",
-    "f_minutes_to_hour",
-    "f_day_of_month",
-    "f_first_friday",
-    # ── Volatility / setup geometry ───────────────────────────────
-    "f_atr",
-    "f_atr_pct",
-    "f_atr_rel_500",
-    "f_range_pctile_500",
-    "f_bar_range_atr",
-    "f_close_pos_in_bar",
-    "f_risk_price",
-    "f_sl_atr",
-    "f_tp1_atr",
-    "f_rr_planned",
-    # ── Exhaustion block ──────────────────────────────────────────
-    "f_swing_dist_atr",
-    "f_ema21_dist_atr",
-    "f_ema200_dist_atr",
-    "f_ema200_dist_pct",
-    "f_exhaustion_count",
-    "f_tag_exhaustion",
-    "f_tag_no_pullback",
-    "f_tag_weak_pullback",
-    "f_tag_rsi_extended",
-    "f_tag_bb_overextended",
-    "f_tag_ema200_near",
-    "f_tag_ema200_far",
-    "f_penalty_total",
-    "f_momentum_fading",
-    # ── Signal quality ────────────────────────────────────────────
-    "f_score_dir",
-    "f_confidence",
-    "f_quality_rank",
-    "f_ind_agree_frac",
-    "f_ind_oppose_frac",
-    "f_is_buy",
-    "f_strat_atr",
-    "f_strat_pb",
-    "f_strat_cp",
-    "f_strat_generic",
-    # ── Multi-timeframe ───────────────────────────────────────────
-    "f_mtf_primary",
-    "f_mtf_c1",
-    "f_mtf_c2",
-    "f_mtf_agreement",
-    # ── Primary-TF indicators ─────────────────────────────────────
-    "f_rsi",
-    "f_rsi_dir",
-    "f_bb_pos",
-    "f_bb_pos_dir",
-    "f_adx",
-    "f_kc_ext_atr",
-    "f_roc_dir",
-    "f_macd_hist_dir",
-    # ── Price path ────────────────────────────────────────────────
-    "f_ret_6_atr",
-    "f_ret_12_atr",
-    "f_ret_24_atr",
-    "f_ret_48_atr",
-    "f_consec_closes_dir",
-    "f_bars_since_burst",
-    # ── Cross-timeframe extension ─────────────────────────────────
-    "f_h1_ext_atr",
-    "f_c_other_ext_atr",
-)
-
-#: Value each feature falls back to when it cannot be computed. Only
-#: ``f_bars_since_burst`` has a non-zero sentinel (its "none in window"
-#: cap); everything else degrades to a neutral 0.0.
-_DEFAULTS: Dict[str, float] = {name: 0.0 for name in FEATURE_ORDER}
-_DEFAULTS["f_bars_since_burst"] = float(BURST_LOOKBACK_BARS)
-
-#: Penalty tags recorded by SignalAggregator._compute_final_signal that
-#: this module exposes as one-hot features.
-_TAG_FEATURES = (
-    ("f_tag_exhaustion", "exhaustion"),
-    ("f_tag_no_pullback", "no_pullback"),
-    ("f_tag_weak_pullback", "weak_pullback"),
-    ("f_tag_rsi_extended", "rsi_extended"),
-    ("f_tag_bb_overextended", "bb_overextended"),
-    ("f_tag_ema200_near", "ema200_near"),
-    ("f_tag_ema200_far", "ema200_far"),
-)
-
-
-# ══════════════════════════════════════════════════════════════════
-#  Small helpers
-# ══════════════════════════════════════════════════════════════════
-
-def _finite(value, default: float = 0.0) -> float:
-    """Coerce to a finite Python float, falling back to ``default``."""
-    try:
-        out = float(value)
-    except (TypeError, ValueError):
-        return float(default)
-    if not math.isfinite(out):
-        return float(default)
-    return out
-
-
-def wilder_atr(df: pd.DataFrame, period: int = 14) -> pd.Series:
-    """Wilder ATR, identical convention to ``engine._compute_atr``.
-
-    True Range, then an EWM with ``alpha = 1/period``,
-    ``min_periods = period`` and ``adjust=False``.
-    """
-    high, low, close = df["High"], df["Low"], df["Close"]
-    tr1 = high - low
-    tr2 = (high - close.shift()).abs()
-    tr3 = (low - close.shift()).abs()
-    tr = pd.concat([tr1, tr2, tr3], axis=1).max(axis=1)
-    return tr.ewm(alpha=1.0 / period, min_periods=period, adjust=False).mean()
-
-
-def decision_time(primary_df: pd.DataFrame, tf: Timeframe) -> pd.Timestamp:
-    """The clock every gate uses: last CLOSED bar label + its duration."""
-    last = pd.Timestamp(primary_df.index[-1])
-    return last + pd.Timedelta(seconds=CANDLE_SECONDS.get(tf, 3600))
-
-
-def session_label(hour_utc: int) -> str:
-    """Trading-session bucket for a UTC hour."""
-    h = int(hour_utc)
-    if 3 <= h <= 6:
-        return "asian"
-    if 7 <= h <= 12:
-        return "london"
-    if 13 <= h <= 16:
-        return "overlap"
-    if 17 <= h <= 22:
-        return "ny"
-    return "off"
-
-
-# ══════════════════════════════════════════════════════════════════
-#  Artifact plumbing
-# ══════════════════════════════════════════════════════════════════
-
-#: path -> (mtime, artifact-or-None)
-_ARTIFACT_CACHE: Dict[str, Tuple[float, Optional[dict]]] = {}
-
-
-def artifact_path_for(primary_tf: Timeframe, base_dir) -> Path:
-    """Where the trained artifact for one primary timeframe lives."""
-    return Path(base_dir) / f"xauusd_{primary_tf.value}.json"
-
-
-def load_artifact(path) -> Optional[dict]:
-    """Load a sizing artifact (JSON object), cached on (path, mtime).
-
-    Returns ``None`` when the file is missing, unreadable, not JSON, or
-    not a JSON object. Never raises.
-    """
-    try:
-        p = Path(path)
-        mtime = p.stat().st_mtime
-    except Exception:
-        return None
-
-    key = str(p)
-    cached = _ARTIFACT_CACHE.get(key)
-    if cached is not None and cached[0] == mtime:
-        return cached[1]
-
-    data: Optional[dict]
-    try:
-        with p.open("r", encoding="utf-8") as fh:
-            loaded = json.load(fh)
-        data = loaded if isinstance(loaded, dict) else None
-    except Exception:
-        logger.debug("position_sizing: artifact %s is unreadable", key,
-                     exc_info=True)
-        data = None
-
-    _ARTIFACT_CACHE[key] = (mtime, data)
-    return data
-
-
-# ══════════════════════════════════════════════════════════════════
-#  Feature construction
-# ══════════════════════════════════════════════════════════════════
-
-def _direction(setup, report) -> float:
-    """+1.0 for a long, -1.0 for a short — from the TRADE, not the score."""
-    direction = getattr(setup, "direction", None)
-    if direction is None:
-        return 1.0 if _finite(getattr(report, "final_score", 0.0)) >= 0 else -1.0
-    return 1.0 if str(direction).upper() == "BUY" else -1.0
-
-
-def _ext_vs_sma(df: pd.DataFrame, d: float) -> float:
-    """Direction-signed distance from the 20-bar SMA in ATR units.
-
-    Computed on the LAST ``CONFIRM_LOOKBACK_BARS`` rows only, so the
-    value is identical on a full history and on its live-depth tail.
-    """
-    if df is None or len(df) < 21:
-        return 0.0
-    tail = df.iloc[-CONFIRM_LOOKBACK_BARS:]
-    sma = tail["Close"].rolling(20).mean().iloc[-1]
-    atr = wilder_atr(tail, 14).iloc[-1]
-    atr_f = _finite(atr, 0.0)
-    if atr_f <= 0:
-        return 0.0
-    return d * (_finite(tail["Close"].iloc[-1]) - _finite(sma)) / atr_f
-
-
-def _fill_time(out: Dict[str, float], dt: pd.Timestamp) -> None:
-    hour = int(dt.hour)
-    out["f_hour_utc"] = float(hour)
-    out["f_hour_sin"] = float(math.sin(2.0 * math.pi * hour / 24.0))
-    out["f_hour_cos"] = float(math.cos(2.0 * math.pi * hour / 24.0))
-    weekday = int(dt.weekday())
-    out["f_weekday"] = float(weekday)
-    label = session_label(hour)
-    out["f_sess_asian"] = 1.0 if label == "asian" else 0.0
-    out["f_sess_london"] = 1.0 if label == "london" else 0.0
-    out["f_sess_overlap"] = 1.0 if label == "overlap" else 0.0
-    out["f_sess_ny"] = 1.0 if label == "ny" else 0.0
-    out["f_minutes_to_hour"] = float((60 - int(dt.minute)) % 60)
-    out["f_day_of_month"] = float(int(dt.day))
-    out["f_first_friday"] = 1.0 if (weekday == 4 and int(dt.day) <= 7) else 0.0
-
-
-def _fill_geometry(out: Dict[str, float], P: pd.DataFrame, setup,
-                   atr: float, norm, d: float) -> None:
-    out["f_atr"] = max(0.0, _finite(atr))
-    ranges = (P["High"] - P["Low"]).astype(float)
-    last_range = _finite(ranges.iloc[-1])
-    mean_range = _finite(ranges.mean())
-    out["f_atr_rel_500"] = (_finite(atr) / mean_range) if mean_range > 0 else 0.0
-    n = int(len(ranges))
-    out["f_range_pctile_500"] = (
-        float((ranges.to_numpy() < last_range).sum()) / n if n else 0.0)
-    out["f_bar_range_atr"] = norm(last_range)
-
-    high = _finite(P["High"].iloc[-1])
-    low = _finite(P["Low"].iloc[-1])
-    close = _finite(P["Close"].iloc[-1])
-    span = high - low
-    if span > 0:
-        out["f_close_pos_in_bar"] = (
-            (close - low) / span if d > 0 else (high - close) / span)
-    else:
-        out["f_close_pos_in_bar"] = 0.5
-
-    entry = _finite(getattr(setup, "entry_price", None))
-    stop = _finite(getattr(setup, "stop_loss", None))
-    tp1 = _finite(getattr(setup, "take_profit_1", None))
-    risk = abs(entry - stop)
-    out["f_risk_price"] = risk
-    out["f_sl_atr"] = norm(risk)
-    out["f_tp1_atr"] = norm(abs(tp1 - entry))
-    out["f_rr_planned"] = _finite(getattr(setup, "risk_reward", None))
-
-
-def _fill_exhaustion(out: Dict[str, float], report, KL: Dict[str, float],
-                     px: float, d: float, norm) -> None:
-    if d > 0:
-        swings = [KL[k] for k in ("swing_low_1", "swing_low_2", "swing_low_3")
-                  if KL.get(k) is not None]
-        out["f_swing_dist_atr"] = norm(px - min(swings)) if swings else 0.0
-    else:
-        swings = [KL[k] for k in ("swing_high_1", "swing_high_2", "swing_high_3")
-                  if KL.get(k) is not None]
-        out["f_swing_dist_atr"] = norm(max(swings) - px) if swings else 0.0
-
-    ema21 = KL.get("ema_medium")
-    out["f_ema21_dist_atr"] = (
-        norm(d * (px - _finite(ema21))) if ema21 is not None else 0.0)
-
-    ema200 = KL.get("ema_trend")
-    ema200_f = _finite(ema200, 0.0) if ema200 is not None else 0.0
-    if ema200 is not None and ema200_f != 0.0:
-        out["f_ema200_dist_atr"] = norm(d * (px - ema200_f))
-        out["f_ema200_dist_pct"] = d * (px - ema200_f) / ema200_f * 100.0
-    else:
-        out["f_ema200_dist_atr"] = 0.0
-        out["f_ema200_dist_pct"] = 0.0
-
-    out["f_exhaustion_count"] = _finite(getattr(report, "exhaustion_count", 0))
-    tags = getattr(report, "penalty_tags", None) or {}
-    for feature, tag in _TAG_FEATURES:
-        out[feature] = 1.0 if tag in tags else 0.0
-    out["f_penalty_total"] = _finite(getattr(report, "penalty_total", 0.0))
-    out["f_momentum_fading"] = 1.0 if getattr(report, "momentum_fading", False) else 0.0
-
-
-def _fill_quality(out: Dict[str, float], report, setup, IND: dict,
-                  d: float) -> None:
-    out["f_score_dir"] = d * _finite(getattr(report, "final_score", 0.0))
-    out["f_confidence"] = _finite(getattr(report, "final_confidence", 0.0))
-    out["f_quality_rank"] = _QUALITY_RANK.get(
-        getattr(report, "signal_quality", ""), 0.0)
-
-    agree = oppose = total = 0
-    for name, ind in (IND or {}).items():
-        if str(name).startswith("_"):
-            continue
-        if _finite(getattr(ind, "weight", 0.0)) == 0.0:
-            continue
-        total += 1
-        signal = getattr(ind, "signal", None)
-        if d > 0:
-            if signal in _BULLISH:
-                agree += 1
-            elif signal in _BEARISH:
-                oppose += 1
-        else:
-            if signal in _BEARISH:
-                agree += 1
-            elif signal in _BULLISH:
-                oppose += 1
-    out["f_ind_agree_frac"] = (agree / total) if total else 0.0
-    out["f_ind_oppose_frac"] = (oppose / total) if total else 0.0
-
-    out["f_is_buy"] = 1.0 if d > 0 else 0.0
-    strategy = str(getattr(setup, "strategy", "") or "")
-    # The ATR path stamps "XAU_ATR"; "XAU" is accepted for legacy setups.
-    out["f_strat_atr"] = 1.0 if strategy in ("XAU_ATR", "XAU") else 0.0
-    out["f_strat_pb"] = 1.0 if strategy == "XAU_PB" else 0.0
-    out["f_strat_cp"] = 1.0 if strategy == "XAU_CP" else 0.0
-    out["f_strat_generic"] = 1.0 if strategy == "GENERIC" else 0.0
-
-
-def _fill_mtf(out: Dict[str, float], report, primary_tf: Timeframe,
-              confirmation_tfs: Sequence[Timeframe], d: float) -> None:
-    scores = getattr(report, "mtf_scores", None) or {}
-    out["f_mtf_primary"] = d * _finite(scores.get(primary_tf.value, 0.0))
-    confirms = list(confirmation_tfs or [])
-    for idx, key in ((0, "f_mtf_c1"), (1, "f_mtf_c2")):
-        if idx < len(confirms):
-            out[key] = d * _finite(scores.get(confirms[idx].value, 0.0))
-        else:
-            out[key] = 0.0
-    out["f_mtf_agreement"] = 1.0 if getattr(report, "mtf_agreement", False) else 0.0
-
-
-def _fill_indicators(out: Dict[str, float], IND: dict, d: float, norm) -> None:
-    IND = IND or {}
-
-    def _val(name, default):
-        ind = IND.get(name)
-        return _finite(getattr(ind, "value", None), default) if ind else default
-
-    rsi = _val("rsi", 50.0)
-    out["f_rsi"] = rsi
-    out["f_rsi_dir"] = d * (rsi - 50.0)
-    bb_pos = _val("bollinger", 0.5)          # bollinger.value = bb_position
-    out["f_bb_pos"] = bb_pos
-    out["f_bb_pos_dir"] = d * (bb_pos - 0.5)
-    out["f_adx"] = _val("adx", 0.0)
-    # keltner.value = close - kc_mid (signed extension from the mid line)
-    out["f_kc_ext_atr"] = norm(d * _val("keltner", 0.0))
-    out["f_roc_dir"] = d * _val("roc", 0.0)
-    # macd.value = histogram
-    out["f_macd_hist_dir"] = d * _val("macd", 0.0)
-
-
-def _fill_path(out: Dict[str, float], P: pd.DataFrame, d: float, norm) -> None:
-    close = P["Close"].astype(float).to_numpy()
-    n = len(close)
-    for lag, key in ((6, "f_ret_6_atr"), (12, "f_ret_12_atr"),
-                     (24, "f_ret_24_atr"), (48, "f_ret_48_atr")):
-        out[key] = norm(d * (close[-1] - close[-1 - lag])) if n >= lag + 1 else 0.0
-
-    run = 0.0
-    if n >= 2:
-        diffs = np.diff(close)
-        last = diffs[-1]
-        if last > 0:
-            sign = 1.0
-        elif last < 0:
-            sign = -1.0
-        else:
-            sign = 0.0
-        if sign != 0.0:
-            count = 0
-            for value in diffs[::-1]:
-                if (value > 0 and sign > 0) or (value < 0 and sign < 0):
-                    count += 1
-                else:
-                    break
-            run = sign * count
-    out["f_consec_closes_dir"] = max(-20.0, min(20.0, d * run))
-
-    out["f_bars_since_burst"] = float(BURST_LOOKBACK_BARS)
-    prev_atr = wilder_atr(P, 14).shift(1).to_numpy(dtype=float)
-    ranges = (P["High"] - P["Low"]).astype(float).to_numpy()
-    start = max(0, n - BURST_LOOKBACK_BARS)
-    for i in range(n - 1, start - 1, -1):
-        threshold = prev_atr[i]
-        if not math.isfinite(threshold) or threshold <= 0:
-            continue
-        if ranges[i] >= BURST_ATR_MULTIPLE * threshold:
-            out["f_bars_since_burst"] = float((n - 1) - i)
-            break
-
-
-def _fill_cross_tf(out: Dict[str, float], dfs: Dict[Timeframe, pd.DataFrame],
-                   confirmation_tfs: Sequence[Timeframe], d: float) -> None:
-    out["f_h1_ext_atr"] = _ext_vs_sma(dfs.get(Timeframe.H1), d)
-    out["f_c_other_ext_atr"] = 0.0
-    for tf in (confirmation_tfs or []):
-        if tf is Timeframe.H1:
-            continue
-        df = dfs.get(tf)
-        if df is None or len(df) < 21:
-            continue
-        out["f_c_other_ext_atr"] = _ext_vs_sma(df, d)
-        break
-
-
-def build_features(
-    report,
-    setup,
-    dfs: Dict[Timeframe, pd.DataFrame],
-    primary_tf: Timeframe,
-    confirmation_tfs: Sequence[Timeframe],
-) -> Dict[str, float]:
-    """Decision-time feature vector for one signal.
-
-    Returns a dict whose keys are exactly ``FEATURE_ORDER`` (in order),
-    every value a finite Python float. Degenerate inputs fall back to
-    the neutral defaults instead of raising — this runs inside a scan.
-    """
-    out: Dict[str, float] = dict(_DEFAULTS)
-
-    try:
-        P = dfs.get(primary_tf)
-        if P is None or len(P) == 0:
-            return {k: out[k] for k in FEATURE_ORDER}
-        P = P.iloc[-MAX_LOOKBACK_BARS:]
-
-        d = _direction(setup, report)
-        primary = getattr(report, "primary_analysis", None)
-        KL = dict(getattr(primary, "key_levels", None) or {})
-        IND = getattr(primary, "indicators", None) or {}
-
-        atr_raw = KL.get("atr")
-        atr = _finite(atr_raw, 0.0)
-        atr_ok = atr > 0
-        atr_div = atr if atr_ok else 1e-9
-
-        def norm(value: float) -> float:
-            """ATR-normalise, or 0.0 when ATR is missing/non-positive."""
-            return (_finite(value) / atr_div) if atr_ok else 0.0
-
-        px = _finite(P["Close"].iloc[-1])
-        out["f_atr_pct"] = _finite(KL.get("atr_pct"), 0.0)
-
-        for step, args in (
-            (_fill_time, (out, decision_time(P, primary_tf))),
-            (_fill_geometry, (out, P, setup, atr, norm, d)),
-            (_fill_exhaustion, (out, report, KL, px, d, norm)),
-            (_fill_quality, (out, report, setup, IND, d)),
-            (_fill_mtf, (out, report, primary_tf, confirmation_tfs, d)),
-            (_fill_indicators, (out, IND, d, norm)),
-            (_fill_path, (out, P, d, norm)),
-            (_fill_cross_tf, (out, dfs, confirmation_tfs, d)),
-        ):
-            try:
-                step(*args)
-            except Exception:
-                logger.debug("position_sizing: %s failed", step.__name__,
-                             exc_info=True)
-    except Exception:
-        logger.debug("position_sizing.build_features failed", exc_info=True)
-
-    return {k: _finite(out.get(k), _DEFAULTS[k]) for k in FEATURE_ORDER}
-
-
-# ══════════════════════════════════════════════════════════════════
-#  Stake multiplier
-# ══════════════════════════════════════════════════════════════════
-
-def stake_multiplier(
-    report,
-    setup,
-    dfs: Dict[Timeframe, pd.DataFrame],
-    primary_tf: Timeframe,
-    confirmation_tfs: Sequence[Timeframe],
-    artifact_dir=None,
-) -> float:
-    """Stake multiplier for a taken setup. Always ``1.0`` at this stage.
-
-    The guard order is already final — no artifact directory, a missing
-    artifact file, an artifact built for a different ``FEATURE_VERSION``,
-    or ANY exception all yield the neutral ``1.0``. Artifact evaluation
-    lands in a later step; until then this is wiring, not a decision.
-    """
-    try:
-        if not artifact_dir:
-            return 1.0
-        artifact = load_artifact(artifact_path_for(primary_tf, artifact_dir))
-        if artifact is None:
-            return 1.0
-        if artifact.get("feature_version") != FEATURE_VERSION:
-            logger.debug(
-                "position_sizing: artifact feature_version %r != %d — neutral",
-                artifact.get("feature_version"), FEATURE_VERSION)
-            return 1.0
-        # Step 1: evaluation not implemented yet.
-        return 1.0
-    except Exception:
-        logger.debug("position_sizing.stake_multiplier failed", exc_info=True)
-        return 1.0

providers/base.py

@@ -100,14 +100,6 @@ class BaseDataProvider(ABC):
     # boundary is missing the just-closed bar (feed latency).
     FRESH_FETCH_RETRY_SECONDS = 3.0
 
-    # Signal-side bad-print filter (see _validate_and_clean): a bar is a
-    # candidate when its close-to-close return sits more than
-    # OUTLIER_SIGMA standard deviations from the frame mean, and is
-    # dropped only when the NEXT close snaps back at least
-    # OUTLIER_REVERT_FRACTION of the way to the pre-spike close.
-    OUTLIER_SIGMA = 10.0
-    OUTLIER_REVERT_FRACTION = 0.8
-
     def __init__(self, config: PlatformConfig):
         self.config = config
         self._cache: Dict[str, pd.DataFrame] = {}
@@ -506,32 +498,20 @@ class BaseDataProvider(ABC):
         returns = returns.replace([float("inf"), float("-inf")], float("nan"))
         std_ret = returns.std()
         if std_ret > 0:
-            # A 10σ close-to-close move is NOT bad data by default. On
-            # three years of Oanda XAU bars every such bar was a real
-            # news/reopen move (NFP, CPI, Sunday gap — 2026-09-07 audit:
-            # 155 flagged M5 bars, 7 snapped back), and dropping them hid
-            # the most violent bar of the day from ATR/exhaustion math —
-            # while the backtester (whole-frame σ) dropped a DIFFERENT
-            # set than live (fetch-window σ). A bad print has the one
-            # signature a real move lacks: the NEXT close returns to the
-            # pre-spike level. Drop a candidate only on that observed
-            # snap-back; the newest bar (no successor yet) is kept as
-            # real, and the snap-back bar itself is never a candidate
-            # (its residual against the pre-spike close is small).
             dev = returns - returns.mean()
-            candidate = dev.abs() > self.OUTLIER_SIGMA * std_ret
-            close = df["Close"]
-            spike = close - close.shift(1)
-            residual = close.shift(-1) - close.shift(1)
-            snapped_back = residual.notna() & (
-                residual.abs()
-                <= (1.0 - self.OUTLIER_REVERT_FRACTION) * spike.abs()
+            outlier_mask = dev.abs() > 10 * std_ret
+            # A spike bar makes the NEXT bar's return look extreme too (it is
+            # measured against the spike). If the previous bar is flagged and
+            # this bar reverses direction, it is the legitimate snap-back —
+            # keep it, or every spike would punch a two-bar hole in the data.
+            recovery = (
+                outlier_mask
+                & outlier_mask.shift(1, fill_value=False)
+                & (dev * dev.shift(1) < 0)
             )
-            outlier_mask = candidate & snapped_back
+            outlier_mask = outlier_mask & ~recovery
             if outlier_mask.any():
-                logger.warning(
-                    f"{symbol}: Removing {int(outlier_mask.sum())} bad-print "
-                    f"bar(s) (>{self.OUTLIER_SIGMA:.0f}σ move that snapped back)")
+                logger.warning(f"{symbol}: Removing {outlier_mask.sum()} extreme outlier bars")
                 df = df[~outlier_mask]
 
         min_required = max(

providers/oanda.py

@@ -32,11 +32,7 @@ class OandaDataProvider(BaseDataProvider):
 
     name = "oanda"
     DATA_ERROR = OandaDataError
-    # Transient-fault retry ladder (see _fetch_oanda_candles): sleeps
-    # between attempts, plus a wall-clock cap on the whole sequence.
-    RETRY_DELAYS_S = (2.0, 5.0, 10.0)
-    RETRY_BUDGET_S = 45.0
-    RETRY_DELAY_S = RETRY_DELAYS_S[0]  # first delay; kept for callers/tests
+    RETRY_DELAY_S = 2.0
 
     OANDA_GRANULARITY = {
         Timeframe.M1: "M1",
@@ -97,11 +93,9 @@ class OandaDataProvider(BaseDataProvider):
         return self._fetch_oanda_candles(symbol, timeframe, outputsize)
 
     def _to_oanda_instrument(self, symbol: str) -> str:
-        try:
+        if symbol in self.OANDA_INSTRUMENT:
             return self.OANDA_INSTRUMENT[symbol]
-        except KeyError:
-            raise OandaDataError(
-                f"Unsupported symbol {symbol!r} for Oanda") from None
+        return symbol.replace("/", "_")
 
     def _fetch_oanda_candles(
         self,
@@ -132,53 +126,34 @@ class OandaDataProvider(BaseDataProvider):
         else:
             params["count"] = min(count, self.MAX_BARS_PER_REQUEST)
 
-        # Retry ladder for transient faults: Oanda's sporadic spurious
-        # 401s (~15-20/day on the practice API — the same token succeeds
-        # on adjacent requests), 5xx and network errors. Other 4xx (bad
-        # request, 429) are never retried; a real credential problem
-        # fails every attempt and raises as before. The 2026-09-07
-        # signal-log audit found 11 boundary scans in two days that lost
-        # their primary frame at xx:00:22-:26 UTC: a single 2 s retry
-        # could not outlast a 10-15 s provider blip at the top of the
-        # hour, and a scan without its primary frame logs a price-0
-        # NEUTRAL row. RETRY_BUDGET_S bounds the whole sequence so a scan
-        # (three fetches) still finishes well inside an M5 bar.
+        # One retry absorbs Oanda's sporadic spurious 401s (~15-20/day on
+        # the practice API — the same token succeeds on adjacent requests)
+        # plus transient 5xx/network faults. A real credential problem
+        # fails both attempts and raises exactly as before. Other 4xx
+        # (bad request, 429) are never retried.
         last_err: Optional[OandaDataError] = None
-        resp = None
-        attempts = 1 + len(self.RETRY_DELAYS_S)
-        started = time.monotonic()
-        for attempt in range(attempts):
+        for attempt in range(2):
             if attempt:
-                delay = self.RETRY_DELAYS_S[attempt - 1]
-                if time.monotonic() - started + delay > self.RETRY_BUDGET_S:
-                    logger.warning(
-                        f"Oanda fetch for {instrument}: retry budget "
-                        f"({self.RETRY_BUDGET_S:.0f}s) exhausted after "
-                        f"{attempt} attempt(s) — giving up")
-                    break
-                time.sleep(delay)
+                time.sleep(self.RETRY_DELAY_S)
             try:
                 resp = self._session.get(url, params=params, timeout=30)
             except requests.RequestException as e:
                 last_err = OandaDataError(
                     f"Oanda candle fetch failed for {instrument}: {e}")
                 last_err.__cause__ = e
-                resp = None
-                logger.warning(
-                    f"Oanda fetch for {instrument} attempt "
-                    f"{attempt + 1}/{attempts} failed: {e}")
                 continue
             if resp.status_code == 401 or resp.status_code >= 500:
                 last_err = OandaDataError(
                     f"Oanda candle fetch failed for {instrument}: "
                     f"HTTP {resp.status_code}: {resp.text[:300]}")
-                logger.warning(
-                    f"Oanda fetch for {instrument} attempt "
-                    f"{attempt + 1}/{attempts} got HTTP {resp.status_code}")
-                resp = None
+                if attempt == 0:
+                    logger.warning(
+                        f"Oanda fetch for {instrument} got HTTP "
+                        f"{resp.status_code}; retrying in "
+                        f"{self.RETRY_DELAY_S}s")
                 continue
             break
-        if resp is None:
+        else:
             raise last_err
 
         if resp.status_code != 200:

providers/tiingo.py

@@ -3,8 +3,6 @@ ForexBot2 — Tiingo data provider
 =================================
 OHLCV candles from Tiingo's FX endpoint
 (`/tiingo/fx/{ticker}/prices`, resampled mid-price bars, no volume).
-Tiingo is streaming-only in production: its REST candles remain available
-for comparison research, but it is not a designated candle fallback.
 All shared behavior (cache, validation, boundary guarantees, range
 pagination) lives in `BaseDataProvider`; this module only maps
 symbols/resample frequencies, sizes the request window, and parses
@@ -65,9 +63,8 @@ class TiingoDataProvider(BaseDataProvider):
     }
 
     # Window padding over the theoretical bar span: FX is closed ~2/7 of
-    # the week (Fri 17:00 → Sun 18:00 New York; see providers/base.py), so
-    # a calendar window must be wider than `bars × candle_seconds` to yield
-    # the requested count.
+    # the week (Fri 22:00 → Sun 22:00 UTC), so a calendar window must be
+    # wider than `bars × candle_seconds` to yield the requested count.
     WINDOW_PAD_FACTOR = 1.6
 
     def __init__(self, config: PlatformConfig, api_key: Optional[str] = None):
@@ -91,11 +88,9 @@ class TiingoDataProvider(BaseDataProvider):
         })
 
     def _to_tiingo_ticker(self, symbol: str) -> str:
-        try:
+        if symbol in self.TIINGO_TICKER:
             return self.TIINGO_TICKER[symbol]
-        except KeyError:
-            raise TiingoDataError(
-                f"Unsupported symbol {symbol!r} for Tiingo") from None
+        return symbol.replace("/", "").replace("=X", "").lower()
 
     # ── Raw fetch ──────────────────────────────────────────────────
 

providers/twelve_data.py

@@ -58,10 +58,10 @@ class TwelveDataProvider(BaseDataProvider):
 
     @staticmethod
     def _to_td_symbol(symbol: str) -> str:
-        if "/" not in symbol:
-            raise DataIntegrityError(
-                f"Unsupported symbol {symbol!r} for Twelve Data")
-        return symbol
+        if "/" in symbol:
+            return symbol
+        clean = symbol.replace("=X", "")
+        return f"{clean[:3]}/{clean[3:]}"
 
     # ── Raw fetch ──────────────────────────────────────────────────
 

scripts/backfill_news_archive.py

@@ -1,230 +0,0 @@
-#!/usr/bin/env python3
-"""
-Seed logs/newsdata.db from the headline blobs still on disk.
-==============================================================
-
-The durable headline corpus (``news_archive.py``) only started collecting
-when its hook shipped. Everything before that survives only inside signal
-logs, in two different shapes:
-
-* ``logs/signals.db`` — modern schema, headlines interned into
-  ``news_snapshots`` (kind ``headlines``) and referenced by hash.
-* an older orphan DB (default ``~/signals.db``) — pre-interning, blobs
-  inline in ``signals.headlines_json``, short-form item keys with no
-  ``description``, ``link``, ``publisher`` or ``score_by_currency``.
-
-Both are opened READ-ONLY. The script is idempotent: re-running it adds
-nothing new and never downgrades a score.
-
-Provenance is RECOMPUTED, not trusted
--------------------------------------
-Neither source records which scorer produced its ``score`` — that flag
-only exists from this change onwards. So each headline is re-scored with
-the live keyword scorer and compared:
-
-* stored score differs from the keyword score  -> ``llm``   (DeepSeek wrote it)
-* exact match on a non-zero score              -> ``keyword``
-* both zero                                    -> ``unknown`` (indistinguishable)
-
-Rows are stamped ``origin='backfill'`` because they come from the
-post-filter pool: ``get_fundamental_context`` drops headlines inside
-±0.1 before they ever reach a signal row, so genuinely neutral headlines
-are under-represented in this history. Rows collected live by the hook
-in ``news_engine`` do not have that bias and are stamped ``'live'``.
-
-Usage
------
-    venv/bin/python scripts/backfill_news_archive.py --dry-run
-    venv/bin/python scripts/backfill_news_archive.py
-    venv/bin/python scripts/backfill_news_archive.py --legacy-db /path/to/old.db
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import os
-import sqlite3
-import sys
-from collections import Counter
-from typing import Any, Dict, Iterator, Tuple
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-import app_config          # noqa: E402
-import news_archive        # noqa: E402
-from config import NewsConfig  # noqa: E402
-from news_engine import NewsEngine  # noqa: E402
-
-
-def _open_ro(path: str) -> sqlite3.Connection:
-    """Read-only, but NOT immutable=1 — that flag is blind to WAL rows."""
-    return sqlite3.connect(f"file:{os.path.abspath(path)}?mode=ro", uri=True)
-
-
-def _has_table(con: sqlite3.Connection, name: str) -> bool:
-    row = con.execute(
-        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?",
-        (name,)).fetchone()
-    return row is not None
-
-
-def iter_blobs(path: str) -> Iterator[Tuple[str, str]]:
-    """Yield (headlines_json, seen_at) from either DB shape."""
-    con = _open_ro(path)
-    try:
-        if _has_table(con, "news_snapshots"):
-            yield from con.execute(
-                "SELECT body, first_seen FROM news_snapshots"
-                " WHERE kind='headlines' ORDER BY first_seen")
-        cols = {r[1] for r in con.execute("PRAGMA table_info(signals)")}
-        if "headlines_json" in cols:
-            order = "logged_at" if "logged_at" in cols else "id"
-            yield from con.execute(
-                "SELECT headlines_json, "
-                + ("logged_at" if "logged_at" in cols else "''")
-                + " FROM signals WHERE headlines_json IS NOT NULL"
-                  f" ORDER BY {order}")
-    finally:
-        con.close()
-
-
-class _Row:
-    """Duck-typed stand-in for NewsSentiment, so news_archive.build_row
-    flattens a stored dict exactly the way it flattens a live headline."""
-
-    __slots__ = ("headline", "source", "timestamp", "currencies_mentioned",
-                 "sentiment_score", "keywords", "sentiment_by_currency",
-                 "link", "description", "publisher", "score_source")
-
-    def __init__(self, item: Dict[str, Any], score_source: str):
-        self.headline = item.get("headline")
-        self.source = item.get("source")
-        self.timestamp = item.get("timestamp")
-        self.currencies_mentioned = item.get("currencies") or []
-        self.sentiment_score = item.get("score")
-        self.keywords = item.get("keywords") or []
-        self.sentiment_by_currency = item.get("score_by_currency") or {}
-        self.link = item.get("link")
-        self.description = item.get("description")
-        self.publisher = item.get("publisher")
-        self.score_source = score_source
-
-
-def classify(engine: NewsEngine, item: Dict[str, Any]) -> str:
-    """Recover which scorer produced this stored score."""
-    text = item.get("headline")
-    if not isinstance(text, str) or not text.strip():
-        return "unknown"
-    try:
-        stored = float(item.get("score") or 0.0)
-    except (TypeError, ValueError):
-        return "unknown"
-    keyword = engine._analyze_headline_sentiment(text)
-    if abs(stored - keyword) > 1e-9:
-        return "llm"
-    if abs(stored) < 1e-9:
-        return "unknown"   # both zero — the two scorers are indistinguishable
-    return "keyword"
-
-
-def collect(paths, engine: NewsEngine):
-    """Newest wins per headline: later blobs carry upgraded LLM scores."""
-    best: Dict[str, Tuple[str, Dict[str, Any], str]] = {}
-    stats = Counter()
-    for path in paths:
-        if not os.path.exists(path):
-            print(f"  skip (not found): {path}")
-            continue
-        blobs = 0
-        for body, seen_at in iter_blobs(path):
-            try:
-                items = json.loads(body)
-            except Exception:
-                stats["bad_blob"] += 1
-                continue
-            blobs += 1
-            for item in items:
-                if not isinstance(item, dict):
-                    continue
-                text = item.get("headline")
-                if not isinstance(text, str) or not text.strip():
-                    continue
-                source = classify(engine, item)
-                prior = best.get(text)
-                if prior is None or (
-                        news_archive._SOURCE_RANK.get(source, 0)
-                        >= news_archive._SOURCE_RANK.get(prior[0], 0)):
-                    best[text] = (source, item, seen_at or "")
-                stats["items"] += 1
-        print(f"  {path}: {blobs} blobs")
-    for source, _, _ in best.values():
-        stats[source] += 1
-    return best, stats
-
-
-def main() -> int:
-    default_legacy = os.path.join(os.path.expanduser("~"), "signals.db")
-    ap = argparse.ArgumentParser(description=__doc__.split("\n")[1])
-    ap.add_argument("--db", default=app_config.signal_log_path(),
-                    help="modern signal-log DB (default: config path)")
-    ap.add_argument("--legacy-db", default=default_legacy,
-                    help=f"older inline-blob DB (default: {default_legacy})")
-    ap.add_argument("--out", default=app_config.news_archive_path(),
-                    help="archive DB to write (default: config path)")
-    ap.add_argument("--dry-run", action="store_true",
-                    help="count only, write nothing")
-    args = ap.parse_args()
-
-    engine = NewsEngine(NewsConfig())
-    print("Reading headline blobs (read-only):")
-    best, stats = collect([args.db, args.legacy_db], engine)
-
-    print(f"\n  headline items scanned : {stats['items']}")
-    print(f"  distinct headlines     : {len(best)}")
-    print(f"    llm (DeepSeek)       : {stats['llm']}")
-    print(f"    keyword              : {stats['keyword']}")
-    print(f"    unknown (both zero)  : {stats['unknown']}")
-    if stats["bad_blob"]:
-        print(f"  unparseable blobs      : {stats['bad_blob']}")
-
-    if args.dry_run:
-        print("\n--dry-run: nothing written.")
-        return 0
-    if not best:
-        print("\nNothing to write.")
-        return 0
-
-    rows = []
-    for text, (source, item, seen_at) in best.items():
-        row = news_archive.build_row(
-            _Row(item, source),
-            teacher_model=None,          # not recorded by either source
-            teacher_prompt_sha1=None,
-            origin="backfill",
-            first_seen=seen_at or None,
-            last_seen=seen_at or None)
-        if row is not None:
-            rows.append(row)
-
-    archive = news_archive.NewsArchive(args.out)
-    try:
-        written = archive.store(rows)
-    finally:
-        archive.close()
-    print(f"\nWrote {written} rows to {args.out}")
-
-    con = _open_ro(args.out)
-    try:
-        print("\nArchive now holds:")
-        for source, count in con.execute(
-                "SELECT score_source, COUNT(*) FROM headlines"
-                " GROUP BY score_source ORDER BY COUNT(*) DESC"):
-            print(f"  {source:8s} {count}")
-    finally:
-        con.close()
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

scripts/backfill_signal_outcomes.py

@@ -1,91 +0,0 @@
-#!/usr/bin/env python3
-"""
-backfill_signal_outcomes.py — seed and advance advisory outcomes for every
-logged trade setup (not only panel rows), one per advisory event.
-
-Pass A (offline, no network): assign setup_key / advisory_event_id to all
-legacy setup rows, mark same-setup re-logs within 60 min as DUPLICATE, and
-seed a PENDING outcome on each canonical row that has none. Existing
-(panel-seeded) outcomes are never overwritten.
-
-Pass B: advance PENDING rows from the closed-candle streams already stored
-in the DB (union across every snapshot of the timeframe — no provider).
-
-Pass C (--reseed-unresolvable): bounded re-seed of continuity-broken rows,
-then Pass B again.
-
-Safe beside the live writers: short per-row transactions with a busy
-timeout. Read-only preview with --dry-run.
-
-    python scripts/backfill_signal_outcomes.py --dry-run
-    python scripts/backfill_signal_outcomes.py
-    python scripts/backfill_signal_outcomes.py --reseed-unresolvable
-"""
-import argparse
-import os
-import sqlite3
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-import app_config  # noqa: E402
-from signal_log import SignalLogger  # noqa: E402
-
-
-def _status_counts(con: sqlite3.Connection) -> dict:
-    return dict(con.execute(
-        "SELECT COALESCE(advisory_outcome_status, 'NULL'), COUNT(*)"
-        " FROM signals WHERE setup_json IS NOT NULL AND setup_json != 'null'"
-        " GROUP BY 1").fetchall())
-
-
-def main(argv=None) -> int:
-    ap = argparse.ArgumentParser(description=__doc__.splitlines()[1])
-    ap.add_argument("--db", default=None, help="defaults to the signal_log.path setting in config.json")
-    ap.add_argument("--dry-run", action="store_true")
-    ap.add_argument("--reseed-unresolvable", action="store_true")
-    ap.add_argument("--symbol", default="XAU/USD")
-    args = ap.parse_args(argv)
-    path = args.db or app_config.signal_log_path()
-    if not os.path.exists(path):
-        print(f"no database at {path}")
-        return 1
-    if args.dry_run:
-        con = sqlite3.connect(f"file:{os.path.abspath(path)}?mode=ro", uri=True)
-        print("setup rows by outcome status:", _status_counts(con))
-        have = {r[1] for r in con.execute("PRAGMA table_info(signals)")}
-        if "setup_key" in have:
-            n_keyless = con.execute(
-                "SELECT COUNT(*) FROM signals WHERE setup_json IS NOT NULL"
-                " AND setup_json != 'null' AND setup_key IS NULL").fetchone()[0]
-            print(f"setup rows without setup_key: {n_keyless}")
-        else:
-            print("setup_key column not present yet (schema migrates on"
-                  " the first non-dry run)")
-        con.close()
-        return 0
-    lg = SignalLogger(path)
-    try:
-        print("before:", _status_counts(lg._conn))
-        print("pass A (seed/dedup):", lg.backfill_seeds())
-        total = 0
-        for _ in range(50):
-            n = lg.resolve_from_store(args.symbol)
-            total += n
-            if n == 0:
-                break
-        print(f"pass B (advance from stored bars): {total} rows changed")
-        if args.reseed_unresolvable:
-            reseeded = lg.reseed_unresolvable()
-            print(f"pass C (re-seed): {len(reseeded)} rows re-seeded")
-            for _ in range(50):
-                if lg.resolve_from_store(args.symbol) == 0:
-                    break
-        print("after:", _status_counts(lg._conn))
-    finally:
-        lg.close()
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

scripts/label_scan_outcomes.py

@@ -1,115 +0,0 @@
-#!/usr/bin/env python3
-"""
-label_scan_outcomes.py — batch driver for the scan_outcomes table.
-
-Labels model-free forward returns (1/4/24 h, $ and ATR-normalized, plus
-MFE/MAE, a 3xATR burst flag and a completeness flag) for every logged scan
-whose 24 h window has closed, from the closed-candle streams already
-stored in the DB. No provider, no network.
-
-The live loggers do the same work opportunistically at the end of every
-scan (time-boxed to ~2 s); this script is the catch-up pass for a backlog
-— a freshly imported database, or rows whose bars only arrived later.
-
-Idempotent: a row is rewritten only when its numbers change, so a settled
-database costs one read pass. Safe beside the live writers (short per-row
-transactions with a busy timeout). Read-only preview with --dry-run.
-
-    python scripts/label_scan_outcomes.py --dry-run
-    python scripts/label_scan_outcomes.py
-    python scripts/label_scan_outcomes.py --db logs/signals.db --budget 5000
-"""
-import argparse
-import os
-import sqlite3
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-import app_config  # noqa: E402
-from signal_log import SCAN_LABEL_MIN_AGE_H, SignalLogger  # noqa: E402
-
-CHUNK = 2000
-
-
-def _counts(con: sqlite3.Connection, symbol: str,
-            labelled: bool = True) -> dict:
-    """Candidate / labelled / complete / pending counts (read-only).
-
-    The age filter uses logged_at (always an ISO string) as a proxy for
-    the bar close — the labeller itself parses the real bar label."""
-    scans = ("SELECT COUNT(*) FROM signals s WHERE s.symbol = ?"
-             " AND s.indicators_json IS NOT NULL AND s.price > 0")
-    due = (scans + " AND s.logged_at <"
-           f" datetime('now', '-{int(SCAN_LABEL_MIN_AGE_H)} hours')")
-    out = {
-        "scan rows with indicators": con.execute(scans, (symbol,)).fetchone()[0],
-        "of those, window elapsed": con.execute(due, (symbol,)).fetchone()[0],
-    }
-    if labelled:
-        out["labelled"] = con.execute(
-            "SELECT COUNT(*) FROM scan_outcomes").fetchone()[0]
-        out["complete"] = con.execute(
-            "SELECT COUNT(*) FROM scan_outcomes WHERE complete = 1"
-        ).fetchone()[0]
-        out["pending (labelled, incomplete)"] = con.execute(
-            "SELECT COUNT(*) FROM scan_outcomes WHERE complete = 0"
-        ).fetchone()[0]
-    return out
-
-
-def main(argv=None) -> int:
-    ap = argparse.ArgumentParser(description=__doc__.splitlines()[1])
-    ap.add_argument("--db", default=None, help="defaults to the signal_log.path setting in config.json")
-    ap.add_argument("--dry-run", action="store_true",
-                    help="read-only candidate counts, writes nothing")
-    ap.add_argument("--budget", type=int, default=100000,
-                    help="maximum rows to write in this run")
-    ap.add_argument("--symbol", default="XAU/USD")
-    args = ap.parse_args(argv)
-    path = args.db or app_config.signal_log_path()
-    if not os.path.exists(path):
-        print(f"no database at {path}")
-        return 1
-
-    if args.dry_run:
-        con = sqlite3.connect(f"file:{os.path.abspath(path)}?mode=ro", uri=True)
-        try:
-            have = {r[0] for r in con.execute(
-                "SELECT name FROM sqlite_master WHERE type='table'")}
-            if "scan_outcomes" not in have:
-                print("scan_outcomes table not present yet (created on the"
-                      " first non-dry run / next logged scan)")
-            for key, value in _counts(con, args.symbol,
-                                      labelled="scan_outcomes" in have).items():
-                print(f"  {key}: {value}")
-        finally:
-            con.close()
-        return 0
-
-    lg = SignalLogger(path)
-    try:
-        print("before:", _counts(lg._conn, args.symbol))
-        written = 0
-        previous_complete = -1
-        while written < args.budget:
-            n = lg.label_scan_outcomes(
-                budget=min(CHUNK, args.budget - written), time_budget=120.0)
-            written += n
-            complete = lg._conn.execute(
-                "SELECT COUNT(*) FROM scan_outcomes WHERE complete = 1"
-            ).fetchone()[0]
-            # Stop on a pass that wrote nothing, or one that could not
-            # complete a single further row (only re-labelling stragglers).
-            if n == 0 or complete == previous_complete:
-                break
-            previous_complete = complete
-        print(f"labelled (rows written): {written}")
-        print("after:", _counts(lg._conn, args.symbol))
-    finally:
-        lg.close()
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

scripts/migrate_signal_log_news.py

@@ -1,177 +0,0 @@
-#!/usr/bin/env python3
-"""
-migrate_signal_log_news.py — move inline news blobs into news_snapshots.
-
-Legacy rows store the full RSS headline list (~33 KB) and calendar events
-(~1.2 KB) inline on EVERY scan row; the same blob repeats ~5× because the
-news caches outlive a candle. This script interns each distinct blob once
-(content hash), stamps `headlines_hash` / `events_hash` on the row and
-NULLs the inline columns. Readers use the `signals_with_news` view or
-`signal_log.read_news`.
-
-Idempotent and resumable (batched transactions). Run --vacuum with BOTH
-live writers (telegram bot, dashboard publisher) stopped — VACUUM needs
-exclusive access and ~1× the final size of free disk.
-
-    python scripts/migrate_signal_log_news.py --dry-run
-    python scripts/migrate_signal_log_news.py
-    python scripts/migrate_signal_log_news.py --vacuum      # bots stopped
-"""
-import argparse
-import hashlib
-import os
-import sqlite3
-import sys
-from datetime import datetime, timezone
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-import app_config  # noqa: E402
-from signal_log import SignalLogger  # noqa: E402
-
-
-def _hash(text: str) -> str:
-    return hashlib.sha256(text.encode("utf-8")).hexdigest()
-
-
-def migrate(con: sqlite3.Connection, batch: int = 500) -> dict:
-    """Intern legacy inline blobs; returns counters. Caller owns the
-    connection (schema must already exist — open a SignalLogger once)."""
-    stats = {"rows": 0, "headlines_interned": 0, "events_interned": 0}
-    now = datetime.now(timezone.utc).isoformat()
-    while True:
-        rows = con.execute(
-            "SELECT id, headlines_json, events_json FROM signals"
-            " WHERE (headlines_json IS NOT NULL AND headlines_hash IS NULL)"
-            "    OR (events_json IS NOT NULL AND events_hash IS NULL)"
-            " ORDER BY id LIMIT ?", (batch,)).fetchall()
-        if not rows:
-            break
-        con.execute("BEGIN IMMEDIATE")
-        try:
-            for rowid, headlines, events in rows:
-                sets, params = [], []
-                if headlines is not None:
-                    h = _hash(headlines)
-                    cur = con.execute(
-                        "INSERT OR IGNORE INTO news_snapshots"
-                        " (hash, kind, body, first_seen, last_seen)"
-                        " VALUES (?, 'headlines', ?, ?, ?)",
-                        (h, headlines, now, now))
-                    stats["headlines_interned"] += cur.rowcount
-                    sets += ["headlines_hash=?", "headlines_json=NULL"]
-                    params.append(h)
-                if events is not None:
-                    h = _hash(events)
-                    cur = con.execute(
-                        "INSERT OR IGNORE INTO news_snapshots"
-                        " (hash, kind, body, first_seen, last_seen)"
-                        " VALUES (?, 'events', ?, ?, ?)",
-                        (h, events, now, now))
-                    stats["events_interned"] += cur.rowcount
-                    sets += ["events_hash=?", "events_json=NULL"]
-                    params.append(h)
-                params.append(rowid)
-                con.execute(
-                    f"UPDATE signals SET {', '.join(sets)} WHERE id=?",
-                    params)
-                stats["rows"] += 1
-            con.commit()
-        except Exception:
-            con.rollback()
-            raise
-    return stats
-
-
-def prune_orphans(con: sqlite3.Connection) -> int:
-    cur = con.execute(
-        "DELETE FROM news_snapshots WHERE hash NOT IN"
-        " (SELECT headlines_hash FROM signals WHERE headlines_hash IS NOT NULL"
-        "  UNION SELECT events_hash FROM signals WHERE events_hash IS NOT NULL)")
-    con.commit()
-    return cur.rowcount
-
-
-def prune_news_older_than(con: sqlite3.Connection, days: int) -> int:
-    """Drop the news reference (not the row) from rows older than N days,
-    then delete snapshots nothing references any more."""
-    # logged_at is ISO8601 with a 'T' and a +00:00 offset; SQLite's
-    # datetime() returns 'YYYY-MM-DD HH:MM:SS'. Comparing them as raw
-    # strings put every row from the cutoff CALENDAR DAY on the wrong
-    # side ('T' > ' '), so a full day never pruned. Normalise to the
-    # same shape first — all logged_at values are UTC (see _iso()).
-    cur = con.execute(
-        "UPDATE signals SET headlines_hash=NULL, events_hash=NULL"
-        " WHERE substr(replace(logged_at, 'T', ' '), 1, 19)"
-        " < datetime('now', ?)", (f"-{int(days)} days",))
-    con.commit()
-    prune_orphans(con)
-    return cur.rowcount
-
-
-def main(argv=None) -> int:
-    ap = argparse.ArgumentParser(description=__doc__.splitlines()[1])
-    ap.add_argument("--db", default=None)
-    ap.add_argument("--dry-run", action="store_true")
-    ap.add_argument("--batch", type=int, default=500)
-    ap.add_argument("--prune-news-older-than", type=int, default=None,
-                    metavar="DAYS")
-    ap.add_argument("--prune-orphans", action="store_true")
-    ap.add_argument("--vacuum", action="store_true",
-                    help="checkpoint + VACUUM (all writers must be stopped)")
-    ap.add_argument("--no-backup", action="store_true")
-    args = ap.parse_args(argv)
-    path = args.db or app_config.signal_log_path()
-    if not os.path.exists(path):
-        print(f"no database at {path}")
-        return 1
-    size0 = os.path.getsize(path)
-    if args.dry_run:
-        con = sqlite3.connect(f"file:{os.path.abspath(path)}?mode=ro", uri=True)
-        n = con.execute(
-            "SELECT COUNT(*) FROM signals WHERE headlines_json IS NOT NULL"
-            " OR events_json IS NOT NULL").fetchone()[0]
-        d = con.execute(
-            "SELECT COUNT(DISTINCT headlines_json) FROM signals"
-            " WHERE headlines_json IS NOT NULL").fetchone()[0]
-        print(f"{n} rows carry inline news blobs ({d} distinct headline "
-              f"blobs); file size {size0 / 1e6:.1f} MB")
-        con.close()
-        return 0
-    if not args.no_backup:
-        backup = f"{path}.pre-news-{datetime.now(timezone.utc):%Y%m%d}"
-        if not os.path.exists(backup):
-            src = sqlite3.connect(path)
-            dst = sqlite3.connect(backup)
-            src.backup(dst)
-            dst.close()
-            src.close()
-            print(f"backup written: {backup}")
-    lg = SignalLogger(path)          # applies the additive schema
-    lg.close()
-    con = sqlite3.connect(path)
-    con.execute("PRAGMA busy_timeout=10000")
-    try:
-        print("migrate:", migrate(con, batch=args.batch))
-        if args.prune_news_older_than is not None:
-            print("rows de-referenced:",
-                  prune_news_older_than(con, args.prune_news_older_than))
-        if args.prune_orphans:
-            print("orphan snapshots removed:", prune_orphans(con))
-        if args.vacuum:
-            try:
-                con.execute("PRAGMA wal_checkpoint(TRUNCATE)")
-                con.execute("VACUUM")
-            except sqlite3.OperationalError as exc:
-                print(f"VACUUM failed ({exc}) — stop the bot and dashboard "
-                      "publisher first, then re-run with --vacuum")
-                return 2
-    finally:
-        con.close()
-    print(f"size {size0 / 1e6:.1f} MB -> {os.path.getsize(path) / 1e6:.1f} MB"
-          + ("" if args.vacuum else " (run --vacuum to reclaim space)"))
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

scripts/shadow_scan.py

@@ -1,192 +0,0 @@
-#!/usr/bin/env python3
-"""
-Phase 4 — live shadow scan (data source change test)
-=====================================================
-Runs TWO data providers side-by-side at every candle close (M5/M15/H1)
-and logs how their signals diverge, so the feed switch can be validated
-under real fetch depths / caching / boundary timing for ≥2 weeks.
-
-Unlike the history studies, this uses the LIVE fetch path (fetch_ohlcv,
-forming-candle drop, freshness checks) for both providers, but calls
-`analyze_from_dataframes(use_news=False)` directly so the (provider-
-independent) news/DeepSeek budget is not burned on shadow scans. The
-weekend filter is applied here (skip the whole cycle), and the session/
-day/hour gates run inside analyze_from_dataframes as in the backtest.
-
-No orders, no Telegram, no signal-log writes — output is a side-by-side
-CSV plus a rolling agreement tally.
-
-Usage:
-    venv/bin/python scripts/shadow_scan.py --a oanda --b tiingo \
-        --tfs M5,M15,H1 --out logs/shadow_oanda_tiingo.csv --once
-    # production (2+ weeks):
-    nohup venv/bin/python scripts/shadow_scan.py --a oanda --b tiingo \
-        --out logs/shadow_oanda_tiingo.csv >> logs/shadow_scan.log 2>&1 &
-
-CSV columns (one row per TF per boundary):
-    ts, tf, a_<field>, b_<field> for direction/grade/setup/tag/entry/sl/tp1,
-    a_price, b_price, a_spread, b_spread, a_warnings, b_warnings,
-    dir_agree, grade_agree, setup_agree
-"""
-
-import argparse
-import csv
-import sys
-import time
-from datetime import datetime, timezone
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
-
-from config import PlatformConfig, Timeframe, XAUUSD
-from pair_signals.xauusd import make_config
-from providers import make_data_provider
-from providers.base import _is_forex_weekend
-from signal_aggregator import SignalAggregator, SignalReport
-
-_TF_LOOKUP = {"M5": Timeframe.M5, "M15": Timeframe.M15, "H1": Timeframe.H1,
-              "H4": Timeframe.H4}
-_TF_MINUTES = {Timeframe.M5: 5, Timeframe.M15: 15, Timeframe.H1: 60,
-               Timeframe.H4: 240}
-_CONFIRMS = {Timeframe.M5: [Timeframe.M15, Timeframe.H1],
-             Timeframe.M15: [Timeframe.M5, Timeframe.H1],
-             Timeframe.H1: [Timeframe.M15, Timeframe.H4],
-             Timeframe.H4: [Timeframe.H1, Timeframe.D1]}
-
-FIELDS = ["direction", "grade", "setup", "tag", "entry", "sl", "tp1"]
-
-
-def seconds_until_boundary(interval_minutes: int, buffer: int = 10) -> float:
-    now = datetime.now(timezone.utc)
-    elapsed = (now.minute % interval_minutes) * 60 + now.second \
-        + now.microsecond / 1e6
-    return interval_minutes * 60 - elapsed + buffer
-
-
-def due_timeframes(wake: datetime, tfs) -> list:
-    minutes = wake.hour * 60 + wake.minute
-    floored = minutes - (minutes % 5)
-    return [tf for tf in tfs if floored % _TF_MINUTES[tf] == 0]
-
-
-def tag_of(report) -> str:
-    fb = report.bullish_reasons[0] if report.bullish_reasons else ""
-    fbe = report.bearish_reasons[0] if report.bearish_reasons else ""
-    if "XAU PULLBACK" in fb or "XAU PULLBACK" in fbe:
-        return "XAU_PB"
-    if "XAU CHART PATTERN" in fb or "XAU CHART PATTERN" in fbe:
-        return "XAU_CP"
-    if "XAU STRATEGY" in fb:
-        return "XAU"
-    return "GENERIC"
-
-
-def scan(provider, symbol, tf) -> SignalReport:
-    confirms = _CONFIRMS[tf]
-    cfg = make_config(tf, confirms)
-    agg = SignalAggregator(cfg)
-    agg.data_provider = provider
-    dfs = {}
-    for t in [tf] + confirms:
-        dfs[t] = provider.fetch_ohlcv(symbol, t)
-    return agg.analyze_from_dataframes(symbol, dfs, use_news=False)
-
-
-def extract(provider, report) -> dict:
-    s = report.trade_setup
-    return {
-        "direction": report.final_signal.value,
-        "grade": report.signal_quality,
-        "setup": 1 if s is not None else 0,
-        "tag": tag_of(report) if s else "",
-        "entry": round(s.entry_price, 3) if s else "",
-        "sl": round(s.stop_loss, 3) if s else "",
-        "tp1": round(s.take_profit_1, 3) if s else "",
-        "price": round(report.current_price, 3),
-        "spread": round(provider.get_spread(XAUUSD), 3),
-        "warnings": " | ".join(report.warnings)[:200],
-    }
-
-
-def main():
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--a", default="oanda")
-    ap.add_argument("--b", default="tiingo")
-    ap.add_argument("--tfs", default="M5,M15,H1")
-    ap.add_argument("--out", default="logs/shadow_scan.csv")
-    ap.add_argument("--once", action="store_true",
-                    help="one immediate scan cycle, then exit")
-    args = ap.parse_args()
-
-    tfs = [_TF_LOOKUP[s.strip()] for s in args.tfs.split(",")]
-    pa = make_data_provider(PlatformConfig(), args.a)
-    pb = make_data_provider(PlatformConfig(), args.b)
-
-    out_path = Path(args.out)
-    out_path.parent.mkdir(parents=True, exist_ok=True)
-    new_file = not out_path.exists()
-    tally = {"cycles": 0, "dir": 0, "grade": 0, "setup": 0}
-
-    with open(out_path, "a", newline="") as fh:
-        w = csv.writer(fh)
-        if new_file:
-            w.writerow(["ts", "tf"] + [f"a_{x}" for x in FIELDS + ["price", "spread", "warnings"]]
-                       + [f"b_{x}" for x in FIELDS + ["price", "spread", "warnings"]]
-                       + ["dir_agree", "grade_agree", "setup_agree"])
-
-        while True:
-            if args.once:
-                wake = datetime.now(timezone.utc)
-            else:
-                wait = seconds_until_boundary(5, buffer=10)
-                time.sleep(wait)
-                wake = datetime.now(timezone.utc)
-
-            if _is_forex_weekend():
-                if args.once:
-                    print("weekend — nothing to scan")
-                    return
-                continue
-
-            for tf in due_timeframes(wake, tfs):
-                try:
-                    ra = scan(pa, XAUUSD, tf)
-                except Exception as e:
-                    print(f"[{wake:%H:%M:%S}] {args.a} {tf.value} FAILED: {e}")
-                    continue
-                try:
-                    rb = scan(pb, XAUUSD, tf)
-                except Exception as e:
-                    print(f"[{wake:%H:%M:%S}] {args.b} {tf.value} FAILED: {e}")
-                    continue
-
-                ea, eb = extract(pa, ra), extract(pb, rb)
-                da = ea["direction"] == eb["direction"]
-                ga = ea["grade"] == eb["grade"]
-                sa = ea["setup"] == eb["setup"]
-                tally["cycles"] += 1
-                tally["dir"] += da; tally["grade"] += ga; tally["setup"] += sa
-
-                row = [wake.isoformat(), tf.value]
-                row += [ea[x] for x in FIELDS + ["price", "spread", "warnings"]]
-                row += [eb[x] for x in FIELDS + ["price", "spread", "warnings"]]
-                row += [int(da), int(ga), int(sa)]
-                w.writerow(row)
-                fh.flush()
-
-                print(f"[{wake:%Y-%m-%d %H:%M:%S}] {tf.value:<5} "
-                      f"{args.a}={ea['direction']:+d}/{ea['grade'] or '-':>2} "
-                      f"{args.b}={eb['direction']:+d}/{eb['grade'] or '-':>2} "
-                      f"dir={'OK' if da else 'FLIP'} "
-                      f"| Δprice {abs(ea['price']-eb['price']):.2f} "
-                      f"| agree dir {tally['dir']}/{tally['cycles']} "
-                      f"grade {tally['grade']}/{tally['cycles']} "
-                      f"setup {tally['setup']}/{tally['cycles']}")
-
-            if args.once:
-                print("shadow scan complete (--once)")
-                return
-
-
-if __name__ == "__main__":
-    main()

signal_aggregator.py

@@ -5,11 +5,9 @@ Combines multi-timeframe technical analysis with optional news context
 into a graded actionable signal.
 """
 
-import hashlib
-import json
 import logging
 import os
-from dataclasses import asdict, dataclass, field
+from dataclasses import dataclass, field
 from datetime import datetime, timezone
 from typing import Dict, List, Optional
 from zoneinfo import ZoneInfo
@@ -18,17 +16,11 @@ import pandas as pd
 
 _NY_TZ = ZoneInfo("America/New_York")
 
-# Prefix of the cost-floor veto both core setup generators emit; the XAU_CP
-# cleanup and the post-archetype stale-veto pass match on it, so it is
-# spelled once.
-_SPREAD_GATE_PREFIX = "SPREAD GATE:"
-
 from chart_patterns import detect_chart_pattern
 from config import PlatformConfig, SignalStrength, Timeframe
 from providers import BaseDataProvider, make_data_provider
-from engine import TechnicalEngine, TechnicalAnalysis, regime_weight_table
+from engine import TechnicalEngine, TechnicalAnalysis
 from news_engine import NewsEngine, FundamentalContext
-import signal_log
 
 logger = logging.getLogger(__name__)
 
@@ -63,22 +55,14 @@ class TradeSetup:
     risk_pips: float
     reward_pips: float
     pip_value: float
-    # Entry archetype — selects trail params and explains the setup to
-    # advisory consumers. setup_pattern is populated only by XAU_CP.
+    # Entry archetype ("GENERIC", "XAU_PB") — selects trail params.
     strategy: str = "GENERIC"
-    setup_pattern: Optional[str] = None
-    # Strategy cost assumption, not a broker/data-provider quote.
-    round_trip_cost: Optional[float] = None
     # Trailing exit params stamped from the PRODUCING config, so per-TF
     # factory overrides (e.g. M15 arm 0.70, M5 dist 0.85) reach every
     # consumer. None = legacy/unstamped setup — formatters fall back to
     # platform defaults; 0.0 means trailing is genuinely disabled.
     trail_arm_r: Optional[float] = None
     trail_dist_r: Optional[float] = None
-    # Post-decision stake multiplier from position_sizing (advisory
-    # overlay). None = unsized — every gate/exit path ignores it, and
-    # the backtester's R accounting is unaffected unless it opts in.
-    stake_multiplier: Optional[float] = None
 
 
 @dataclass
@@ -104,25 +88,6 @@ class SignalReport:
     news_description: str = ""
     news_summary: str = ""  # LLM market-context blurb (display-only)
 
-    # LLM agent-panel consensus (advisory/DISPLAY-ONLY). Populated
-    # post-hoc by consumers via agent_panel.attach_panel — NEVER inside
-    # analyze_from_dataframes (backtest parity) and never read by any
-    # gate or scoring path.
-    panel_verdict: str = ""        # "AGREE" / "NEUTRAL" / "OPPOSE" / ""
-    panel_confidence: float = 0.0
-    panel_rationale: str = ""
-    panel_takes: List[Dict] = field(default_factory=list)
-    panel_created_at: str = ""
-    panel_cache_status: str = ""
-    panel_payload_hash: str = ""
-    panel_verdict_source: str = ""  # "macro" / "chair" / ""
-
-    # Violent-move watch line (advisory/DISPLAY-ONLY). Stamped post-hoc
-    # by surfaces via violent_watch.attach_watch — never populated
-    # inside analyze_from_dataframes (backtest parity) and never read by
-    # any gate, grade or setup generator.
-    watch_line: str = ""
-
     final_signal: SignalStrength = SignalStrength.NEUTRAL
     final_score: float = 0.0
     final_confidence: float = 0.0
@@ -134,30 +99,23 @@ class SignalReport:
     bullish_reasons: List[str] = field(default_factory=list)
     bearish_reasons: List[str] = field(default_factory=list)
     warnings: List[str] = field(default_factory=list)
-    # Warnings a shipped setup superseded, removed from `warnings` but
-    # logged so they stay auditable (metadata only): the vetoes the
-    # XAU_CP acceptance stripped and, since 2026-09-08, a core
-    # generator's stale spread-gate veto when a later generator produced
-    # the setup (`_drop_stale_spread_gate_vetoes`).
-    cleaned_warnings: List[str] = field(default_factory=list)
 
     current_price: float = 0.0
     spread_estimate: float = 0.0
     market_regime: str = ""
     volatility_regime: str = ""
     exhaustion_count: int = 0
-    # Record-keeping only (no gate reads these): the penalty tags that
-    # fired in _compute_final_signal (tag -> RAW pre-decay amount), the
-    # decayed total actually subtracted from confidence, and whether the
-    # momentum-fading half of the exhaustion block was true.
-    penalty_tags: Dict[str, float] = field(default_factory=dict)
-    penalty_total: float = 0.0
-    momentum_fading: bool = False
 
 
 class SignalAggregator:
     """Master aggregator for live signal analysis."""
 
+    # Diagnostic funnel for the M5 pullback route (shared across
+    # instances; read by backtest tooling, harmless in live use).
+    pb_funnel: Dict[str, int] = {}
+    # Same, for the chart-pattern route (XAU_CP).
+    cp_funnel: Dict[str, int] = {}
+
     def __init__(self, config: PlatformConfig):
         self.config = config
         self.data_provider = make_data_provider(config)
@@ -174,14 +132,9 @@ class SignalAggregator:
     def analyze_pair(
         self,
         symbol: Optional[str] = None,
-        log_source: Optional[str] = None,
+        context: Optional[FundamentalContext] = None,
     ) -> SignalReport:
-        """Live-mode entry: fetch all needed timeframes then delegate.
-
-        Every report produced here (including weekend/day-filter early
-        returns) is persisted to the signal-history DB via _log_signal —
-        fail-safe, and never reached by the backtester, which calls
-        analyze_from_dataframes directly (live == backtest parity)."""
+        """Live-mode entry: fetch all needed timeframes then delegate."""
         if symbol is None:
             symbol = self.config.symbol
 
@@ -202,7 +155,6 @@ class SignalAggregator:
                 report.trade_setup = None
                 report.warnings.append("WEEKEND FILTER: XAU/USD market closed — no liquidity")
                 report.action_recommendation = "NO TRADE — Weekend (market closed)"
-                self._log_signal(report, log_source, blocked=True)
                 return report
 
             if xau_cfg.allowed_weekdays and now_utc.weekday() not in xau_cfg.allowed_weekdays:
@@ -220,7 +172,6 @@ class SignalAggregator:
                     f"DAY FILTER: {day_names.get(now_utc.weekday(), '?')} not in allowed days ({allowed})"
                 )
                 report.action_recommendation = f"NO TRADE — Only trading {allowed}"
-                self._log_signal(report, log_source, blocked=True)
                 return report
 
         timeframes = [self.config.primary_timeframe] + list(
@@ -245,6 +196,7 @@ class SignalAggregator:
         report = self.analyze_from_dataframes(
             symbol=symbol,
             dfs=dfs,
+            context=context,
             use_news=True,
         )
 
@@ -297,105 +249,8 @@ class SignalAggregator:
                     f"{symbol}: live session filter errored — filter SKIPPED this scan"
                 )
 
-        self._log_signal(report, log_source, dfs)
         return report
 
-    def _log_signal(
-        self, report: SignalReport, source: Optional[str],
-        dataframes: Optional[Dict[Timeframe, pd.DataFrame]] = None,
-        blocked: bool = False,
-    ) -> None:
-        """Persist the scan to the signal-history DB (signal_log.py).
-
-        Fail-safe by contract — a logging failure must never break a
-        live scan. The news context re-fetch rides the NewsEngine's
-        class-level caches, so it costs nothing within a scan and
-        matches the context the signal just used; if it fails, the row
-        is still written, just without the news JSON blobs. ``blocked``
-        marks weekend/day early returns: no news fetch, no snapshots —
-        a ~1 KB liveness row (or nothing, per signal_log.log_blocked_scans).
-        A scan whose PRIMARY fetch failed is logged as ``scan_kind``
-        ``failed`` (price 0, no signal): liveness evidence that readers
-        skip like ``blocked``, so it never pollutes NEUTRAL statistics."""
-        context = None
-        if not blocked:
-            try:
-                context = self._get_news_engine().get_fundamental_context()
-            except Exception:
-                logger.debug("signal log: news context unavailable",
-                             exc_info=True)
-        kind = "blocked" if blocked else "scan"
-        if not blocked and dataframes is not None:
-            primary = dataframes.get(self.config.primary_timeframe)
-            if primary is None or primary.empty:
-                kind = "failed"
-        extra = {"scan_kind": kind}
-        for key, fingerprint in (
-                ("config_fingerprint", self.gate_config_fingerprint),
-                ("engine_fingerprint", self.engine_config_fingerprint)):
-            try:
-                extra[key] = fingerprint()
-            except Exception:
-                logger.debug("signal log: %s unavailable", key, exc_info=True)
-        signal_log.log_signal(
-            report,
-            context=context,
-            source=source or "live",
-            timeframe=self.config.primary_timeframe.value,
-            dataframes=dataframes,
-            extra=extra,
-        )
-
-    def gate_config_fingerprint(self) -> str:
-        """Short hash of the gate configuration this process runs with.
-
-        Logged on every row so two surfaces disagreeing on the same bar
-        (2026-08-19: the dashboard process applied a different blocked-
-        hours list than the bot for a day) is diagnosable from the DB."""
-        xs = self.config.xau_strategy
-        bt = self.config.backtest
-        payload = {
-            "primary": self.config.primary_timeframe.value,
-            "confirm": [tf.value for tf in self.config.confirmation_timeframes],
-            "blocked_hours": list(xs.blocked_hours_utc),
-            "sell_blocked_hours": list(xs.sell_blocked_hours_utc),
-            "allowed_weekdays": list(xs.allowed_weekdays),
-            "blocked_regimes": list(xs.blocked_regimes),
-            "sell_blocked_regimes": list(xs.sell_blocked_regimes),
-            "sell_min_quality": list(xs.sell_min_quality),
-            "sell_vol_regimes": list(xs.sell_vol_regimes),
-            "big_candle": xs.big_candle_atr_multiple,
-            "cp": bool(xs.cp_enabled),
-            "pb": bool(xs.m5_pullback_enabled),
-            "session": [bt.session_filter, bt.session_start_hour,
-                        bt.session_end_hour],
-            "regime_filter": bool(bt.regime_filter),
-        }
-        return hashlib.sha1(
-            json.dumps(payload, sort_keys=True, default=str).encode("utf-8")
-        ).hexdigest()[:12]
-
-    def engine_config_fingerprint(self) -> str:
-        """Short hash of the SCORING configuration this process runs with.
-
-        gate_config_fingerprint covers the post-grade gates; this covers
-        what produced the composite in the first place — indicator
-        parameters, signal thresholds/geometry, the regime weight
-        multipliers actually in force (override included) and the
-        timeframe set. Logged on every row so indicator-weight research
-        can tell which rows share a scoring configuration."""
-        payload = {
-            "technical": asdict(self.config.technical),
-            "signals": asdict(self.config.signals),
-            "regime_weights": regime_weight_table(self.config.technical),
-            "primary": self.config.primary_timeframe.value,
-            "confirm": [tf.value for tf in self.config.confirmation_timeframes],
-            "display_name": self.config.display_name,
-        }
-        return hashlib.sha1(
-            json.dumps(payload, sort_keys=True, default=str).encode("utf-8")
-        ).hexdigest()[:12]
-
     def _apply_regime_filter_live(
         self, report: SignalReport, dfs: Dict[Timeframe, pd.DataFrame]
     ) -> None:
@@ -669,10 +524,16 @@ class SignalAggregator:
     ) -> None:
         """Build an XAU_PB trade setup if a pullback candidate passes gates."""
         cfg = self.config.xau_strategy
+        # Rejection funnel counters (class-level diagnostic, cheap).
+        funnel = SignalAggregator.pb_funnel
+
+        def _reject(reason: str) -> None:
+            funnel[reason] = funnel.get(reason, 0) + 1
 
         pb = self._detect_m5_pullback(report, primary_df)
         if pb is None:
             return
+        _reject("detected")
         direction = pb["direction"]
         is_buy = direction == "BUY"
 
@@ -680,6 +541,7 @@ class SignalAggregator:
         # price mid-pullback scores weaker on the momentum indicators).
         signed = report.final_score if is_buy else -report.final_score
         if signed < cfg.m5_pullback_min_score:
+            _reject("score")
             return
 
         # Confirmation TF must agree (reuse Gate-3 threshold).
@@ -694,6 +556,7 @@ class SignalAggregator:
                 confirmed = True
                 break
         if not confirmed:
+            _reject("confirmation_tf")
             return
 
         # Never against multi-TF consensus (mirror of gate 6f).
@@ -702,13 +565,16 @@ class SignalAggregator:
         bear_n = sum(1 for s in sigs if s in (SignalStrength.SELL, SignalStrength.STRONG_SELL))
         thr = self.config.signals.mtf_agreement_threshold
         if (is_buy and bear_n >= thr) or (not is_buy and bull_n >= thr):
+            _reject("mtf_opposition")
             return
 
         # Volatility gates match the breakout path.
         if report.volatility_regime in cfg.m5_blocked_vol_regimes:
+            _reject("vol_regime")
             return
         if (not is_buy and report.volatility_regime
                 and report.volatility_regime not in cfg.sell_vol_regimes):
+            _reject("sell_vol")
             return
 
         atr = report.primary_analysis.key_levels.get("atr", 0.0)
@@ -729,6 +595,7 @@ class SignalAggregator:
         min_tp = (self.config.signals.round_trip_cost
                   * self.config.signals.min_tp_cost_multiple)
         if 1.5 * sl_dist < min_tp:
+            _reject("spread_gate")
             return
 
         report.trade_setup = TradeSetup(
@@ -744,10 +611,10 @@ class SignalAggregator:
             reward_pips=1.5 * sl_dist / pip_value,
             pip_value=pip_value,
             strategy="XAU_PB",
-            round_trip_cost=self.config.signals.round_trip_cost,
             trail_arm_r=self.config.signals.pb_trail_arm_r,
             trail_dist_r=self.config.signals.pb_trail_dist_r,
         )
+        _reject("accepted")   # funnel tick, mirrors the CP path
         if report.signal_quality not in ("A+", "A", "B"):
             report.signal_quality = "B"
         report.final_signal = (SignalStrength.BUY if is_buy
@@ -774,6 +641,10 @@ class SignalAggregator:
         structural gates plus the validated directional filters below.
         """
         cfg = self.config.xau_strategy
+        funnel = SignalAggregator.cp_funnel
+
+        def _reject(reason: str) -> None:
+            funnel[reason] = funnel.get(reason, 0) + 1
 
         pa = report.primary_analysis
         if pa is None:
@@ -786,6 +657,7 @@ class SignalAggregator:
         pat = detect_chart_pattern(primary_df, atr, cfg)
         if pat is None:
             return
+        _reject("detected")
         direction = pat["direction"]
         is_buy = direction == "BUY"
 
@@ -795,6 +667,7 @@ class SignalAggregator:
         bear_n = sum(1 for s in sigs if s in (SignalStrength.SELL, SignalStrength.STRONG_SELL))
         thr = self.config.signals.mtf_agreement_threshold
         if (is_buy and bear_n >= thr) or (not is_buy and bull_n >= thr):
+            _reject("mtf_opposition")
             return
 
         # Directional protections — gold's macro-uptrend asymmetry is
@@ -803,6 +676,7 @@ class SignalAggregator:
         if not is_buy:
             if (report.volatility_regime
                     and report.volatility_regime not in cfg.sell_vol_regimes):
+                _reject("sell_vol")
                 return
             if cfg.sell_blocked_hours_utc:
                 try:
@@ -818,18 +692,21 @@ class SignalAggregator:
                     close_h = None
                     logger.warning(
                         f"{report.symbol}: bar-close hour underivable — "
-                        f"CP SELL hour gate FAILS CLOSED (short rejected)"
+                        f"SELL hour gate skipped"
                     )
-                if close_h is None or close_h in cfg.sell_blocked_hours_utc:
+                if close_h in cfg.sell_blocked_hours_utc:
+                    _reject("sell_hour")
                     return
             if self.config.primary_timeframe == Timeframe.M5:
                 rsi_ind = pa.indicators.get("rsi")
                 if rsi_ind is not None and rsi_ind.value < cfg.m5_sell_min_rsi:
+                    _reject("m5_sell_rsi")
                     return
 
         # M5 chop-transition volatility block (validated, both ways).
         if (self.config.primary_timeframe == Timeframe.M5
                 and report.volatility_regime in cfg.m5_blocked_vol_regimes):
+            _reject("vol_regime")
             return
 
         entry = report.current_price
@@ -850,10 +727,12 @@ class SignalAggregator:
         # the RR floor or the spread gate, the pattern is untradeable.
         rr = height / sl_dist if sl_dist > 0 else 0.0
         if rr < self.config.signals.min_rr_ratio - 1e-9:
+            _reject("rr")
             return
         min_tp = (self.config.signals.round_trip_cost
                   * self.config.signals.min_tp_cost_multiple)
         if height < min_tp:
+            _reject("spread_gate")
             return
 
         report.trade_setup = TradeSetup(
@@ -869,8 +748,6 @@ class SignalAggregator:
             reward_pips=height / pip_value,
             pip_value=pip_value,
             strategy="XAU_CP",
-            setup_pattern=pat["pattern"],
-            round_trip_cost=self.config.signals.round_trip_cost,
             trail_arm_r=self.config.signals.trail_arm_r,
             trail_dist_r=self.config.signals.trail_dist_r,
         )
@@ -886,77 +763,7 @@ class SignalAggregator:
                   f"height {height / atr:.1f} ATR")
         target = report.bullish_reasons if is_buy else report.bearish_reasons
         target.insert(0, marker)
-        self._clean_accepted_cp_warnings(report)
-
-    @staticmethod
-    def _clean_accepted_cp_warnings(report: SignalReport) -> None:
-        """Drop superseded composite-path warnings after XAU_CP accepts.
-
-        This is metadata-only. Pattern detection, gates, confidence,
-        exhaustion state, setup geometry, and signal selection are already
-        final when this runs.
-        """
-        stale_prefixes = (
-            _SPREAD_GATE_PREFIX,
-            "EXHAUSTION BLOCK",
-            "EXHAUSTED TREND FILTER:",
-            "MTF OPPOSITION:",
-            "SELL FILTER: ",
-            "M5 FILTER: |score|",
-            "M5 FILTER: no MTF agreement",
-            "M5 FILTER: no confirmation TF",
-            "M5 FILTER: SELL quality",
-            "M5 FILTER: SELL blocked",
-        )
-        extension_prefixes = (
-            "TREND EXHAUSTION",
-            "NO PULLBACK",
-            "FAR FROM EMA200",
-            "Extended — RSI",
-            "Weak pullback",
-        )
-        filtered: List[str] = []
-        dropped: List[str] = []
-        for warning in report.warnings:
-            if warning.startswith(stale_prefixes):
-                dropped.append(warning)
-                continue
-            if (warning.startswith("REGIME FILTER: '")
-                    and warning.endswith("blocked for XAU")):
-                dropped.append(warning)
-                continue
-            if (warning.startswith("M5 FILTER: '")
-                    and "volatility" in warning):
-                dropped.append(warning)
-                continue
-            filtered.append(warning)
-
-        extension_items = [
-            (index, warning)
-            for index, warning in enumerate(filtered)
-            if warning.startswith(extension_prefixes)
-        ]
-        if not extension_items:
-            report.warnings[:] = filtered
-            report.cleaned_warnings.extend(dropped)
-            return
-        first_index = extension_items[0][0]
-        priority = {prefix: rank for rank, prefix in enumerate(extension_prefixes)}
-        strongest = min(
-            (warning for _, warning in extension_items),
-            key=lambda warning: next(
-                rank for prefix, rank in priority.items()
-                if warning.startswith(prefix)),
-        )
-        kept = [
-            warning for warning in filtered
-            if not warning.startswith(extension_prefixes)
-        ]
-        kept.insert(first_index, strongest)
-        dropped.extend(
-            warning for _, warning in extension_items if warning != strongest)
-        report.warnings[:] = kept
-        report.cleaned_warnings.extend(dropped)
+        _reject("accepted")
 
     # ── Public: bar-replayable entry point ─────────────────────────
 
@@ -1189,16 +996,11 @@ class SignalAggregator:
                     close_h = None
                     logger.warning(
                         f"{report.symbol}: bar-close hour underivable — "
-                        f"SELL hour gate FAILS CLOSED (short blocked)"
+                        f"SELL hour gate skipped"
                     )
-                # Fail CLOSED: an underivable close hour blocks the short
-                # (`None in [15, 17]` used to pass silently). Never fires
-                # on parseable timestamps, so pinned books are unchanged.
-                if (close_h is None
-                        or close_h in self.config.xau_strategy.sell_blocked_hours_utc):
-                    hour_txt = "??" if close_h is None else f"{close_h:02}"
+                if close_h in self.config.xau_strategy.sell_blocked_hours_utc:
                     report.warnings.append(
-                        f"SELL FILTER: {hour_txt}:00 UTC entry blocked for shorts"
+                        f"SELL FILTER: {close_h:02}:00 UTC entry blocked for shorts"
                     )
                     report.signal_quality = "D"
                     report.trade_setup = None
@@ -1233,7 +1035,16 @@ class SignalAggregator:
             self._apply_m5_filters(report)
 
         # ── 7. Trade setup for actionable signals ──────────────────
-        self._attach_core_setup(report, pip_value)
+        # ATR generator is XAU-only: it hardcodes gold pip value and
+        # 2-decimal price rounding.
+        if (report.trade_setup is None and report.signal_quality in ("A+", "A")
+                and report.display_name == "XAU/USD"):
+            atr = (report.primary_analysis.key_levels.get("atr", 0)
+                   if report.primary_analysis else 0)
+            if atr > 0:
+                report.trade_setup = self._generate_xau_atr_trade_setup(report, atr)
+        if report.trade_setup is None and report.signal_quality in ("A+", "A", "B", "C"):
+            report.trade_setup = self._generate_trade_setup(report, pip_value)
 
         # ── 8. Reasons ─────────────────────────────────────────────
         self._collect_reasons(report)
@@ -1270,60 +1081,11 @@ class SignalAggregator:
                             for w in report.warnings)):
             self._try_chart_pattern(report, primary_df, pip_value)
 
-        # ── 11. Stale spread-gate vetoes ─────────────────────────────
-        # Whichever archetype shipped the setup, a core generator's
-        # cost-floor veto no longer describes it.
-        self._drop_stale_spread_gate_vetoes(report)
-
         return report
 
     # ── Internals ──────────────────────────────────────────────────
 
-    # Only A+/A reach the XAU ATR generator (gate at the call site).
-    _TP_QUALITY_SCALE = {"A+": 2.0, "A": 1.5}
-
-    def _attach_core_setup(self, report: SignalReport, pip_value: float) -> None:
-        """Step 7: the XAU ATR generator (A+/A) first, the swing generator
-        (GENERIC) for anything it declined. A veto either one leaves
-        behind is reconciled after every archetype has run
-        (`_drop_stale_spread_gate_vetoes`)."""
-        # ATR generator is XAU-only: it hardcodes gold pip value and
-        # 2-decimal price rounding.
-        if (report.trade_setup is None and report.signal_quality in ("A+", "A")
-                and report.display_name == "XAU/USD"):
-            atr = (report.primary_analysis.key_levels.get("atr", 0)
-                   if report.primary_analysis else 0)
-            if atr > 0:
-                report.trade_setup = self._generate_xau_atr_trade_setup(report, atr)
-        if report.trade_setup is None and report.signal_quality in ("A+", "A", "B", "C"):
-            report.trade_setup = self._generate_trade_setup(report, pip_value)
-
-    @staticmethod
-    def _drop_stale_spread_gate_vetoes(report: SignalReport) -> None:
-        """Step 11: runs once after every setup archetype (core ATR/swing,
-        XAU_PB, XAU_CP) has had its turn.
-
-        Both core generators veto a target under the cost floor with a
-        `_SPREAD_GATE_PREFIX` warning. When a later generator then ships
-        a setup with a bigger target, that veto is stale — the 2026-09-08
-        07:45 UTC A-grade alert said "costs would eat the edge" beside a
-        $26.49 TP1. A stale veto moves to `cleaned_warnings` (metadata
-        only, the XAU_CP cleanup's contract; that path has already
-        stripped its own copy by the time this runs, and the PB path is
-        covered here). Nothing downstream reads warnings while a setup
-        exists, so trade books are unchanged — verified 2026-09-08 on the
-        3-yr H1/M15 books; only the exported `warnings` text column
-        differs.
-        """
-        if report.trade_setup is None:
-            return
-        kept: List[str] = []
-        stale: List[str] = []
-        for warning in report.warnings:
-            (stale if warning.startswith(_SPREAD_GATE_PREFIX) else kept).append(warning)
-        if stale:
-            report.warnings[:] = kept
-            report.cleaned_warnings.extend(stale)
+    _TP_QUALITY_SCALE = {"A+": 2.0, "A": 1.5, "B": 1.0, "C": 1.0}
 
     def _generate_xau_atr_trade_setup(
         self, report: SignalReport, atr: float
@@ -1340,7 +1102,7 @@ class SignalAggregator:
         else:
             is_buy = report.final_score >= 0
 
-        quality_scale = self._TP_QUALITY_SCALE[report.signal_quality]
+        quality_scale = self._TP_QUALITY_SCALE.get(report.signal_quality, 0.75)
         if report.signal_quality == "A" and report.exhaustion_count >= 2:
             quality_scale = 1.0
         tp_mult = cfg.tp_atr_multiplier * quality_scale
@@ -1372,7 +1134,7 @@ class SignalAggregator:
         min_tp = self.config.signals.round_trip_cost * self.config.signals.min_tp_cost_multiple
         if tp_distance < min_tp:
             report.warnings.append(
-                f"{_SPREAD_GATE_PREFIX} target ${tp_distance:.2f} < ${min_tp:.2f} minimum — "
+                f"SPREAD GATE: target ${tp_distance:.2f} < ${min_tp:.2f} minimum — "
                 f"costs would eat the edge"
             )
             return None
@@ -1389,8 +1151,6 @@ class SignalAggregator:
             risk_pips=round(sl_distance / pip_value, 1),
             reward_pips=round(tp_distance / pip_value, 1),
             pip_value=pip_value,
-            strategy="XAU_ATR",
-            round_trip_cost=self.config.signals.round_trip_cost,
             trail_arm_r=self.config.signals.trail_arm_r,
             trail_dist_r=self.config.signals.trail_dist_r,
         )
@@ -1611,12 +1371,6 @@ class SignalAggregator:
             if total_penalty >= MAX_PENALTY:
                 break
 
-        # Record-keeping for position_sizing / signal-log forensics.
-        # Purely additive — nothing below reads these fields.
-        report.penalty_tags = {name: float(amount) for name, amount in penalties}
-        report.penalty_total = float(total_penalty)
-        report.momentum_fading = bool(momentum_fading)
-
         confidence -= total_penalty
         confidence = max(0.0, min(95.0, confidence))
 
@@ -1791,7 +1545,7 @@ class SignalAggregator:
         min_tp = self.config.signals.round_trip_cost * self.config.signals.min_tp_cost_multiple
         if tp1_distance < min_tp:
             report.warnings.append(
-                f"{_SPREAD_GATE_PREFIX} target ${tp1_distance:.2f} < ${min_tp:.2f} minimum — "
+                f"SPREAD GATE: target ${tp1_distance:.2f} < ${min_tp:.2f} minimum — "
                 f"costs would eat the edge"
             )
             return None
@@ -1811,7 +1565,6 @@ class SignalAggregator:
             risk_pips=risk_pips,
             reward_pips=reward_pips,
             pip_value=pip_value,
-            round_trip_cost=self.config.signals.round_trip_cost,
             trail_arm_r=self.config.signals.trail_arm_r,
             trail_dist_r=self.config.signals.trail_dist_r,
         )

static/dashboard.css

@@ -1,351 +0,0 @@
-/* Gold Desk — XAU/USD dashboard styles
- *
- * Builds on Bootstrap 5.3 color modes (data-bs-theme). Only app-specific
- * tokens and components live here; layout, typography, tables, badges,
- * cards, alerts, modals, etc. come from Bootstrap.
- */
-
-:root {
-  --xau-font-sans: "Inter", system-ui, -apple-system, "Segoe UI", sans-serif;
-  --xau-font-mono: "JetBrains Mono", ui-monospace, "SF Mono", Consolas, Menlo, monospace;
-  --xau-gold: #f5b301;
-}
-
-/* Forced dark theme (owner decision 2026-08-16): dark background, pure
- * white text (no gray), vivid red/green for data accents. Loaded after
- * Bootstrap so these overrides win. */
-[data-bs-theme="dark"] {
-  --bs-body-bg: #0a0e14;
-  --bs-secondary-bg: #111823;
-  --bs-tertiary-bg: #0e141d;
-  --bs-body-color: #ffffff;
-  --bs-emphasis-color: #ffffff;
-  --bs-heading-color: #ffffff;
-  --bs-secondary-color: rgba(255, 255, 255, 0.92);
-  --bs-tertiary-color: rgba(255, 255, 255, 0.8);
-  --bs-border-color: rgba(255, 255, 255, 0.14);
-  --bs-card-bg: #0e141d;
-  --bs-card-border-color: rgba(255, 255, 255, 0.12);
-  --bs-modal-bg: #0e141d;
-  --bs-success: #22c55e;
-  --bs-success-rgb: 34, 197, 94;
-  --bs-success-text-emphasis: #4ade80;
-  --bs-success-bg-subtle: rgba(34, 197, 94, 0.16);
-  --bs-success-border-subtle: rgba(34, 197, 94, 0.45);
-  --bs-danger: #ef4444;
-  --bs-danger-rgb: 239, 68, 68;
-  --bs-danger-text-emphasis: #f87171;
-  --bs-danger-bg-subtle: rgba(239, 68, 68, 0.16);
-  --bs-danger-border-subtle: rgba(239, 68, 68, 0.45);
-}
-
-html, body {
-  font-family: var(--xau-font-sans);
-  letter-spacing: -0.005em;
-  -webkit-font-smoothing: antialiased;
-  background-color: var(--bs-body-bg);
-  color: var(--bs-body-color);
-}
-
-.num, .xau-price, .xau-symbol {
-  font-family: var(--xau-font-mono);
-  font-variant-numeric: tabular-nums;
-}
-
-.xau-symbol {
-  font-size: 0.95rem;
-  font-weight: 700;
-  letter-spacing: 0.08em;
-  color: var(--xau-gold);
-}
-
-.xau-price {
-  line-height: 1;
-  color: #ffffff;
-}
-
-.live-dot {
-  display: inline-block;
-  width: 8px;
-  height: 8px;
-  border-radius: 50%;
-  background: var(--bs-success);
-  margin-right: 5px;
-  animation: pulse 2.4s ease-in-out infinite;
-}
-
-.live-dot.closed {
-  background: var(--bs-warning);
-  animation: none;
-}
-
-@keyframes pulse {
-  50% { opacity: 0.35; }
-}
-
-/* signal-coloured accents */
-.sig-buy   { --xau-signal: var(--bs-success); }
-.sig-sell  { --xau-signal: var(--bs-danger); }
-.sig-neutral { --xau-signal: var(--bs-secondary); }
-
-.border-signal {
-  border-left: 3px solid var(--xau-signal, var(--bs-secondary)) !important;
-}
-
-.text-signal {
-  color: var(--xau-signal, var(--bs-secondary)) !important;
-}
-
-.tf-name {
-  font-family: var(--xau-font-mono);
-  font-weight: 700;
-  font-size: 1.1rem;
-  color: var(--bs-primary);
-  letter-spacing: 0.05em;
-}
-
-.tf-selector {
-  display: grid;
-  grid-template-columns: repeat(4, minmax(0, 1fr));
-  gap: 0.5rem;
-  margin-bottom: 0.75rem;
-}
-
-.tf-select {
-  min-width: 0;
-  padding: 0.7rem 0.8rem;
-  border: 1px solid var(--bs-border-color);
-  border-radius: var(--bs-border-radius);
-  background: var(--bs-tertiary-bg);
-  color: var(--bs-body-color);
-  text-align: left;
-  transition: border-color 0.15s ease, background-color 0.15s ease, box-shadow 0.15s ease;
-}
-
-.tf-select:hover {
-  border-color: rgba(var(--bs-primary-rgb), 0.65);
-}
-
-.tf-select[aria-pressed="true"] {
-  border-color: var(--xau-gold);
-  background: rgba(245, 179, 1, 0.09);
-  box-shadow: inset 0 0 0 1px rgba(245, 179, 1, 0.16);
-}
-
-.tf-select .tf-name {
-  font-size: 1rem;
-}
-
-.workspace-card {
-  overflow: hidden;
-}
-
-.workspace-signal {
-  font-family: var(--xau-font-mono);
-  font-size: clamp(1.55rem, 3vw, 2.35rem);
-  font-weight: 700;
-  letter-spacing: 0.03em;
-  line-height: 1;
-}
-
-.workspace-meta {
-  min-width: 0;
-}
-
-.workspace-reasons > div + div {
-  margin-top: 0.35rem;
-}
-
-/* score bar: zero-centered diverging with midpoint tick */
-.scorebar {
-  display: flex;
-  align-items: center;
-  gap: 0.75rem;
-}
-
-.scorebar .gauge-container {
-  position: relative;
-  flex: 1;
-  min-width: 60px;
-  height: 10px;
-  border-radius: 5px;
-  background: var(--bs-secondary-bg);
-  overflow: hidden;
-}
-
-.scorebar .gauge-container::after {
-  content: "";
-  position: absolute;
-  left: 50%;
-  top: 0;
-  bottom: 0;
-  width: 2px;
-  background: var(--bs-secondary);
-  transform: translateX(-1px);
-}
-
-.scorebar .fill {
-  position: absolute;
-  top: 0;
-  bottom: 0;
-  border-radius: 4px;
-  width: var(--score-width);
-  transition: width 0.25s ease;
-}
-
-.scorebar .fill.pos {
-  left: 50%;
-  background: var(--bs-success);
-}
-
-.scorebar .fill.neg {
-  right: 50%;
-  background: var(--bs-danger);
-}
-
-.scorebar .val {
-  width: 58px;
-  text-align: right;
-  font-size: 0.95rem;
-  font-weight: 500;
-}
-
-.scorebar .lbl {
-  font-size: 0.8rem;
-  color: var(--bs-secondary);
-  width: 44px;
-}
-
-/* rail chips in hero */
-.rail-chip {
-  border: 1px solid var(--bs-border-color);
-  border-radius: var(--bs-border-radius);
-  background: var(--bs-tertiary-bg);
-  color: var(--bs-body-color);
-  font-family: var(--xau-font-mono);
-  font-size: 0.9rem;
-  line-height: 1.3;
-  padding: 0.3rem 0.5rem;
-}
-
-.rail-chip:hover, .rail-chip[aria-current="true"] {
-  border-color: var(--xau-gold);
-}
-
-.rail-chip:focus-visible, .tf-select:focus-visible, .wire-item:focus-visible,
-.cal-event:focus-visible {
-  outline: 2px solid var(--xau-gold);
-  outline-offset: 2px;
-}
-
-/* detail dialog sections */
-.d-stat .k {
-  display: block;
-  font-size: 0.7rem;
-  letter-spacing: 0.08em;
-  text-transform: uppercase;
-  color: var(--bs-secondary);
-  font-weight: 600;
-}
-
-.d-stat .v {
-  font-size: 1.15rem;
-  font-weight: 600;
-}
-
-.d-table {
-  font-size: 0.9rem;
-}
-
-.detail-table {
-  overflow-x: auto;
-}
-
-.d-table .ind-k {
-  font-family: var(--xau-font-mono);
-  font-weight: 600;
-  white-space: nowrap;
-}
-
-.wbar {
-  display: block;
-  height: 4px;
-  border-radius: 2px;
-  background: var(--bs-secondary-bg);
-  margin-top: 3px;
-}
-
-.wbar > i {
-  display: block;
-  height: 100%;
-  border-radius: 2px;
-  background: var(--bs-primary);
-}
-
-/* news */
-.wire-item {
-  display: flex;
-  width: 100%;
-  border: 0;
-  border-radius: var(--bs-border-radius);
-  background: transparent;
-  color: inherit;
-  cursor: pointer;
-  font: inherit;
-  text-align: start;
-}
-
-.wire-item:hover {
-  background: rgba(var(--bs-primary-rgb), 0.08);
-}
-
-.cal-event {
-  display: block;
-  width: 100%;
-  padding: 0;
-  border: 0;
-  background: transparent;
-  color: inherit;
-  text-align: inherit;
-}
-
-.cal-event:hover {
-  color: var(--bs-primary);
-  text-decoration: underline;
-}
-
-.wire-score {
-  font-family: var(--xau-font-mono);
-  font-size: 0.85rem;
-  font-weight: 700;
-  padding: 2px 7px;
-  border-radius: 5px;
-  white-space: nowrap;
-}
-
-/* calendar highlight for next upcoming event */
-.cal-next-up td:first-child {
-  box-shadow: inset 4px 0 0 var(--bs-primary);
-  padding-left: 0.75rem !important;
-}
-
-/* reduced motion */
-@media (prefers-reduced-motion: reduce) {
-  .live-dot { animation: none; }
-  .scorebar .fill { transition: none; }
-  * { transition: none !important; }
-}
-
-@media (max-width: 575.98px) {
-  .tf-selector {
-    grid-template-columns: repeat(2, minmax(0, 1fr));
-  }
-
-  .workspace-card .card-body {
-    padding: 1rem;
-  }
-
-  .rail-chip {
-    flex: 1 1 calc(50% - 0.5rem);
-  }
-}

static/dashboard.js

@@ -1,968 +0,0 @@
-"use strict";
-/* Gold Desk — XAU/USD dashboard frontend
- *
- * Reads snapshot.json + news.json published by dashboard_publisher.py and
- * renders a responsive, keyboard-accessible dashboard. Uses Bootstrap 5 for
- * layout/components/modals and a small custom CSS file for app-specific
- * visuals (score bar, signal colours, mono fonts).
- *
- * This file is intentionally self-contained: it mirrors a few constants from
- * the Python signal pipeline so the HTML does not depend on telegram_bot.
- */
-
-const IND_ORDER = ["ema_trend", "rsi", "macd", "bollinger", "adx", "roc", "keltner", "divergence", "candlestick"];
-const IND_SHORT = {
-  ema_trend: "EMA", rsi: "RSI", macd: "MACD", bollinger: "BB",
-  adx: "ADX", roc: "ROC", keltner: "KC", divergence: "Div", candlestick: "Cndl"
-};
-const ARROW = { STRONG_BUY: "⇈", BUY: "↑", NEUTRAL: "→", SELL: "↓", STRONG_SELL: "⇊" };
-const TF_LABEL = { "5min": "M5", "15min": "M15", "1h": "H1", "4h": "H4" };
-const TF_ORDER = ["5min", "15min", "1h", "4h"];
-const TF_SECONDS = { "5min": 300, "15min": 900, "1h": 3600, "4h": 14400 };
-const FALLBACK_POLL_MS = 60000;
-const RETRY_MS = 15000;
-const HIGH_IMPACT_PREF = "goldDeskHighImpactOnly";
-const TIMEFRAME_PREF = "goldDeskSelectedTimeframe";
-
-const LEVEL_LABEL = {
-  ema_fast: "EMA fast", ema_medium: "EMA medium", ema_slow: "EMA slow", ema_trend: "EMA trend",
-  bb_upper: "Bollinger upper", bb_mid: "Bollinger mid", bb_lower: "Bollinger lower",
-  kc_upper: "Keltner upper", kc_mid: "Keltner mid", kc_lower: "Keltner lower",
-  swing_high_1: "Swing high 1", swing_high_2: "Swing high 2", swing_high_3: "Swing high 3",
-  swing_low_1: "Swing low 1", swing_low_2: "Swing low 2", swing_low_3: "Swing low 3"
-};
-
-let snapshot = null, news = null;
-let clockOffset = 0;
-let lastPollOk = null;
-let lastSnapText = null, lastNewsText = null;
-let refetchTimer = null;
-let fallbackTimer = null;
-let pollInFlight = false;
-let pollQueued = false;
-let selectedTf = "5min";
-
-let detailTf = null;
-let ndState = null;
-const aiMemo = new Map();
-const aiErr = new Map();
-const aiPending = new Set();
-
-let tfModal = null;
-let newsModal = null;
-let tfModalOpen = false;
-let newsModalOpen = false;
-
-const $ = id => document.getElementById(id);
-
-/* ── helpers ──────────────────────────────────────────────────── */
-const esc = s => String(s ?? "").replace(/[&<>"']/g,
-  c => ({"&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;"}[c]));
-
-const fmt$ = v => v == null ? "—" : Number(v).toLocaleString("en-US",
-  { minimumFractionDigits: 2, maximumFractionDigits: 2 });
-
-const sigClass = s => s?.includes("BUY") ? "buy" : s?.includes("SELL") ? "sell" : "neutral";
-const sigLabel = s => (s || "NEUTRAL").replace("_", " ");
-const nowServer = () => Date.now() / 1000 + clockOffset;
-
-function localGet(key) {
-  try { return window.localStorage.getItem(key); } catch { return null; }
-}
-
-function localSet(key, value) {
-  try { window.localStorage.setItem(key, value); } catch { /* storage is optional */ }
-}
-
-function announce(message) {
-  const live = $("dashboard-live");
-  if (live) live.textContent = message;
-}
-
-function fmtCountdown(secs) {
-  if (secs == null || !isFinite(secs)) return "--:--";
-  secs = Math.max(0, Math.floor(secs));
-  const h = Math.floor(secs / 3600), m = Math.floor((secs % 3600) / 60), s = secs % 60;
-  const mm = String(m).padStart(2, "0"), ss = String(s).padStart(2, "0");
-  return h > 0 ? `${h}:${mm}:${ss}` : `${mm}:${ss}`;
-}
-
-function fmtAge(iso) {
-  if (!iso) return "";
-  const secs = nowServer() - Date.parse(iso) / 1000;
-  if (secs < 90) return `${Math.max(0, Math.round(secs))}s ago`;
-  if (secs < 5400) return `${Math.round(secs / 60)}m ago`;
-  if (secs < 172800) return `${Math.round(secs / 3600)}h ago`;
-  return `${Math.round(secs / 86400)}d ago`;
-}
-
-const signed = (v, dp = 2) => v == null ? "—" : (v >= 0 ? "+" : "") + Number(v).toFixed(dp);
-const badgeCls = s => s === "buy" ? "bg-success-subtle text-success-emphasis border-success-subtle"
-                 : s === "sell" ? "bg-danger-subtle text-danger-emphasis border-danger-subtle"
-                 : "bg-secondary-subtle text-secondary-emphasis border-secondary-subtle";
-
-/* ── theme ────────────────────────────────────────────────────── */
-// Dark theme is forced site-wide (owner decision 2026-08-16): dark
-// background, pure-white text, red/green data accents.
-document.documentElement.setAttribute("data-bs-theme", "dark");
-
-/* ── hero ─────────────────────────────────────────────────────── */
-function renderHero() {
-  if (!snapshot) return;
-  let price = null;
-  for (const tf of TF_ORDER) {
-    const p = snapshot.timeframes[tf]?.report?.current_price;
-    if (p) { price = p; break; }
-  }
-  $("hero-price").textContent = price ? fmt$(price) : "—";
-
-  $("hero-rail").innerHTML = TF_ORDER.map(tf => {
-    const r = snapshot.timeframes[tf]?.report;
-    const sig = r?.signal || null;
-    const cls = sigClass(sig);
-    const arrow = sig ? (ARROW[sig] || "→") : "·";
-    const score = r?.score != null ? r.score.toFixed(0) : "–";
-    const textColor = cls === "buy" ? "text-success" : cls === "sell" ? "text-danger" : "text-secondary";
-    return `<button type="button" class="rail-chip" data-select-tf="${tf}"
-      aria-current="${tf === selectedTf ? "true" : "false"}"
-      title="${esc(sigLabel(sig))} (score ${esc(score)}) — select ${TF_LABEL[tf]} workspace">
-      <span class="tf-name">${TF_LABEL[tf]}</span>
-      <span class="${textColor} fw-bold">${arrow}</span>
-      <span>${esc(score)}</span>
-    </button>`;
-  }).join("");
-
-  const closed = snapshot.market_closed;
-  $("hero-status").innerHTML =
-    `<span class="live-dot${closed ? " closed" : ""}"></span>${closed ? "CLOSED" : "OPEN"}`;
-  $("banner-closed").classList.toggle("d-none", !closed);
-}
-
-/* ── market context strip ─────────────────────────────────────── */
-function renderContext() {
-  if (!news) return;
-  $("context").classList.remove("d-none");
-  $("ctx-blurb").innerHTML = news.llm_summary
-    ? `<strong>Market context:</strong> ${esc(news.llm_summary)}`
-    : `<strong>Market context:</strong> <span class="text-secondary">no summary available</span>`;
-
-  let chips = "";
-  for (const cur of ["XAU", "USD"]) {
-    const v = news.sentiment?.[cur];
-    if (v == null) continue;
-    const cls = v > 0.05 ? "text-bg-success" : v < -0.05 ? "text-bg-danger" : "text-bg-secondary";
-    chips += `<span class="badge ${cls}" title="Mean headline sentiment for ${cur} (−1…+1)">
-      ${cur} news <span class="num">${v >= 0 ? "+" : ""}${v.toFixed(2)}</span></span>`;
-  }
-  let risk = "clear", riskDesc = "No significant events";
-  if (snapshot) {
-    for (const tf of TF_ORDER) {
-      const r = snapshot.timeframes[tf]?.report;
-      if (r?.event_risk && r.event_risk !== "clear") { risk = r.event_risk; riskDesc = r.event_description; break; }
-      if (r?.event_description) riskDesc = r.event_description;
-    }
-  }
-  if (news.high_impact_warning) { risk = "danger"; riskDesc = news.warning_message || riskDesc; }
-  const riskCls = risk === "clear" ? "text-bg-success"
-                : risk === "danger" ? "text-bg-danger"
-                : "text-bg-warning";
-  chips += `<span class="badge ${riskCls}" title="${esc(riskDesc)}">event risk: ${risk.toUpperCase()}</span>`;
-  $("ctx-chips").innerHTML = chips;
-}
-
-/* ── timeframe cards ──────────────────────────────────────────── */
-function scoreBar(label, score) {
-  const v = Math.max(-100, Math.min(100, score ?? 0));
-  const width = (Math.abs(v) / 2).toFixed(1);
-  return `<div class="scorebar">
-    <span class="lbl">${label}</span>
-    <div class="gauge-container" aria-hidden="true">
-      <i class="fill ${v >= 0 ? "pos" : "neg"}" style="--score-width:${width}%"></i>
-    </div>
-    <span class="val num ${v > 0 ? "text-success" : v < 0 ? "text-danger" : ""}">${v >= 0 ? "+" : ""}${v.toFixed(1)}</span>
-  </div>`;
-}
-
-function setupCard(ts) {
-  if (!ts) return "";
-  const dir = ts.direction === "BUY" ? "buy" : "sell";
-  const dirIcon = ts.direction === "BUY" ? "bi-arrow-up-circle-fill" : "bi-arrow-down-circle-fill";
-  const trail = (ts.trail_arm_r != null && ts.trail_dist_r != null)
-    ? `arm ${ts.trail_arm_r}R · dist ${ts.trail_dist_r}R` : "platform default";
-  return `<div class="card border-signal sig-${dir} bg-body-tertiary">
-    <div class="card-body py-2 px-3">
-      <div class="d-flex align-items-baseline gap-2 mb-2">
-        <span class="fw-bold text-signal sig-${dir}"><i class="bi ${dirIcon} me-1"></i>${ts.direction}</span>
-        <span class="small text-secondary font-monospace">${esc(ts.strategy || "")}</span>
-        <span class="small text-secondary ms-auto font-monospace">R:R ${(+ts.risk_reward).toFixed(1)}</span>
-      </div>
-      <div class="table-responsive"><table class="table table-sm table-borderless mb-0">
-        <tbody><tr><td class="text-secondary"><i class="bi bi-box-arrow-in-right me-1"></i>Entry</td><td class="num">${fmt$(ts.entry)}</td>
-            <td class="text-secondary ps-4"><i class="bi bi-flag-fill me-1 text-success"></i>TP1</td><td class="num text-success">${fmt$(ts.take_profit_1)}</td></tr>
-        <tr><td class="text-secondary"><i class="bi bi-shield-x me-1 text-danger"></i>Stop</td><td class="num text-danger">${fmt$(ts.stop_loss)}</td>
-            <td class="text-secondary ps-4"><i class="bi bi-flag-fill me-1 text-success"></i>TP2</td><td class="num text-success">${fmt$(ts.take_profit_2)}</td></tr>
-        <tr><td class="text-secondary"><i class="bi bi-signpost-split me-1"></i>Trail</td><td class="num" colspan="3">${esc(trail)}</td></tr></tbody>
-      </table></div>
-    </div>
-  </div>`;
-}
-
-function workspacePanel(tf) {
-  const entry = snapshot?.timeframes?.[tf] || {};
-  const r = entry.report;
-  if (!r) {
-    return `<article class="card workspace-card border-signal sig-neutral" id="workspace-panel" aria-labelledby="workspace-panel-heading">
-      <div class="card-body py-4 text-center text-secondary">
-        <h3 class="tf-name mb-2" id="workspace-panel-heading">${TF_LABEL[tf]}</h3>
-        ${entry.status === "error"
-          ? `<span class="badge text-bg-danger">SCAN UNAVAILABLE</span><div class="small mt-2">Showing again when the next scan succeeds.</div>`
-          : "Warming up — no completed scan for this timeframe yet."}
-      </div>
-    </article>`;
-  }
-
-  const cls = sigClass(r.signal);
-  const textCls = cls === "buy" ? "text-success" : cls === "sell" ? "text-danger" : "text-secondary";
-  const qHi = r.quality === "A+" || r.quality === "A";
-  const pa = r.primary_analysis;
-  const meta = [r.market_regime, r.volatility_regime && r.volatility_regime + " volatility",
-    r.mtf_agreement ? "MTF aligned" : "MTF mixed"].filter(Boolean).join(" · ");
-  const facts = `<div class="row row-cols-2 row-cols-md-4 g-2 mt-1">
-    ${tile("Confidence", r.confidence != null ? Math.round(r.confidence) + "%" : "—")}
-    ${tile("Quality", esc(r.quality || "—"), qHi ? "text-primary" : "")}
-    ${tile("Event risk", esc((r.event_risk || "—").toUpperCase()), r.event_risk === "clear" ? "text-success" : r.event_risk ? "text-warning" : "")}
-    ${tile("MTF", r.mtf_agreement ? "aligned" : "mixed", r.mtf_agreement ? "text-success" : "")}
-  </div>`;
-  const setup = r.trade_setup
-    ? setupCard(r.trade_setup)
-    : `<div class="text-secondary small">No setup published. The signal has not cleared its setup gate.</div>`;
-  return `<article class="card workspace-card border-signal sig-${cls}" id="workspace-panel" aria-labelledby="workspace-panel-heading">
-    <div class="card-body p-3 p-lg-4">
-      <div class="row g-4">
-        <div class="col-lg-7 workspace-meta">
-          <div class="d-flex justify-content-between align-items-start gap-3 flex-wrap mb-3">
-            <div>
-              <div class="d-flex align-items-center gap-2 flex-wrap mb-2">
-                <h3 class="tf-name mb-0" id="workspace-panel-heading">${TF_LABEL[tf]} signal</h3>
-                <span class="badge ${badgeCls(cls)}">${esc(sigLabel(r.signal))}</span>
-                ${r.quality ? `<span class="badge ${qHi ? "text-bg-primary" : "text-bg-secondary"}">quality ${esc(r.quality)}</span>` : ""}
-                ${entry.status === "error" ? `<span class="badge text-bg-danger">STALE</span>` : ""}
-              </div>
-              <div class="workspace-signal ${textCls}">${esc(r.action || sigLabel(r.signal))}</div>
-            </div>
-            <button type="button" class="btn btn-sm btn-outline-primary" data-open-detail="${tf}">
-              Full analysis <i class="bi bi-arrow-up-right ms-1"></i>
-            </button>
-          </div>
-          ${scoreBar("score", pa ? pa.composite_score : r.score)}
-          <div class="small text-secondary mt-2">${esc(meta)}</div>
-          ${facts}
-          <div class="workspace-reasons mt-3 pt-3 border-top">${listBlock(r) || "<div class=\"small text-secondary\">No supporting reasons recorded on this scan.</div>"}</div>
-          <div class="small text-secondary mt-3">${entry.scanned_at ? "scanned " + fmtAge(entry.scanned_at) : ""} · closes in <span class="num" data-close="${entry.next_close_epoch || ""}" data-interval="${TF_SECONDS[tf]}">--:--</span></div>
-        </div>
-        <div class="col-lg-5">
-          <div class="small text-uppercase text-secondary fw-bold mb-2">Trade plan</div>
-          ${setup}
-        </div>
-      </div>
-    </div>
-  </article>`;
-}
-
-function renderCards() {
-  if (!snapshot) return;
-  if (!TF_LABEL[selectedTf]) selectedTf = "5min";
-  const selector = TF_ORDER.map(tf => {
-    const entry = snapshot.timeframes[tf] || {};
-    const r = entry.report;
-    const cls = r ? sigClass(r.signal) : "neutral";
-    const arrow = r ? (ARROW[r.signal] || "→") : "·";
-    const score = r?.score != null ? signed(r.score, 0) : "—";
-    const signal = r ? sigLabel(r.signal) : entry.status === "error" ? "UNAVAILABLE" : "WARMING";
-    return `<button type="button" class="tf-select" data-select-tf="${tf}" aria-pressed="${tf === selectedTf}">
-      <span class="d-flex align-items-center justify-content-between gap-2">
-        <span class="tf-name">${TF_LABEL[tf]}</span>
-        <span class="${cls === "buy" ? "text-success" : cls === "sell" ? "text-danger" : "text-secondary"} fw-bold">${arrow}</span>
-      </span>
-      <span class="small ${cls === "buy" ? "text-success" : cls === "sell" ? "text-danger" : "text-secondary"}">${esc(signal)}</span>
-      <span class="small num float-end">${esc(score)}</span>
-    </button>`;
-  }).join("");
-  $("tf-grid").innerHTML = `<div class="tf-selector">${selector}</div>${workspacePanel(selectedTf)}`;
-  tick();
-}
-
-/* ── timeframe detail dialog ──────────────────────────────────── */
-const tile = (k, v, cls = "") =>
-  `<div class="col"><div class="card d-stat bg-body-tertiary h-100"><div class="card-body p-2">
-    <span class="k">${esc(k)}</span>
-    <span class="v ${cls}">${v}</span>
-  </div></div></div>`;
-
-const SEC_ICON = {
-  "At a glance": "bi-speedometer2",
-  "Trade setup": "bi-bullseye",
-  "Why this signal": "bi-lightbulb",
-  "Multi-timeframe confirmation": "bi-layers",
-  "News context": "bi-newspaper",
-  "AI panel (advisory)": "bi-robot",
-  "Scan detail": "bi-clock-history",
-  "Score": "bi-graph-up",
-  "From the source": "bi-quote",
-  "AI gold-impact summary": "bi-stars",
-  "Timing": "bi-alarm",
-  "Numbers": "bi-calculator"
-};
-const sec = (title, html) => {
-  if (!html) return "";
-  const icon = SEC_ICON[title];
-  return `<section class="mb-4"><h6 class="text-uppercase text-secondary small fw-bold mb-2 d-flex align-items-center gap-2">${icon ? `<i class="bi ${icon} text-primary"></i>` : ""}<span>${esc(title)}</span><span class="flex-fill border-bottom"></span></h6>${html}</section>`;
-};
-
-function indTable(src) {
-  const rows = IND_ORDER.map(name => {
-    const i = src?.[name];
-    const k = `<td class="ind-k">${IND_SHORT[name]}</td>`;
-    if (!i) return `<tr>${k}<td class="text-secondary">— absent</td>
-      <td class="num text-end">—</td><td class="num text-end">—</td><td class="num text-end">—</td>
-      <td class="text-secondary small">no signal this scan</td></tr>`;
-    const w = i.weight == null ? null : +i.weight;
-    return `<tr>${k}
-      <td class="fw-bold ${sigClass(i.signal) === "buy" ? "text-success" : sigClass(i.signal) === "sell" ? "text-danger" : "text-secondary"}">${esc(sigLabel(i.signal))}</td>
-      <td class="num text-end">${i.value != null ? (+i.value).toFixed(2) : "—"}</td>
-      <td class="num text-end">${i.confidence != null ? Math.round(i.confidence) : "—"}</td>
-      <td class="num text-end">${w != null ? w.toFixed(1) : "—"}${w != null
-        ? `<i class="wbar"><i style="width:${Math.min(100, w / 2 * 100)}%"></i></i>` : ""}</td>
-      <td class="text-secondary small">${esc(i.description || "")}</td></tr>`;
-  }).join("");
-  return `<div class="detail-table"><table class="table table-sm d-table"><thead><tr>
-    <th>Indicator</th><th>Signal</th><th class="text-end">Value</th><th class="text-end">Conf</th>
-    <th class="text-end">Weight</th><th>Reading</th></tr></thead><tbody>${rows}</tbody></table></div>`;
-}
-
-function ladder(levels, price) {
-  if (!levels) return "";
-  const atr = levels.atr;
-  const rows = Object.keys(LEVEL_LABEL)
-    .filter(k => levels[k] != null)
-    .map(k => ({ label: LEVEL_LABEL[k], v: +levels[k] }));
-  if (!rows.length) return "";
-  if (price != null) rows.push({ here: true, v: +price });
-  rows.sort((a, b) => b.v - a.v);
-  const body = rows.map(row => {
-    if (row.here) return `<tr class="table-primary fw-bold"><td><i class="bi bi-geo-alt-fill me-1"></i>current price</td>
-      <td class="num text-end">${fmt$(row.v)}</td><td class="num text-end">—</td><td class="small text-secondary">spot</td></tr>`;
-    const d = price != null ? row.v - price : null;
-    const cls = d == null ? "" : d > 0 ? "text-danger" : "text-success";
-    const icon = d == null ? "" : d > 0 ? "bi-arrow-up-short" : "bi-arrow-down-short";
-    return `<tr>
-      <td class="text-secondary">${esc(row.label)}</td>
-      <td class="num text-end ${cls}">${fmt$(row.v)}</td>
-      <td class="num text-end ${cls}">${d == null ? "" : signed(d) + (atr ? ` · ${signed(d / atr, 2)} ATR` : "")}</td>
-      <td class="small ${cls}">${d == null ? "" : `<i class="bi ${icon}"></i>${d > 0 ? "resistance" : "support"}`}</td></tr>`;
-  }).join("");
-  return `<div class="detail-table"><table class="table table-sm ladder"><thead><tr><th>Level</th><th class="text-end">Price</th>
-    <th class="text-end">vs. spot</th><th>Role</th></tr></thead><tbody>${body}</tbody></table></div>`;
-}
-
-function analysisBlock(a, price) {
-  if (!a) return "";
-  const pats = a.pattern_signals?.length
-    ? `<div class="d-flex flex-wrap gap-1 mb-2">${a.pattern_signals.map(p =>
-        `<span class="badge text-bg-primary bg-opacity-10">${esc(p)}</span>`).join("")}</div>` : "";
-  const atr = a.atr_context
-    ? `<div class="small text-secondary mb-2">${esc(a.atr_context.description ||
-        "ATR " + (+a.atr_context.value).toFixed(2))}</div>` : "";
-  const lv = ladder(a.key_levels, price);
-  return indTable(a.indicators) + atr + pats +
-    (lv ? `<div class="small text-secondary fw-semibold mt-2 mb-1">Key levels</div>${lv}` : "");
-}
-
-function listBlock(r) {
-  const group = (title, arr, cls, icon) => arr?.length
-    ? `<div class="small text-secondary fw-semibold mt-2 mb-1">${title} (${arr.length})</div>
-       <div class="small">${arr.map(t => `<div class="${cls}"><i class="bi ${icon} me-1"></i>${esc(t)}</div>`).join("")}</div>` : "";
-  // Violent-move watch: display-only advisory line, stamped post-hoc by
-  // the publisher. Empty unless the feature is enabled.
-  const watch = r.watch_line
-    ? `<div class="small text-warning fw-semibold mt-2">${esc(r.watch_line)}</div>`
-    : "";
-  return watch
-       + group("Warnings", r.warnings, "text-warning", "bi-exclamation-triangle-fill")
-       + group("Bullish", r.bullish_reasons, "text-success", "bi-arrow-up-circle-fill")
-       + group("Bearish", r.bearish_reasons, "text-danger", "bi-arrow-down-circle-fill");
-}
-
-function detailHTML(tf) {
-  const entry = snapshot?.timeframes?.[tf] || {};
-  const r = entry.report;
-  if (!r) return `<div class="text-center text-secondary py-4">${entry.status === "error"
-    ? `Scan error — ${esc(entry.error || "unknown")}`
-    : "Warming up — no completed scan for this timeframe yet."}</div>`;
-
-  const pa = r.primary_analysis;
-  const price = r.current_price;
-  const cls = sigClass(r.signal);
-  const textCls = cls === "buy" ? "text-success" : cls === "sell" ? "text-danger" : "text-secondary";
-  const borderCls = cls === "buy" ? "border-start border-success border-4"
-                  : cls === "sell" ? "border-start border-danger border-4"
-                  : "border-start border-secondary border-4";
-
-  let out = `<div class="card ${borderCls} mb-3"><div class="card-body">
-    <div class="fw-bold fs-5 ${textCls}">${esc(r.action || sigLabel(r.signal))}</div>
-    ${r.news_description || r.event_description
-      ? `<div class="small text-secondary">${esc([r.event_description, r.news_description].filter(Boolean).join(" · "))}</div>`
-      : ""}
-  </div></div>`;
-
-  const ns = r.news_sentiment;
-  out += sec("At a glance", `<div class="row row-cols-2 row-cols-md-3 row-cols-lg-4 g-2">
-    ${tile("Signal", esc(sigLabel(r.signal)), textCls)}
-    ${tile("Score", signed(r.score, 1), textCls)}
-    ${tile("Confidence", r.confidence != null ? Math.round(r.confidence) + "%" : "—")}
-    ${tile("Quality", esc(r.quality || "—"))}
-    ${tile("Regime", esc(r.market_regime || "—"))}
-    ${tile("Volatility", esc(r.volatility_regime || "—"))}
-    ${tile("Event risk", esc((r.event_risk || "—").toUpperCase()),
-        r.event_risk === "clear" ? "text-success" : r.event_risk ? "text-warning" : "")}
-    ${tile("News sentiment", ns != null ? signed(ns, 2) : "—",
-        ns > 0.05 ? "text-success" : ns < -0.05 ? "text-danger" : "")}
-    ${tile("MTF", r.mtf_agreement ? "aligned" : "mixed", r.mtf_agreement ? "text-success" : "")}
-    ${tile("Primary score", signed(r.primary_score, 1), sigClass(r.primary_signal) === "buy" ? "text-success" : sigClass(r.primary_signal) === "sell" ? "text-danger" : "")}
-    ${tile("Exhaustion", esc(r.exhaustion_count ?? "—"), r.exhaustion_count ? "text-warning" : "")}
-  </div>`);
-
-  out += sec("Trade setup", r.trade_setup
-    ? setupCard(r.trade_setup)
-    : `<div class="text-secondary small">No setup published for this timeframe — the engine only emits entry/stop/target once a signal clears its quality gate.</div>`);
-
-  out += sec("Why this signal", listBlock(r) || `<div class="text-secondary small">No reasons recorded on this scan.</div>`);
-
-  out += sec(`${TF_LABEL[tf]} indicators (primary)`,
-    analysisBlock(pa, price) || indTable(r.indicators) +
-      (r.key_levels ? `<div class="small text-secondary fw-semibold mt-2 mb-1">Key levels</div>${ladder(r.key_levels, price)}` : ""));
-
-  const mtf = Object.entries(r.mtf_signals || {});
-  if (mtf.length) {
-    const chips = mtf.map(([k, sig]) => {
-      const s = r.mtf_scores?.[k];
-      const sc = sigClass(sig);
-      return `<span class="badge ${badgeCls(sc)}">${esc(TF_LABEL[k] || k)}
-        <span class="${sc === "buy" ? "text-success" : sc === "sell" ? "text-danger" : "text-secondary"}">${esc(sigLabel(sig))}</span>
-        <span class="num">${s != null ? signed(s, 1) : ""}</span></span>`;
-    }).join("");
-    const confs = (r.confirmation_analyses || []).filter(Boolean).map(a => {
-      const lbl = TF_LABEL[a.timeframe] || a.timeframe;
-      const sc = sigClass(a.composite_signal);
-      return `<details data-dkey="conf-${esc(a.timeframe)}">
-        <summary><strong>${esc(lbl)}</strong>
-          <span class="${sc === "buy" ? "text-success" : sc === "sell" ? "text-danger" : "text-secondary"} fw-bold">${esc(sigLabel(a.composite_signal))}</span>
-          <span class="text-secondary small">${signed(a.composite_score, 1)} · conf ${a.confidence != null ? Math.round(a.confidence) : "—"} · ${esc(a.market_regime || "")}</span></summary>
-        ${analysisBlock(a, price)}</details>`;
-    }).join("");
-    out += sec("Multi-timeframe confirmation",
-      `<div class="d-flex flex-wrap gap-2 align-items-center mb-2">${chips}
-        <span class="fw-bold ${r.mtf_agreement ? "text-success" : "text-secondary"}">${r.mtf_agreement ? "✓ aligned" : "✗ mixed"}</span></div>${confs}`);
-  }
-
-  if (r.news_summary || r.news_description) {
-    out += sec("News context", `<div class="text-secondary small">
-      ${r.news_description ? `<strong>${esc(r.news_description)}</strong>${ns != null ? ` (${signed(ns, 2)})` : ""}<br>` : ""}
-      ${esc(r.news_summary || "")}</div>`);
-  }
-
-  if (r.panel_verdict) {
-    const vcls = r.panel_verdict === "AGREE" ? "text-success"
-      : r.panel_verdict === "OPPOSE" ? "text-danger" : "text-secondary";
-    const takes = (r.panel_takes || []).map(t =>
-      `<tr><td class="ind-k">${esc(t.persona)}</td>
-        <td>${esc(t.stance)}</td>
-        <td class="num text-end">${t.confidence != null ? Math.round(t.confidence) : "—"}</td>
-        <td class="small text-secondary">${esc(t.note || "")}</td></tr>`).join("");
-    out += sec("AI panel (advisory)", `<div class="text-secondary small mb-2">
-      <strong class="${vcls}">${esc(r.panel_verdict)}</strong> ·
-      conf ${Math.round(r.panel_confidence || 0)}%<br>
-      ${esc(r.panel_rationale || "")}</div>
-      ${takes ? `<div class="detail-table"><table class="table table-sm d-table"><tbody>${takes}</tbody></table></div>` : ""}`);
-  }
-
-  out += sec("Scan detail", `<div class="detail-table"><table class="table table-sm d-table"><tbody>
-    <tr><td class="ind-k">Instrument</td><td colspan="5">${esc(r.display_name || r.symbol || "—")}</td></tr>
-    <tr><td class="ind-k">Candle</td><td colspan="5" class="num">${esc(r.timestamp || "—")}</td></tr>
-    <tr><td class="ind-k">Scanned</td><td colspan="5">${esc(entry.scanned_at || "—")}
-      (${fmtAge(entry.scanned_at)}${entry.scan_seconds ? `, took ${entry.scan_seconds}s` : ""})</td></tr>
-    <tr><td class="ind-k">Status</td><td colspan="5">${esc(entry.status || "—")}${entry.error
-      ? ` — <span class="text-danger">${esc(entry.error)}</span>
-          ${entry.error_at ? `(${fmtAge(entry.error_at)})` : ""}` : ""}</td></tr>
-   </tbody></table></div>`);
-  return out;
-}
-
-function renderDetail() {
-  if (!detailTf || !tfModalOpen) return;
-  const entry = snapshot?.timeframes?.[detailTf] || {};
-  const r = entry.report;
-  const openKeys = new Set([...document.querySelectorAll("#detail-body details[open][data-dkey]")]
-    .map(d => d.dataset.dkey));
-  const scroll = $("detail-body").scrollTop;
-
-  $("detail-title").textContent = TF_LABEL[detailTf] || detailTf;
-  $("detail-price").textContent = r?.current_price ? fmt$(r.current_price) : "—";
-  $("detail-sub").innerHTML = `${snapshot?.symbol ? esc(snapshot.symbol) + " · " : ""}closes in
-    <span class="num" data-close="${entry.next_close_epoch || ""}" data-interval="${TF_SECONDS[detailTf]}">--:--</span>`;
-  $("detail-body").innerHTML = detailHTML(detailTf);
-
-  document.querySelectorAll("#detail-body details[data-dkey]").forEach(d => {
-    if (openKeys.has(d.dataset.dkey)) d.open = true;
-  });
-  $("detail-body").scrollTop = scroll;
-  tick();
-}
-
-function openDetail(tf) {
-  if (!TF_LABEL[tf]) return;
-  detailTf = tf;
-  tfModal.show();
-  renderDetail();
-  $("detail-body").scrollTop = 0;
-  $("detail-close").focus();
-}
-
-$("detail-close")?.addEventListener("click", () => tfModal.hide());
-
-/* ── calendar ─────────────────────────────────────────────────── */
-function renderCalendar() {
-  const body = $("cal-body");
-  if (!news) return;
-  const zone = Intl.DateTimeFormat().resolvedOptions().timeZone || "local time";
-  $("calendar-note").textContent = `times local (${zone}) · USD events${news.generated_at ? " · updated " + fmtAge(news.generated_at) : ""}`;
-  const hiOnly = $("hi-only").checked;
-  let events = (news.events || []);
-  if (hiOnly) events = events.filter(e => e.impact === "high");
-  if (!events.length) {
-    body.innerHTML = `<tr><td colspan="7" class="text-secondary text-center py-3">${
-      news.calendar_available ? "No matching events this week." : "Calendar unavailable — source could not be reached."}</td></tr>`;
-    return;
-  }
-  const now = nowServer();
-  let nextUpMarked = false, lastDay = "", rows = "";
-  for (const e of events) {
-    const t = Date.parse(e.time) / 1000;
-    const d = new Date(t * 1000);
-    const day = d.toLocaleDateString(undefined, { weekday: "short", month: "short", day: "numeric" });
-    if (day !== lastDay) {
-      rows += `<tr class="table-group-divider"><td colspan="7" class="text-primary small fw-bold text-uppercase">${esc(day)}</td></tr>`;
-      lastDay = day;
-    }
-    const past = t < now;
-    const nextUp = !past && !nextUpMarked && (nextUpMarked = true);
-    const time = d.toLocaleTimeString(undefined, { hour: "2-digit", minute: "2-digit" });
-    const surp = e.surprise
-      ? `<span class="${e.surprise === "positive" ? "text-success" : e.surprise === "negative" ? "text-danger" : "text-secondary"}" title="actual ${e.surprise === "positive" ? "above" : e.surprise === "negative" ? "below" : "matches"} forecast (numeric only — not gold direction)">${e.surprise === "positive" ? "▲" : e.surprise === "negative" ? "▼" : "="}</span>` : "";
-    rows += `<tr class="${past ? "text-secondary" : ""}${nextUp ? " cal-next-up" : ""}">
-      <td class="num text-nowrap">${esc(time)}</td>
-      <td class="text-secondary">${esc(e.currency)}</td>
-      <td><span class="badge ${e.impact === "high" ? "text-bg-danger" : e.impact === "medium" ? "text-bg-warning" : "text-bg-secondary"}">${e.impact === "medium" ? "Med" : e.impact.charAt(0).toUpperCase() + e.impact.slice(1)}</span></td>
-      <td><button type="button" class="cal-event" data-ev-key="${esc(evKey(e))}" aria-haspopup="dialog">${esc(e.event)}</button></td>
-      <td class="num text-end">${esc(e.forecast ?? "—")}</td>
-      <td class="num text-end">${esc(e.previous ?? "—")}</td>
-      <td class="num text-end">${esc(e.actual ?? "—")} ${surp}</td></tr>`;
-  }
-  body.innerHTML = rows;
-}
-
-/* ── newswire ─────────────────────────────────────────────────── */
-function renderWireTotal() {
-  const box = $("wire-total");
-  const scored = (news?.headlines || []).filter(h => typeof h.score === "number");
-  if (!scored.length) { box.classList.add("d-none"); return; }
-  box.classList.remove("d-none");
-
-  const sum = scored.reduce((a, h) => a + h.score, 0);
-  const mean = sum / scored.length;
-  const up = scored.filter(h => h.score > 0.05).length;
-  const dn = scored.filter(h => h.score < -0.05).length;
-  const cls = mean > 0.05 ? "text-success" : mean < -0.05 ? "text-danger" : "text-secondary";
-  const dir = mean > 0.05 ? "BULLISH" : mean < -0.05 ? "BEARISH" : "NEUTRAL";
-  const wrapCls = mean > 0.05 ? "bg-success-subtle border-success-subtle"
-                : mean < -0.05 ? "bg-danger-subtle border-danger-subtle"
-                : "bg-body-tertiary";
-
-  box.className = `card mb-3 ${wrapCls}`;
-  box.innerHTML = `<div class="card-body p-2">
-    <div class="d-flex align-items-baseline gap-2 flex-wrap">
-      <span class="small text-secondary text-uppercase fw-bold">Total news score</span>
-      <span class="display-6 fw-bold ${cls}">${signed(mean, 2)}</span>
-      <span class="fw-bold ${cls}">${dir}</span>
-      <span class="ms-auto text-end small text-secondary">${scored.length} headlines · sum ${signed(sum, 1)}<br>
-        ${up} bullish · ${dn} bearish</span>
-    </div>
-  </div>`;
-  box.title = `Mean sentiment of all ${scored.length} scored headlines (−1…+1). Sum ${signed(sum, 1)}. Only headlines from the last 4 hours that lean one way are published (neutral ones are removed before scoring).`;
-}
-
-function renderWire() {
-  if (!news) return;
-  const items = (news.headlines || []).slice(0, 40);
-  $("wire-note").textContent = news.generated_at ? `updated ${fmtAge(news.generated_at)}` : "";
-  renderWireTotal();
-  if (!items.length) {
-    $("wire").innerHTML = `<div class="text-secondary small py-2">No gold or dollar headlines in the last 4 hours.</div>`;
-    return;
-  }
-  const item = h => {
-    const s = h.score;
-    const cls = s == null || Math.abs(s) < 0.05 ? "zero" : s > 0 ? "pos" : "neg";
-    const scoreCls = cls === "pos" ? "bg-success-subtle text-success"
-                   : cls === "neg" ? "bg-danger-subtle text-danger"
-                   : "bg-body-tertiary text-secondary";
-    const scoreTxt = s == null ? "·" : (s > 0 ? "+" : "") + s.toFixed(1);
-    let host = "";
-    try { host = new URL(h.source).hostname.replace(/^www\./, ""); } catch {}
-    const content = `<span class="wire-score ${scoreCls}" title="Headline sentiment ${esc(String(s ?? "n/a"))} (−1…+1)${h.currencies?.length ? " · " + esc(h.currencies.join(", ")) : ""}">${scoreTxt}</span>
-       <span class="small text-secondary flex-fill">${esc(h.headline)}</span>
-       <span class="small text-secondary text-nowrap">${esc(h.publisher || host)}${h.timestamp ? " · " + fmtAge(h.timestamp) : ""}</span>`;
-    if (!h.id) return `<div class="wire-item gap-2 align-items-baseline py-1 px-1">${content}</div>`;
-    return `<button type="button" class="wire-item gap-2 align-items-baseline py-1 px-1" data-news-id="${h.id}" aria-haspopup="dialog">
-      ${content}
-    </button>`;
-  };
-  const head = items.slice(0, 12).map(item).join("");
-  const rest = items.slice(12);
-  const moreOpen = $("wire")?.querySelector("details.more")?.open ?? false;
-  $("wire").innerHTML = head + (rest.length
-    ? `<details class="more mt-1"${moreOpen ? " open" : ""}><summary class="small text-secondary">${rest.length} more headlines…</summary>
-       <div class="mt-1">${rest.map(item).join("")}</div></details>` : "");
-}
-
-/* ── news / calendar detail dialog ────────────────────────────── */
-const evKey = e => `${e.time}|${e.currency}|${e.event}`;
-const findHeadline = id => (news?.headlines || []).find(h => h.id === id);
-const findEvent = key => (news?.events || []).find(e => evKey(e) === key);
-
-const AI_ERR_MSG = {
-  unknown_id: "This headline has left the live feed — no summary available.",
-  cap_exceeded: "Daily AI budget reached — try again tomorrow.",
-  no_key: "AI summaries are not configured on this server."
-};
-
-async function fetchAiSummary(id) {
-  aiPending.add(id);
-  try {
-    const res = await fetch("news_summary.php?id=" + encodeURIComponent(id));
-    const data = await res.json().catch(() => null);
-    if (res.ok && data?.summary) aiMemo.set(id, data);
-    else aiErr.set(id, data?.error || `http_${res.status}`);
-  } catch {
-    aiErr.set(id, "network");
-  } finally {
-    aiPending.delete(id);
-    if (ndState?.kind === "wire" && ndState.id === id) renderNewsDlg();
-  }
-}
-
-function scoreTiles(h) {
-  const cls = v => v > 0.05 ? "text-success" : v < -0.05 ? "text-danger" : "text-secondary";
-  let tiles = tile("Overall sentiment", signed(h.score, 2), cls(h.score));
-  const xau = h.score_by_currency?.XAU;
-  if (xau != null) tiles += tile("Gold (XAU) effect", signed(xau, 2), cls(xau));
-  for (const [cur, v] of Object.entries(h.score_by_currency || {})) {
-    if (cur !== "XAU") tiles += tile(cur, signed(v, 2), cls(v));
-  }
-  let out = `<div class="row row-cols-2 row-cols-md-3 g-2">${tiles}</div>`;
-  if (h.keywords?.length) {
-    out += `<div class="small text-secondary mt-2">Matched keywords: ${esc(h.keywords.join(", "))}</div>`;
-  }
-  out += `<div class="small text-secondary">Scores range −1 (bearish) to +1 (bullish); the gold score flips monetary-policy keywords (hawkish = dollar-bullish, gold-bearish).</div>`;
-  return out;
-}
-
-function aiSection(id) {
-  if (aiMemo.has(id)) {
-    const m = aiMemo.get(id);
-    return `<div class="text-secondary small">${esc(m.summary)}</div>` + (m.generated_at
-      ? `<div class="small text-secondary">generated ${fmtAge(m.generated_at)}</div>` : "");
-  }
-  if (aiPending.has(id)) return `<div class="text-secondary small">Generating gold-impact summary…</div>`;
-  if (aiErr.has(id)) {
-    return `<div class="text-secondary small">${esc(AI_ERR_MSG[aiErr.get(id)] || "AI summary unavailable right now.")}</div>`;
-  }
-  return `<div class="text-secondary small">Generating gold-impact summary…</div>`;
-}
-
-function wireDlgHTML() {
-  const fresh = findHeadline(ndState.id);
-  const h = fresh || ndState.snap;
-  if (!h) return "";
-  let host = "";
-  try { host = new URL(h.source).hostname.replace(/^www\./, ""); } catch {}
-  const when = h.timestamp
-    ? `${new Date(h.timestamp).toLocaleString(undefined, { dateStyle: "medium", timeStyle: "short" })} · ${fmtAge(h.timestamp)}` : "";
-  let out = `<div class="fs-5 fw-semibold mb-1">${esc(h.headline)}</div>
-    <div class="small text-secondary mb-3">${esc(h.publisher || host)}${when ? " · " + when : ""}</div>`;
-  if (!fresh) out += `<div class="small text-secondary mb-2">No longer in the live feed — showing saved copy.</div>`;
-  out += sec("Score", scoreTiles(h));
-  if (h.description) out += sec("From the source", `<div class="text-secondary small">${esc(h.description)}</div>`);
-  out += sec("AI gold-impact summary", aiSection(ndState.id));
-  if (h.link && /^https?:\/\//.test(h.link)) {
-    out += `<a class="btn btn-outline-primary" href="${esc(h.link)}" target="_blank" rel="noopener noreferrer">Read full article ↗</a>`;
-  }
-  return out;
-}
-
-function calDlgHTML() {
-  const fresh = findEvent(ndState.key);
-  const e = fresh || ndState.snap;
-  if (!e) return "";
-  const t = Date.parse(e.time) / 1000;
-  const when = new Date(e.time).toLocaleString(undefined, { dateStyle: "full", timeStyle: "short" });
-  const impact = `<span class="badge ${e.impact === "high" ? "text-bg-danger" : e.impact === "medium" ? "text-bg-warning" : "text-bg-secondary"}">${e.impact === "medium" ? "Med" : e.impact.charAt(0).toUpperCase() + e.impact.slice(1)}</span>`;
-  let out = `<div class="fs-5 fw-semibold mb-1">${esc(e.event)}</div>
-    <div class="small text-secondary mb-3">${esc(e.currency)} · ${when}</div>`;
-  const timing = t > nowServer()
-    ? `releases in <span class="num" data-close="${t}">--:--</span>`
-    : `released ${fmtAge(e.time)}`;
-  out += sec("Timing", `<div class="text-secondary small">${impact} impact · ${timing}</div>`);
-  const surpTxt = { positive: "actual above forecast", negative: "actual below forecast", inline: "actual matches forecast" }[e.surprise];
-  out += sec("Numbers", `<div class="row row-cols-3 g-2">${
-      tile("Forecast", esc(e.forecast ?? "—")) +
-      tile("Previous", esc(e.previous ?? "—")) +
-      tile("Actual", esc(e.actual ?? "—"), e.surprise === "positive" ? "text-success" : e.surprise === "negative" ? "text-danger" : "")
-    }</div>` + (surpTxt
-      ? `<div class="small text-secondary mt-2">${surpTxt} (numeric only — not gold direction)</div>` : ""));
-  return out;
-}
-
-function renderNewsDlg() {
-  if (!ndState || !newsModalOpen) return;
-  const scroll = $("nd-body").scrollTop;
-  $("nd-title").textContent = ndState.kind === "wire" ? "Newswire" : "Economic calendar";
-  $("nd-body").innerHTML = ndState.kind === "wire" ? wireDlgHTML() : calDlgHTML();
-  $("nd-body").scrollTop = scroll;
-  tick();
-}
-
-function openNewsDetail(id) {
-  const h = findHeadline(id);
-  if (!h) return;
-  ndState = { kind: "wire", id, snap: h };
-  aiErr.delete(id);
-  newsModal.show();
-  if (!aiMemo.has(id) && !aiPending.has(id)) fetchAiSummary(id);
-  renderNewsDlg();
-  $("nd-body").scrollTop = 0;
-  $("nd-close").focus();
-}
-
-function openCalDetail(key) {
-  const e = findEvent(key);
-  if (!e) return;
-  ndState = { kind: "cal", key, snap: e };
-  newsModal.show();
-  renderNewsDlg();
-  $("nd-body").scrollTop = 0;
-  $("nd-close").focus();
-}
-
-$("nd-close")?.addEventListener("click", () => newsModal.hide());
-
-/* ── polling ──────────────────────────────────────────────────── */
-async function fetchJSON(name) {
-  const res = await fetch(name, { cache: "no-cache" });
-  if (!res.ok) throw new Error(`${name}: HTTP ${res.status}`);
-  const dateHdr = res.headers.get("Date");
-  const text = await res.text();
-  return { text, serverNow: dateHdr ? Date.parse(dateHdr) / 1000 : null };
-}
-
-function tfFingerprint(entry) {
-  if (!entry) return "";
-  return JSON.stringify({
-    status: entry.status,
-    scanned_at: entry.scanned_at,
-    error: entry.error,
-    error_at: entry.error_at,
-    report: entry.report
-  });
-}
-
-function scheduleRefresh(delay) {
-  clearTimeout(refetchTimer);
-  refetchTimer = setTimeout(() => {
-    refetchTimer = null;
-    poll();
-  }, Math.max(0, delay));
-}
-
-function scheduleFallbackRefresh() {
-  clearTimeout(fallbackTimer);
-  fallbackTimer = setTimeout(() => {
-    fallbackTimer = null;
-    poll();
-    scheduleFallbackRefresh();
-  }, FALLBACK_POLL_MS);
-}
-
-function scheduleBoundaryRefresh() {
-  if (!snapshot) return;
-  const nextScan = Math.min(...TF_ORDER.map(tf =>
-    snapshot.timeframes[tf]?.next_scan_epoch || Infinity));
-  if (!isFinite(nextScan)) return;
-  const delay = (nextScan - nowServer() + 2) * 1000;
-  scheduleRefresh(delay > 0 ? delay : RETRY_MS);
-}
-
-async function poll() {
-  if (pollInFlight) {
-    pollQueued = true;
-    return;
-  }
-  pollInFlight = true;
-  try {
-    const [snap, nw] = await Promise.all([
-      fetchJSON("snapshot.json"),
-      fetchJSON("news.json").catch(() => null)
-    ]);
-    const priorSnapshot = snapshot;
-    let snapChanged = false;
-    let selectedChanged = false;
-    let cardsChanged = false;
-    let detailChanged = false;
-    if (snap.text !== lastSnapText) {
-      const incoming = JSON.parse(snap.text);
-      if (!priorSnapshot || incoming.server_time_epoch >= priorSnapshot.server_time_epoch) {
-        snapChanged = true;
-        selectedChanged = tfFingerprint(priorSnapshot?.timeframes?.[selectedTf]) !==
-          tfFingerprint(incoming.timeframes?.[selectedTf]);
-        // renderCards() redraws the selector for EVERY timeframe, so it
-        // must react to any of them changing — not just the selected one.
-        // Gating it on selectedChanged left the other buttons showing a
-        // stale arrow/score (up to 4 h on H4) while renderHero's rail,
-        // which redraws unconditionally, already showed the new value.
-        cardsChanged = TF_ORDER.some(tf =>
-          tfFingerprint(priorSnapshot?.timeframes?.[tf]) !==
-          tfFingerprint(incoming.timeframes?.[tf]));
-        detailChanged = detailTf != null && tfFingerprint(priorSnapshot?.timeframes?.[detailTf]) !==
-          tfFingerprint(incoming.timeframes?.[detailTf]);
-        snapshot = incoming;
-        lastSnapText = snap.text;
-      }
-    }
-    const newsChanged = nw != null && nw.text !== lastNewsText;
-    if (newsChanged) { news = JSON.parse(nw.text); lastNewsText = nw.text; }
-    clockOffset = (snap.serverNow ?? snapshot.server_time_epoch) - Date.now() / 1000;
-    lastPollOk = new Date();
-    $("banner-fail").classList.add("d-none");
-
-    if (snapChanged) {
-      renderHero();
-      if (cardsChanged) renderCards();
-      if (detailChanged) renderDetail();
-      if (selectedChanged) {
-        const signal = snapshot.timeframes[selectedTf]?.report?.signal;
-        if (signal) announce(`${TF_LABEL[selectedTf]} signal updated: ${sigLabel(signal)}.`);
-      }
-    }
-    if (newsChanged) {
-      renderCalendar();
-      renderWire();
-      renderNewsDlg();
-    }
-    if (snapChanged || newsChanged) renderContext();
-    tick();
-    scheduleBoundaryRefresh();
-  } catch (err) {
-    const last = lastPollOk ? ` Last update ${lastPollOk.toLocaleTimeString()}.` : "";
-    const b = $("banner-fail");
-    b.textContent = `Dashboard data is temporarily unavailable.${last} Retrying…`;
-    b.classList.remove("d-none");
-    scheduleRefresh(RETRY_MS);
-  } finally {
-    pollInFlight = false;
-    if (pollQueued) {
-      pollQueued = false;
-      poll();
-    }
-  }
-}
-
-function tick() {
-  if (!snapshot) return;
-  const now = nowServer();
-  document.querySelectorAll("[data-close]").forEach(el => {
-    const t = parseFloat(el.dataset.close);
-    if (!t) return;
-    let secs = t - now;
-    if (secs < 0) {
-      const interval = parseFloat(el.dataset.interval);
-      secs = interval > 0 ? ((secs % interval) + interval) % interval : 0;
-    }
-    el.textContent = fmtCountdown(secs);
-  });
-  const m5 = snapshot.timeframes["5min"]?.next_close_epoch;
-  if (m5) {
-    let secs = m5 - now;
-    if (secs < 0) secs = ((secs % TF_SECONDS["5min"]) + TF_SECONDS["5min"]) % TF_SECONDS["5min"];
-    $("hero-cd").textContent = fmtCountdown(secs);
-  }
-  const pubEpoch = snapshot.server_time_epoch;
-  if (pubEpoch) {
-    const age = now - pubEpoch;
-    const el = $("hero-updated");
-    const stale = !snapshot.market_closed && age > 900;
-    el.textContent = (stale ? "STALE — published " : "published ") +
-      fmtAge(new Date(pubEpoch * 1000).toISOString());
-    el.classList.toggle("text-warning", stale);
-    el.classList.toggle("fw-semibold", stale);
-  }
-}
-
-/* ── boot ─────────────────────────────────────────────────────── */
-document.addEventListener("DOMContentLoaded", () => {
-  tfModal = new bootstrap.Modal($("tf-detail"));
-  newsModal = new bootstrap.Modal($("news-detail"));
-  const storedTf = localGet(TIMEFRAME_PREF);
-  if (TF_LABEL[storedTf]) selectedTf = storedTf;
-  $("hi-only").checked = localGet(HIGH_IMPACT_PREF) === "true";
-
-  $("tf-detail")?.addEventListener("shown.bs.modal", () => { tfModalOpen = true; renderDetail(); });
-  $("tf-detail")?.addEventListener("hidden.bs.modal", () => { tfModalOpen = false; detailTf = null; });
-  $("news-detail")?.addEventListener("shown.bs.modal", () => { newsModalOpen = true; renderNewsDlg(); });
-  $("news-detail")?.addEventListener("hidden.bs.modal", () => { newsModalOpen = false; ndState = null; });
-
-  const selectTf = tf => {
-    if (!TF_LABEL[tf]) return;
-    const changed = selectedTf !== tf;
-    selectedTf = tf;
-    localSet(TIMEFRAME_PREF, tf);
-    renderHero();
-    renderCards();
-    if (changed) announce(`${TF_LABEL[tf]} workspace selected.`);
-  };
-  $("hero-rail")?.addEventListener("click", e => {
-    const button = e.target.closest("[data-select-tf]");
-    if (button) selectTf(button.dataset.selectTf);
-  });
-  $("tf-grid")?.addEventListener("click", e => {
-    const detail = e.target.closest("[data-open-detail]");
-    if (detail) return openDetail(detail.dataset.openDetail);
-    const button = e.target.closest("[data-select-tf]");
-    if (button) selectTf(button.dataset.selectTf);
-  });
-  $("wire")?.addEventListener("click", e => {
-    const button = e.target.closest("[data-news-id]");
-    if (button) openNewsDetail(button.dataset.newsId);
-  });
-  $("cal-body")?.addEventListener("click", e => {
-    const button = e.target.closest("[data-ev-key]");
-    if (button) openCalDetail(button.dataset.evKey);
-  });
-  $("hi-only")?.addEventListener("change", () => {
-    localSet(HIGH_IMPACT_PREF, String($("hi-only").checked));
-    renderCalendar();
-  });
-
-  poll();
-  scheduleFallbackRefresh();
-  setInterval(tick, 1000);
-});

static/news_summary.php

@@ -11,7 +11,7 @@
  *   - CACHE FOREVER: headlines are immutable, so each id costs at most
  *     one DeepSeek call, ever (ai/<id>.json). Cache is checked BEFORE
  *     the whitelist on purpose: summaries stay reachable after the
- *     headline ages out of the 4 h feed window.
+ *     headline ages out of the 24 h feed window.
  *   - DAILY CAP: file-based UTC-day counter, FAIL CLOSED on counter I/O
  *     errors (a perms mistake must not mean unmetered spend).
  *
@@ -51,10 +51,7 @@ In 1-2 plain-English sentences (max 45 words) explain what this news
 means for the GOLD (XAU/USD) price and WHY it is bullish, bearish, or
 neutral. If your reading clearly contradicts xau_score, add "(the
 automated score disagrees)". Never write the words "xau_score" or
-"score" — a general reader sees this text. Write for someone who has
-never traded: short simple sentences, no acronyms or abbreviations
-(say "the US central bank", not "the Fed"; "interest rates", not
-"yields").
+"score" — a general reader sees this text.
 Gold doctrine: hawkish Fed / rate hikes / rising real yields / strong
 dollar are gold-bearish; easing / rate cuts / safe-haven flows /
 geopolitical stress / weak dollar are gold-bullish.
@@ -73,12 +70,7 @@ function respond(int $status, array $obj, string $cache = 'no-store'): void
 function atomic_write(string $path, string $data): bool
 {
     $tmp = $path . '.tmp.' . getmypid();
-    $written = @file_put_contents($tmp, $data);
-    // A SHORT write (disk full / quota) returns a byte count, NOT false.
-    // Renaming that into place would publish a truncated JSON body which
-    // the cache-first branch above then serves forever.
-    if ($written === false || $written !== strlen($data)) {
-        @unlink($tmp);
+    if (@file_put_contents($tmp, $data) === false) {
         return false;
     }
     if (!@rename($tmp, $path)) {
@@ -112,7 +104,9 @@ if (!is_array($news)) {
 }
 $hit = null;
 foreach ($news['headlines'] ?? [] as $h) {
-    $hid = $h['id'] ?? '';
+    // Fallback sha1(): a publisher that predates the id field still
+    // serves a valid whitelist during the restart window.
+    $hid = $h['id'] ?? sha1($h['headline'] ?? '');
     if ($hid === $id) {
         $hit = $h;
         break;
@@ -123,14 +117,44 @@ if ($hit === null) {
                   'message' => 'Headline is no longer in the feed.']);
 }
 
-// ── 4. API key ──────────────────────────────────────────────────────
+// ── 4. daily cap (fail closed) ──────────────────────────────────────
+$dailyCap = (int)(getenv('AI_SUMMARY_DAILY_CAP') ?: 150);
+$capFh = @fopen(CAP_FILE, 'c+');
+if ($capFh === false || !flock($capFh, LOCK_EX)) {
+    respond(503, ['error' => 'cap_io']);
+}
+$today = gmdate('Y-m-d');
+$cap = json_decode((string)stream_get_contents($capFh), true);
+if (!is_array($cap) || ($cap['day'] ?? '') !== $today) {
+    $cap = ['day' => $today, 'count' => 0];
+}
+if ($cap['count'] >= $dailyCap) {
+    flock($capFh, LOCK_UN);
+    fclose($capFh);
+    respond(429, ['error' => 'cap_exceeded']);
+}
+// Increment BEFORE the LLM call — attempts count against the cap
+// (same semantics as llm_client._under_cap).
+$cap['count']++;
+rewind($capFh);
+if (ftruncate($capFh, 0) === false
+        || fwrite($capFh, json_encode($cap)) === false) {
+    flock($capFh, LOCK_UN);
+    fclose($capFh);
+    respond(503, ['error' => 'cap_io']);
+}
+fflush($capFh);
+flock($capFh, LOCK_UN);
+fclose($capFh);
+
+// ── 5. API key ──────────────────────────────────────────────────────
 $keyFile = getenv('DEEPSEEK_KEY_FILE') ?: '/etc/gold-dashboard/deepseek.key';
 $apiKey = trim((string)@file_get_contents($keyFile));
 if ($apiKey === '') {
     respond(503, ['error' => 'no_key']);
 }
 
-// ── 5. per-id stampede lock, then re-check the cache ────────────────
+// ── 6. per-id stampede lock, then re-check the cache ────────────────
 $lockFh = @fopen(CACHE_DIR . '/.ht_lock_' . $id, 'c');
 if ($lockFh !== false) {
     flock($lockFh, LOCK_EX);   // blocks while a concurrent winner generates
@@ -145,51 +169,6 @@ if ($lockFh !== false) {
         exit;
     }
 }
-$releaseLock = function () use (&$lockFh) {
-    if ($lockFh !== false) {
-        @flock($lockFh, LOCK_UN);
-        @fclose($lockFh);
-        $lockFh = false;
-    }
-};
-
-// ── 6. daily cap (fail closed) ──────────────────────────────────────
-// Counted HERE, not before the lock: a burst of clicks on one headline
-// makes exactly one DeepSeek call, and the losers are served from the
-// re-checked cache above — charging them against the budget (and
-// charging a request that then found no key) could exhaust the whole
-// 150/day allowance with zero LLM calls made.
-$dailyCap = (int)(getenv('AI_SUMMARY_DAILY_CAP') ?: 150);
-$capFh = @fopen(CAP_FILE, 'c+');
-if ($capFh === false || !flock($capFh, LOCK_EX)) {
-    $releaseLock();
-    respond(503, ['error' => 'cap_io']);
-}
-$today = gmdate('Y-m-d');
-$cap = json_decode((string)stream_get_contents($capFh), true);
-if (!is_array($cap) || ($cap['day'] ?? '') !== $today) {
-    $cap = ['day' => $today, 'count' => 0];
-}
-if ($cap['count'] >= $dailyCap) {
-    flock($capFh, LOCK_UN);
-    fclose($capFh);
-    $releaseLock();
-    respond(429, ['error' => 'cap_exceeded']);
-}
-// Increment BEFORE the LLM call — attempts count against the cap
-// (same semantics as llm_client._under_cap).
-$cap['count']++;
-rewind($capFh);
-if (ftruncate($capFh, 0) === false
-        || fwrite($capFh, json_encode($cap)) === false) {
-    flock($capFh, LOCK_UN);
-    fclose($capFh);
-    $releaseLock();
-    respond(503, ['error' => 'cap_io']);
-}
-fflush($capFh);
-flock($capFh, LOCK_UN);
-fclose($capFh);
 
 // ── 7. DeepSeek call (exact chat_json body shape) ───────────────────
 $userPayload = json_encode([

telegram_bot.py

@@ -50,7 +50,7 @@ import threading
 import time
 from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime, timezone
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set
+from typing import Dict, List, Optional, Set
 
 from telegram import InlineKeyboardButton, InlineKeyboardMarkup, Update
 from telegram.constants import ParseMode
@@ -68,20 +68,13 @@ from config import (
 )
 from app_config import (
     get_app_config, telegram_token, telegram_chat_id,
-    telegram_channel_id, violent_watch_enabled,
+    telegram_channel_id,
 )
 from providers import BaseDataProvider, make_data_provider
-from agent_panel import attach_panel, panel_health
 from llm_client import get_llm_client
-from market_outlook import maybe_run_outlook
 from news_engine import FundamentalContext, NewsEngine
 from pair_signals import PAIR_PRODUCERS
 from signal_aggregator import SignalAggregator, SignalReport
-from signal_log import gate_reason
-from violent_watch import attach_watch, should_send_headsup
-
-if TYPE_CHECKING:  # type hints only — ViolentWatch is never needed at runtime here
-    from violent_watch import ViolentWatch
 
 _APP = get_app_config()
 
@@ -97,17 +90,13 @@ _log_formatter = logging.Formatter(
 )
 _console_handler = logging.StreamHandler()
 _console_handler.setFormatter(_log_formatter)
-# One file per (server-local) day × 30 backups = 30 days retention.
-# 2026-09-08: was 3-hourly × 240 — a 4-day review meant 18 files. The old
-# `telegram.log.YYYY-MM-DD_HH` files do not match the daily handler's
-# cleanup pattern and need one manual delete on the live box.
-# Handlers sit on the ROOT logger (force=True guarantees ownership even
-# if something logged first), so every module logger — data_provider,
-# news_engine, signal_aggregator, websockets, telegram.ext — lands in the
-# same file.
+# 3-hour rotation × 240 backups = 30 days retention. Handlers sit on the
+# ROOT logger (force=True guarantees ownership even if something logged
+# first), so every module logger — data_provider, news_engine,
+# signal_aggregator, websockets, telegram.ext — lands in the same file.
 _file_handler = logging.handlers.TimedRotatingFileHandler(
     os.path.join(_LOG_DIR, "telegram.log"),
-    when="midnight", backupCount=30, encoding="utf-8",
+    when="h", interval=3, backupCount=240, encoding="utf-8",
 )
 _file_handler.setFormatter(_log_formatter)
 logging.basicConfig(level=getattr(logging, _APP.log_level, logging.INFO),
@@ -193,6 +182,13 @@ TF_CONFIRMATIONS = {
     Timeframe.H4:  [Timeframe.H1, Timeframe.D1],
 }
 
+AUTOSCAN_CONFIRMATIONS = {
+    Timeframe.M5:  [Timeframe.M15, Timeframe.H1],
+    Timeframe.M15: [Timeframe.M5, Timeframe.H1],
+    Timeframe.H1:  [Timeframe.M15, Timeframe.H4],
+    Timeframe.H4:  [Timeframe.H1, Timeframe.D1],
+}
+
 _executor = ThreadPoolExecutor(max_workers=4)
 
 # ── Markdown escaping ─────────────────────────────────────────────────
@@ -204,7 +200,7 @@ def _escape_md(text: str) -> str:
 
 
 def _fmt_price(price: float) -> str:
-    return f"{price:.2f}"
+    return f"{price:.2f}" if price >= 10 else f"{price:.5f}"
 
 
 # ── Shared data provider (single cache across all scans) ─────────────
@@ -343,10 +339,7 @@ def format_signal_message(report: SignalReport) -> str:
         f"📊 Regime: {regime_str} · Vol: {vol_str}",
     ]
     if report.action_recommendation:
-        # Pattern identifiers use underscores, which legacy Telegram Markdown
-        # can misparse even when escaped inside this italicized action line.
-        action = report.action_recommendation.replace("_", " ")
-        lines.append(f"_{_escape_md(action)}_")
+        lines.append(f"_{_escape_md(report.action_recommendation)}_")
 
     if report.trade_setup and sig not in (SignalStrength.NEUTRAL,):
         ts = report.trade_setup
@@ -412,25 +405,8 @@ def format_signal_message(report: SignalReport) -> str:
         extra.append(f"📋 {ind_summary}")
     for w in report.warnings:
         extra.append(f"⚠️ {_escape_md(w)}")
-    # Display-only violent-move watch (violent_watch.attach_watch stamps
-    # it post-hoc; it is empty unless the feature is enabled).
-    if report.watch_line:
-        extra.append(_escape_md(report.watch_line))
     if report.news_summary:
         extra.append(f"📰 {_escape_md(report.news_summary)}")
-    if report.panel_verdict:
-        mark = {"AGREE": "✅", "OPPOSE": "⛔", "NEUTRAL": "➖"}.get(
-            report.panel_verdict, "")
-        panel_label = ("AI macro view"
-                      if report.panel_verdict_source == "macro" else "AI Panel")
-        extra.append(
-            f"🤖 {panel_label} {mark} *{_escape_md(report.panel_verdict)}* "
-            f"({report.panel_confidence:.0f}%) — "
-            f"{_escape_md(report.panel_rationale)}")
-        if report.panel_takes:
-            votes = " · ".join(
-                f"{t['persona']}: {t['stance']}" for t in report.panel_takes)
-            extra.append(f"   _{_escape_md(votes)}_")
     if extra:
         lines += [""] + extra
 
@@ -439,6 +415,37 @@ def format_signal_message(report: SignalReport) -> str:
     return "\n".join(lines)
 
 
+# ── Auto-scan summary format ─────────────────────────────────────────
+
+def _format_scan_summary(reports: List[SignalReport], tf: Timeframe) -> str:
+    now = datetime.now(timezone.utc).strftime("%H:%M UTC")
+    lines = [
+        f"📊 *{tf.value} Auto-Scan* | {now}",
+        "━━━━━━━━━━━━━━━━━━━━",
+    ]
+    for r in reports:
+        emoji = _SIGNAL_EMOJI.get(r.final_signal, "⬜")
+        sig = _signal_label(r.final_signal)
+        mtf = "✅" if r.mtf_agreement else "❌"
+        lines.append(
+            f"{emoji} *{r.display_name}*  {sig}  "
+            f"{_QUALITY_EMOJI.get(r.signal_quality, '')}{r.signal_quality}  "
+            f"{r.final_score:+.1f}  {r.final_confidence:.0f}%  {mtf}MTF"
+        )
+    lines.append("━━━━━━━━━━━━━━━━━━━━")
+
+    actionable = [r for r in reports if r.signal_quality in ("A+", "A", "B", "C")]
+    if actionable:
+        names = ", ".join(r.display_name for r in actionable)
+        lines.append(f"🔔 Actionable: {names}")
+    else:
+        lines.append("No actionable signals this cycle.")
+
+    lines += ["", _DISCLAIMER]
+
+    return "\n".join(lines)
+
+
 # ── Keyboard helper ───────────────────────────────────────────────────
 
 def _scan_button_rows() -> List[List[InlineKeyboardButton]]:
@@ -457,6 +464,10 @@ def _scan_button_rows() -> List[List[InlineKeyboardButton]]:
     return rows
 
 
+def scan_keyboard() -> InlineKeyboardMarkup:
+    return InlineKeyboardMarkup(_scan_button_rows())
+
+
 def main_menu_keyboard() -> InlineKeyboardMarkup:
     """Full interactive menu: scans + settings + status in one keyboard."""
     autoscan_label = "🔔 Auto-scan: ON" if _autoscan_chats else "🔕 Auto-scan: OFF"
@@ -502,12 +513,18 @@ def _run_signal_sync(
             timeframe, [Timeframe.M15, Timeframe.H4]
         )
     # Per-pair tuned configs (regime weights, gates, sessions) come from
-    # the pair_signals factories; every scanned symbol is registered there
-    # (SCAN_PAIRS is validated against PAIR_INFO at import).
-    config = PAIR_PRODUCERS[symbol](timeframe, confirmation_tfs)
+    # the pair_signals factories; a plain PlatformConfig would silently
+    # run every pair with XAU defaults.
+    factory = PAIR_PRODUCERS.get(symbol)
+    if factory is not None:
+        config = factory(timeframe, confirmation_tfs)
+    else:
+        config = PlatformConfig()
+        config.primary_timeframe = timeframe
+        config.confirmation_timeframes = confirmation_tfs
     aggregator = SignalAggregator(config)
     aggregator.data_provider = _get_shared_dp()
-    return aggregator.analyze_pair(symbol=symbol, log_source="telegram")
+    return aggregator.analyze_pair(symbol=symbol)
 
 
 async def run_signal(
@@ -516,50 +533,9 @@ async def run_signal(
     confirmation_tfs: Optional[List[Timeframe]] = None,
 ) -> SignalReport:
     loop = asyncio.get_running_loop()
-    report = await loop.run_in_executor(
+    return await loop.run_in_executor(
         _executor, _run_signal_sync, symbol, timeframe, confirmation_tfs
     )
-    _remember_scan(symbol, timeframe, report)
-    return report
-
-
-# Most recent report per (symbol, timeframe) keyed by the closed-bar epoch
-# it was computed for. Closed-bar inputs are identical within a bar, so a
-# rescan inside the same bar (the WS level-breach handler fired up to
-# ~5 min apart all day, logging 3.5 M15 rows per candle — 2026-09 audit)
-# can reuse it instead of re-fetching, re-scoring and re-logging.
-_last_scan: Dict[str, tuple] = {}
-_TF_SECONDS_CACHE = {
-    Timeframe.M1: 60, Timeframe.M5: 300, Timeframe.M15: 900,
-    Timeframe.M30: 1800, Timeframe.H1: 3600, Timeframe.H4: 14400,
-    Timeframe.D1: 86400,
-}
-
-
-def _bar_epoch(tf: Timeframe, now: Optional[float] = None) -> int:
-    """Epoch of the most recent candle boundary for ``tf`` (UTC-anchored,
-    the same grid _seconds_until_boundary sleeps to)."""
-    secs = _TF_SECONDS_CACHE.get(tf, 900)
-    t = int(time.time() if now is None else now)
-    return t - t % secs
-
-
-def _remember_scan(symbol: str, tf: Timeframe, report: SignalReport) -> None:
-    _last_scan[f"{symbol}_{tf.value}"] = (_bar_epoch(tf), report)
-
-
-async def _cached_or_scan(
-    symbol: str,
-    timeframe: Timeframe,
-    confirmation_tfs: Optional[List[Timeframe]] = None,
-) -> SignalReport:
-    """run_signal, unless this bar already has a report for the key."""
-    cached = _last_scan.get(f"{symbol}_{timeframe.value}")
-    if cached is not None and cached[0] == _bar_epoch(timeframe):
-        logger.info(
-            f"{symbol} {timeframe.value}: reusing this bar's scan — no rescan")
-        return cached[1]
-    return await run_signal(symbol, timeframe, confirmation_tfs)
 
 
 # ── Alert gating (quality threshold + dedup) ─────────────────────────
@@ -580,60 +556,17 @@ def _passes_quality(quality: str) -> bool:
     return _QUALITY_RANK.get(quality, 0) >= _QUALITY_RANK.get(_alert_min_quality, 1)
 
 
-def _is_alert_candidate(report: SignalReport, tf_label: str) -> bool:
-    """An alert is a trade call: non-neutral, grade at/above the
-    threshold, AND a trade setup.
-
-    Until 2026-09-08 the setup was not required, so a signal every
-    generator had declined (spread gate, RR floor, swing geometry) still
-    went out — 07:50 UTC that day sent a B-grade SELL with no setup under
-    a "costs would eat the edge" warning. Skips are logged at INFO so the
-    decision stays visible in telegram.log."""
-    if report.final_signal == SignalStrength.NEUTRAL:
-        return False
-    if not _passes_quality(report.signal_quality):
-        return False
-    if report.trade_setup is None:
-        why = gate_reason(report.warnings, report.cleaned_warnings,
-                          report.signal_quality, False)
-        logger.info(
-            f"alert skipped: {report.display_name} {tf_label} "
-            f"{report.final_signal.name} grade {report.signal_quality} "
-            f"has no trade setup ({why})")
-        return False
-    return True
-
-
-def _alert_sent_line(report: SignalReport, tf_label: str, delivered_to: int,
-                     kind: str = "alert") -> str:
-    """One INFO line per alert that reached a chat. Only failures were
-    logged before 2026-09-08, so that day's log review had to reconstruct
-    what was sent from panel rows and dedup skips."""
-    s = report.trade_setup
-    setup = (f"{s.direction} entry {s.entry_price:.2f} sl {s.stop_loss:.2f} "
-             f"tp1 {s.take_profit_1:.2f} [{s.strategy}]" if s else "no setup")
-    rowid = getattr(report, "_signal_log_rowid", None)
-    return (f"{kind} sent: {report.display_name} {tf_label} "
-            f"{report.final_signal.name} grade {report.signal_quality} — "
-            f"{setup} — signal-log row #{rowid} → {delivered_to} chat(s)")
-
-
-_last_alert_quality: Dict[str, str] = {}   # key → grade of last ALERTED signal
-
-
 def _alert_fingerprint(report: SignalReport) -> tuple:
-    """Alert identity for dedup: signal and setup direction.
+    """Alert identity for dedup: signal, quality, setup direction.
 
     Entry movement is checked separately in _should_alert against the
     last ALERTED entry/risk (a stable reference). Deriving a price bucket
     from the CURRENT SL distance here made the fingerprint churn on
-    normal ATR/anchor drift — near-duplicate alerts every candle. The
-    grade is deliberately NOT part of the identity (2026-09 audit: an
-    A↔B flicker on an unchanged idea re-sent the full alert ~4× in 45
-    min); a grade UPGRADE re-alerts via _should_alert instead."""
+    normal ATR/anchor drift — near-duplicate alerts every candle."""
     setup = report.trade_setup
     return (
         report.final_signal.name,
+        report.signal_quality,
         setup.direction if setup else None,
     )
 
@@ -653,12 +586,6 @@ def _should_alert(key: str, report: SignalReport) -> bool:
         return True
     if time.time() - _last_alert_ts.get(key, 0) >= _ALERT_REARM_SECONDS:
         return True
-    # Same idea at a HIGHER grade than last alerted → worth re-sending;
-    # a downgrade or flicker back is not.
-    last_q = _last_alert_quality.get(key)
-    if last_q is not None and (_QUALITY_RANK.get(report.signal_quality, 0)
-                               > _QUALITY_RANK.get(last_q, 0)):
-        return True
     setup = report.trade_setup
     last = _last_alert_setup.get(key)
     if setup is not None and last is not None:
@@ -671,7 +598,6 @@ def _should_alert(key: str, report: SignalReport) -> bool:
 def _commit_alert(key: str, report: SignalReport) -> None:
     _last_alert_fp[key] = _alert_fingerprint(report)
     _last_alert_ts[key] = time.time()
-    _last_alert_quality[key] = report.signal_quality
     setup = report.trade_setup
     if setup is not None:
         _last_alert_setup[key] = (
@@ -720,17 +646,6 @@ async def _send_alert(bot, chat_id: int, text: str) -> bool:
         return False
 
 
-async def _broadcast(bot, text: str) -> int:
-    """Send `text` to every signal target; returns how many chats got it.
-    Shared by the scan and breach paths so the delivery rule cannot
-    drift between them."""
-    delivered = 0
-    for chat_id in _signal_targets():
-        if await _send_alert(bot, chat_id, text):
-            delivered += 1
-    return delivered
-
-
 # ── Candle-boundary scheduler ────────────────────────────────────────
 
 _autoscan_chats: Set[int] = set()
@@ -767,33 +682,6 @@ def _seconds_until_boundary(interval_minutes: int, buffer: int = 10) -> float:
     return remaining
 
 
-# ── Violent-move watch (display-only) ────────────────────────────────
-# Per-process heads-up budget: {"day": date, "count": int, "sent": {...}}.
-_watch_state: Dict[str, Any] = {}
-
-
-def _format_headsup(report: SignalReport, w: "ViolentWatch") -> str:
-    """Standalone owner heads-up. Advisory only — it never asks anyone
-    to touch an open trade: every protective action tested lost money
-    (sizing_campaign_2026-09.md), so the stop stays the protection."""
-    lines = [
-        f"🌪 *Heads-up — {_escape_md(report.display_name)}*: "
-        f"violent-move conditions for the next hour.",
-        _escape_md(" · ".join(w.triggers)),
-    ]
-    if w.likely_direction == "down":
-        lines.append("If a big move comes, it has usually gone DOWN from "
-                     "readings like these.")
-    elif w.likely_direction == "up":
-        lines.append("If a big move comes, it has usually gone UP from "
-                     "readings like these.")
-    lines.append(
-        "If you have a trade open, expect big bars. Your stop is the "
-        "protection — past tests show closing or tightening early loses "
-        "money.")
-    return "\n".join(lines)
-
-
 async def _scan_timeframe(
     bot,
     tf: Timeframe,
@@ -820,49 +708,11 @@ async def _scan_timeframe(
 
     _update_level_cache(reports)
 
-    # Violent-move watch: DISPLAY-ONLY. Runs before the alert loop so
-    # the stamped line is present when format_signal_message renders,
-    # and can never break an alert (whole block is best-effort).
-    try:
-        vw_settings = get_app_config().violent_watch_settings
-        if violent_watch_enabled():
-            now = datetime.now(timezone.utc)
-            for report in reports:
-                w = attach_watch(report, tf, vw_settings, now)
-                if not (tf == _HEADSUP_TF
-                        and vw_settings.get("headsup", True)
-                        and OWNER_ID is not None):
-                    continue
-                min_level = vw_settings.get("headsup_min_level", "hot")
-                if w.level == "hot" or (min_level == "watch"
-                                        and w.level == "watch"):
-                    ok, why = should_send_headsup(
-                        _watch_state, now, w.kinds,
-                        int(vw_settings.get("max_headsup_per_day", 12)))
-                    if ok:
-                        if await _send_alert(bot, OWNER_ID,
-                                             _format_headsup(report, w)):
-                            logger.info(
-                                f"violent watch: heads-up sent ({w.level}: "
-                                f"{', '.join(w.kinds)}) signal-log row "
-                                f"#{getattr(report, '_signal_log_rowid', None)}")
-                        else:
-                            logger.warning(
-                                "violent watch: heads-up send failed "
-                                "(this hour's slot is consumed)")
-                    else:
-                        logger.info(
-                            f"violent watch: heads-up skipped ({why})")
-                else:
-                    logger.debug(
-                        f"violent watch: {report.symbol} {tf.value} "
-                        f"level={w.level} (min {min_level}) — no heads-up")
-    except Exception:
-        logger.exception("violent watch failed")
-
-    # Same gate as the breach path: non-neutral, grade ≥ threshold, and a
-    # trade setup (_is_alert_candidate logs the no-setup skips).
-    candidates = [r for r in reports if _is_alert_candidate(r, tf.value)]
+    candidates = [
+        r for r in reports
+        if r.final_signal != SignalStrength.NEUTRAL   # same gate as breach path
+        and _passes_quality(r.signal_quality)
+    ]
     for report in candidates:
         key = f"{report.symbol}_{tf.value}"
         # The lock spans check → send → commit so a concurrent breach
@@ -870,42 +720,15 @@ async def _scan_timeframe(
         async with _alert_lock(key):
             if not _should_alert(key, report):
                 continue
-            # Panel AFTER the quality gate + dedup decision so no LLM
-            # spend on suppressed alerts. Runs on the scan executor to
-            # keep the event loop free; this key's alert lock is held
-            # for the panel's wall time (≤~25 s), which only delays a
-            # same-key duplicate that _should_alert would drop anyway.
-            await asyncio.get_running_loop().run_in_executor(
-                _executor, attach_panel, report)
             detail = format_signal_message(report)
-            delivered = await _broadcast(bot, detail)
+            delivered = False
+            for chat_id in _signal_targets():
+                if await _send_alert(bot, chat_id, detail):
+                    delivered = True
             # Commit only when the alert reached at least one target;
             # a fully failed send stays eligible for the next scan.
             if delivered:
                 _commit_alert(key, report)
-                logger.info(_alert_sent_line(report, tf.value, delivered))
-
-    # This independent advisory call runs only after the H1 scan completes.
-    # Offloading the LLM request keeps the event loop responsive.
-    if tf == Timeframe.H1:
-        try:
-            outlook = await asyncio.get_running_loop().run_in_executor(
-                _executor, maybe_run_outlook, "telegram")
-            settings = get_app_config().market_outlook_settings
-            if (outlook is not None and settings.get("telegram", False)
-                    and OWNER_ID is not None):
-                forming = {
-                    "buy": "possible buy forming",
-                    "sell": "possible sell forming",
-                    "none": "no clear setup forming",
-                }[outlook.setup_forming]
-                message = (
-                    f"🔭 AI outlook: {outlook.bias}, {forming} "
-                    f"({outlook.confidence:.0f}%) — {_escape_md(outlook.note)}"
-                )
-                await _send_alert(bot, OWNER_ID, message)
-        except Exception:
-            logger.exception("Hourly AI market outlook failed")
 
 
 # ── WebSocket level-breach monitor ──────────────────────────────────
@@ -1003,7 +826,7 @@ async def _handle_breach(bot, pair_symbol: str, price: float, level_name: str) -
     logger.info(f"Level breach: {display} crossed {label} at {price} — rescanning M15")
 
     try:
-        report = await _cached_or_scan(pair_symbol, Timeframe.M15)
+        report = await run_signal(pair_symbol, Timeframe.M15)
         _update_level_cache([report])
     except Exception:
         logger.exception(f"Breach scan failed for {display}")
@@ -1020,7 +843,8 @@ async def _handle_breach(bot, pair_symbol: str, price: float, level_name: str) -
 
     # Alert only when the rescan produced an actionable, changed signal —
     # same gates as the auto-scanner, so a breach can't spam the channel.
-    if not _is_alert_candidate(report, Timeframe.M15.value):
+    if (report.final_signal == SignalStrength.NEUTRAL
+            or not _passes_quality(report.signal_quality)):
         return
 
     key = f"{pair_symbol}_{Timeframe.M15.value}"
@@ -1031,20 +855,17 @@ async def _handle_breach(bot, pair_symbol: str, price: float, level_name: str) -
         if not _should_alert(key, report):
             return
 
-        # Same post-gate panel attach as the auto-scan path (see
-        # _scan_timeframe for the spend/lock rationale).
-        await asyncio.get_running_loop().run_in_executor(
-            _executor, attach_panel, report)
         header = (
             f"⚡ *LEVEL BREAK* — {_escape_md(display)} crossed "
             f"*{_escape_md(label)}* @ `{_fmt_price(price)}`\n\n"
         )
         detail = header + format_signal_message(report)
-        sent = await _broadcast(bot, detail)
+        sent = False
+        for chat_id in _signal_targets():
+            if await _send_alert(bot, chat_id, detail):
+                sent = True
         if sent:
             _commit_alert(key, report)
-            logger.info(_alert_sent_line(report, Timeframe.M15.value, sent,
-                                         kind="breach alert"))
 
 
 def _breach_task_done(task: asyncio.Task) -> None:
@@ -1155,7 +976,7 @@ async def _candle_loop(bot, tf: Timeframe, interval_minutes: int) -> None:
             logger.info(f"{label} auto-scan: firing")
             try:
                 await _scan_timeframe(
-                    bot, tf, TF_CONFIRMATIONS[tf]
+                    bot, tf, AUTOSCAN_CONFIRMATIONS[tf]
                 )
             except Exception:
                 # One failed cycle must not kill the loop for good —
@@ -1175,15 +996,6 @@ _AUTOSCAN_LOOPS = [
     (tf, _TF_INTERVAL_MINUTES[tf]) for tf in _APP.scan_timeframes
 ]
 
-# The violent-move heads-up is evaluated on the FASTEST scanned loop
-# only — it is a "next hour" warning, so once per fastest candle is the
-# most frequent it can be, and every slower loop would just repeat it.
-_HEADSUP_TF: Optional[Timeframe] = min(
-    _APP.scan_timeframes,
-    key=lambda tf: _TF_INTERVAL_MINUTES.get(tf, 10 ** 6),
-    default=None,
-)
-
 
 _ws_task: Optional[asyncio.Task] = None
 
@@ -1381,10 +1193,6 @@ def _news_health_lines() -> List[str]:
         else:
             lines.append(f"  ✅ DeepSeek: {used}/{cap} calls today,"
                          f" {scored} headlines scored")
-        if get_app_config().agent_panel_settings.get("enabled", False):
-            ph = panel_health()
-            lines.append(f"  🤖 AI panel: {ph['used_today']}/{ph['cap']}"
-                         " panels today")
     return lines
 
 

violent_watch.py

@@ -1,426 +0,0 @@
-"""
-ForexBot2 — violent-move watch (DISPLAY-ONLY heads-up)
-=======================================================
-Flags the conditions under which a "violent bar" (range >= 3x the prior
-ATR14) is unusually likely, so a human reading an alert knows the next
-hour may be wild. It is a WARNING LINE and a standalone heads-up
-message — nothing here ever changes a signal, grade, gate, trade setup
-or backtest result.
-
-Hard independence rule (mirrors market_outlook.py / agent_panel.py):
-this module is NEVER imported by `signal_aggregator`, `engine`, or
-anything under `backtest/`. Surfaces (telegram_bot, main, the dashboard
-publisher) attach the result POST-HOC via `attach_watch`, exactly the
-way panels are attached.
-
-Evidence — 3-year study, backtest/research/sizing_campaign_2026-09.md
-(addenda):
-  * A violent bar is a bar whose range >= 3x the prior ATR14.
-  * Hot clock slots run about 3-4x the normal rate: bars closing
-    12:00-14:59 UTC on weekdays, the 01:xx UTC hour on any weekday, and
-    the Sunday reopen (22:xx-23:xx UTC).
-  * The 30 minutes after a high-impact US release run about 10x normal.
-    Share of releases followed by a violent bar within the hour: NFP
-    97%, CPI 91%, FOMC 88%, retail sales 64%, PPI 54%, PCE 50%,
-    jobless claims 44%.
-  * Indicator readings taken BEFORE the bar each lift the odds 1.3-3x:
-    price outside the Bollinger bands, RSI < 30 or >= 70, close outside
-    the Keltner channel, a rising ADX (slope > 3), volatility regime
-    "extreme", market regime "transitioning", and MACD histogram
-    negative and accelerating down (the engine emits STRONG_SELL for
-    exactly that case).
-  * Only the three PRICE-EXTREME readings (Bollinger, RSI, Keltner)
-    earn a level. A bar-by-bar sweep of the 3-year window showed the
-    first draft — which let any two readings raise the level — flagged
-    27% of all bars "hot" at merely ~2x odds (~13 heads-ups/day). The
-    shipped rule (calendar, or a clock slot plus one price extreme)
-    flags 8.6% of bars: next-bar odds 3-5x normal, ~20% chance within
-    the hour, about 4 hot hours a day. The remaining readings (ADX
-    slope, volatility regime, regime transition, MACD acceleration) are
-    CONTEXT: they are printed once a level exists, and never create one.
-  * Direction: bearish readings preceded a DOWN burst 70-78% of the
-    time on BOTH M5 and M15. Bullish readings preceded an UP burst only
-    ~60% of the time and only on M5 — so an "up" call is made on M5
-    alone.
-  * Open trades are NOT to be touched. Every protective action tested
-    (early close, tightened stop, partial exit) lost money; the stop
-    already IS the protection. The heads-up says so explicitly.
-"""
-
-import logging
-from dataclasses import dataclass, field
-from datetime import datetime, timedelta, timezone
-from typing import Any, Dict, List, Optional, Tuple
-
-from config import SignalStrength, Timeframe
-
-logger = logging.getLogger(__name__)
-
-# Calendar window either side of a high-impact US release (minutes).
-RELEASE_WINDOW_MINUTES = 30
-# Hot clock slots (UTC), from the 3-year bar study.
-US_DATA_HOURS = (12, 13, 14)
-ASIAN_OPEN_HOUR = 1
-SUNDAY_REOPEN_HOURS = (22, 23)
-# ADX slope above this counts as "trend strength rising".
-ADX_SLOPE_HOT = 3.0
-
-_LEVEL_NONE = "none"
-_LEVEL_WATCH = "watch"
-_LEVEL_HOT = "hot"
-
-
-@dataclass
-class ViolentWatch:
-    """One scan's violent-move assessment. Advisory/display-only."""
-    level: str = _LEVEL_NONE          # "none" | "watch" | "hot"
-    triggers: List[str] = field(default_factory=list)   # plain language
-    kinds: List[str] = field(default_factory=list)      # calendar/clock/extreme
-    readings: int = 0                 # PRICE-EXTREME readings (0..3)
-    likely_direction: str = ""        # "down" | "up" | ""
-
-    def line(self) -> str:
-        """One-line alert text; "" when nothing fired."""
-        if self.level == _LEVEL_NONE or not self.triggers:
-            return ""
-        if self.level == _LEVEL_HOT:
-            head = "🌪 Violent-move watch (HOT): "
-            odds = " — odds of a violent bar about 3–5× normal"
-        else:
-            head = "🌪 Violent-move watch (watch): "
-            odds = " — about 2× normal"
-        tail = ""
-        if self.likely_direction == "down":
-            tail = " — if one comes it has usually gone DOWN"
-        elif self.likely_direction == "up":
-            tail = " — if one comes it has usually gone UP"
-        return head + " · ".join(self.triggers) + odds + tail
-
-
-# ── Calendar ─────────────────────────────────────────────────────────
-
-def _is_high_impact_usd(event: Any) -> bool:
-    """USD + high impact, tolerating either the `impact` string or the
-    `is_high_impact` property (EconomicEvent exposes both)."""
-    try:
-        if str(getattr(event, "currency", "")).upper() != "USD":
-            return False
-        impact = getattr(event, "impact", None)
-        if isinstance(impact, str):
-            return impact.strip().lower() == "high"
-        return bool(getattr(event, "is_high_impact", False))
-    except Exception:
-        return False
-
-
-def _calendar_trigger(events: Optional[List[Any]],
-                      now: datetime) -> Optional[str]:
-    """Soonest high-impact USD release inside +/-30 min of `now`."""
-    if not events:
-        return None
-    window = timedelta(minutes=RELEASE_WINDOW_MINUTES)
-    best: Optional[Tuple[float, str]] = None
-    for ev in events or []:
-        if not _is_high_impact_usd(ev):
-            continue
-        when = getattr(ev, "time", None)
-        if not isinstance(when, datetime) or when.tzinfo is None:
-            continue
-        try:
-            delta = when - now
-        except Exception:
-            continue
-        name = str(getattr(ev, "event", "") or "US release").strip()
-        if timedelta(0) < delta <= window:
-            minutes = max(1, int(round(delta.total_seconds() / 60)))
-            text = f"US high-impact release in {minutes} min ({name})"
-            rank = abs(delta.total_seconds())
-        elif -window <= delta < timedelta(0):
-            minutes = max(1, int(round(-delta.total_seconds() / 60)))
-            text = f"US release just out ({name}, {minutes} min ago)"
-            rank = abs(delta.total_seconds())
-        else:
-            continue
-        if best is None or rank < best[0]:
-            best = (rank, text)
-    return None if best is None else best[1]
-
-
-# ── Clock ────────────────────────────────────────────────────────────
-
-def _clock_trigger(now: datetime) -> Optional[str]:
-    """Hot clock slot for `now` (UTC), or None."""
-    try:
-        moment = now.astimezone(timezone.utc) if now.tzinfo else now
-    except Exception:
-        return None
-    weekday, hour = moment.weekday(), moment.hour   # Mon=0 .. Sun=6
-    if weekday <= 4 and hour in US_DATA_HOURS:
-        return "US data hours (12:00–15:00 UTC)"
-    if weekday <= 4 and hour == ASIAN_OPEN_HOUR:
-        return "Asian open hour (01:00 UTC)"
-    if weekday == 6 and hour in SUNDAY_REOPEN_HOURS:
-        return "Sunday reopen"
-    return None
-
-
-# ── Indicator readings ───────────────────────────────────────────────
-
-def _indicator_value(analysis: Any, name: str) -> Optional[float]:
-    try:
-        ind = (analysis.indicators or {}).get(name)
-        if ind is None:
-            return None
-        value = float(ind.value)
-    except Exception:
-        return None
-    if value != value:      # NaN
-        return None
-    return value
-
-
-def _indicator_signal(analysis: Any, name: str) -> Optional[SignalStrength]:
-    try:
-        ind = (analysis.indicators or {}).get(name)
-        return None if ind is None else ind.signal
-    except Exception:
-        return None
-
-
-def _keltner_side(analysis: Any) -> Optional[str]:
-    """"above" / "below" / None — is the close outside the channel?
-
-    The engine's keltner indicator carries `value = close - kc_mid` and
-    stashes kc_upper/kc_lower/kc_mid in `key_levels` (engine.py
-    _compute_keltner). Comparing the value against the band half-widths
-    is therefore exact AND survives the squeeze branches, which
-    overwrite the indicator's signal/description but still publish the
-    same levels. The description text is only a fallback for analyses
-    that were built without key_levels.
-    """
-    value = _indicator_value(analysis, "keltner")
-    levels = getattr(analysis, "key_levels", None) or {}
-    try:
-        upper = levels.get("kc_upper")
-        lower = levels.get("kc_lower")
-        mid = levels.get("kc_mid")
-        if value is not None and None not in (upper, lower, mid):
-            if value > float(upper) - float(mid):
-                return "above"
-            if value < float(lower) - float(mid):
-                return "below"
-            return None
-    except Exception:
-        pass
-    try:
-        desc = str((analysis.indicators or {})["keltner"].description).lower()
-    except Exception:
-        return None
-    if "above keltner upper" in desc:
-        return "above"
-    if "below keltner lower" in desc:
-        return "below"
-    return None
-
-
-def _collect_readings(analysis: Any) -> Tuple[List[str], List[str], int, int]:
-    """(price_extreme_phrases, context_phrases, bearish, bullish).
-
-    Only the PRICE-EXTREME group earns a level (see the module
-    docstring); the context group is printed alongside once a level
-    exists but can never create one. Both groups feed direction.
-    """
-    extreme: List[str] = []
-    context: List[str] = []
-    bear = bull = 0
-    if analysis is None:
-        return extreme, context, bear, bull
-
-    # ── Price extremes (level-bearing) ──
-    bb = _indicator_value(analysis, "bollinger")
-    if bb is not None:
-        if bb < 0:
-            extreme.append("price below the Bollinger bands")
-            bear += 1
-        elif bb > 1:
-            extreme.append("price above the Bollinger bands")
-            bull += 1
-
-    rsi = _indicator_value(analysis, "rsi")
-    if rsi is not None:
-        if rsi < 30:
-            extreme.append(f"RSI {rsi:.0f}, oversold extreme")
-            bear += 1
-        elif rsi >= 70:
-            extreme.append(f"RSI {rsi:.0f}, overbought extreme")
-            bull += 1
-
-    kc_side = _keltner_side(analysis)
-    if kc_side is not None:
-        extreme.append("price outside the Keltner channel")
-        if kc_side == "above":
-            bull += 1
-        else:
-            bear += 1
-
-    # ── Context (never raises the level) ──
-    slope = getattr(analysis, "_adx_slope", None)
-    try:
-        if slope is not None and float(slope) > ADX_SLOPE_HOT:
-            context.append("trend strength rising")
-    except (TypeError, ValueError):
-        pass
-
-    if str(getattr(analysis, "volatility_regime", "")) == "extreme":
-        context.append("volatility extreme")
-
-    regime = str(getattr(analysis, "market_regime", ""))
-    if regime == "transitioning":
-        context.append("regime transitioning")
-
-    macd = _indicator_signal(analysis, "macd")
-    if macd == SignalStrength.STRONG_SELL:
-        context.append("downside momentum accelerating")
-        bear += 1
-    elif macd == SignalStrength.STRONG_BUY:
-        context.append("upside momentum accelerating")
-        bull += 1
-
-    # Regime contributes direction only — it has no phrase of its own
-    # beyond "regime transitioning" above.
-    if regime == "trending_down":
-        bear += 1
-    elif regime == "trending_up":
-        bull += 1
-
-    return extreme, context, bear, bull
-
-
-# ── Assessment ───────────────────────────────────────────────────────
-
-def assess(report: Any,
-           now: datetime,
-           events: Optional[List[Any]] = None,
-           primary_tf: Optional[Timeframe] = None) -> ViolentWatch:
-    """Score violent-move conditions. Every input may be missing or
-    malformed — the answer then degrades to "no trigger". Never raises."""
-    triggers: List[str] = []
-    kinds: List[str] = []
-
-    calendar = _calendar_trigger(events, now)
-    if calendar:
-        triggers.append(calendar)
-        kinds.append("calendar")
-
-    clock = _clock_trigger(now)
-    if clock:
-        triggers.append(clock)
-        kinds.append("clock")
-
-    analysis = getattr(report, "primary_analysis", None)
-    extreme_phrases, context_phrases, bear, bull = _collect_readings(analysis)
-    triggers.extend(extreme_phrases)
-    readings = len(extreme_phrases)
-    if readings >= 1:
-        kinds.append("extreme")
-
-    # Level rule (bar-by-bar validated, see module docstring): only
-    # price extremes and the clock/calendar create a level.
-    if calendar or (clock and readings >= 1):
-        level = _LEVEL_HOT
-    elif clock or readings >= 2:
-        level = _LEVEL_WATCH
-    else:
-        level = _LEVEL_NONE
-
-    # Context readings are colour, printed only once a level exists.
-    if level != _LEVEL_NONE:
-        triggers.extend(context_phrases)
-
-    direction = ""
-    if bear >= 2 and bear > bull:
-        direction = "down"
-    elif bull >= 2 and bull > bear:
-        # Bullish readings only earned a call on M5 in the study.
-        tf_value = getattr(primary_tf, "value", primary_tf)
-        if tf_value == Timeframe.M5.value:
-            direction = "up"
-
-    return ViolentWatch(level=level, triggers=triggers, kinds=kinds,
-                        readings=readings, likely_direction=direction)
-
-
-def _log_watch(report: Any, watch: "ViolentWatch") -> None:
-    """Persist the assessment onto the scan's signals.db row (fail-safe).
-
-    The row is inserted inside analyze_pair — BEFORE a surface attaches
-    the watch — so these columns arrive by post-hoc update, exactly like
-    the panel verdict. Level and kinds are written even when the display
-    line is off: evaluating the watch needs the assessment, not the text.
-    """
-    try:
-        from signal_log import log_watch_update
-        log_watch_update(report, watch)
-    except Exception:
-        logger.debug("violent watch: logging failed", exc_info=True)
-
-
-def attach_watch(report: Any,
-                 primary_tf: Optional[Timeframe],
-                 settings: Dict[str, Any],
-                 now: Optional[datetime] = None) -> ViolentWatch:
-    """Stamp `report.watch_line` with the display line (advisory only).
-
-    Contract: NEVER raises. A broken watch must not cost an alert send
-    or a dashboard publish — a failure logs and returns level "none".
-    """
-    try:
-        settings = settings or {}
-        if not settings.get("enabled", False):
-            return ViolentWatch()
-        moment = now or datetime.now(timezone.utc)
-        events: Optional[List[Any]] = None
-        try:
-            # Lazy import keeps this module import-light and free of any
-            # signal-path dependency. The calendar rides NewsEngine's
-            # class-level cache, so a warm cache costs no network I/O.
-            from config import PlatformConfig
-            from news_engine import NewsEngine
-            events = NewsEngine(PlatformConfig().news).get_week_events(["USD"])
-        except Exception as exc:
-            logger.debug("violent watch: calendar unavailable (%s)", exc)
-        watch = assess(report, moment, events, primary_tf)
-        if settings.get("alert_line", True):
-            report.watch_line = watch.line()
-        _log_watch(report, watch)
-        return watch
-    except Exception:
-        logger.exception("violent watch: assessment failed")
-        return ViolentWatch()
-
-
-def should_send_headsup(state: Dict[str, Any],
-                        now: datetime,
-                        kinds: List[str],
-                        max_per_day: int) -> Tuple[bool, str]:
-    """Rate-limit the standalone heads-up message.
-
-    `state` is a mutable dict owned by the caller:
-        {"day": date, "count": int, "sent": {(hour_iso, kinds): True}}
-    One message per UTC hour per trigger combination, capped per UTC day.
-    """
-    day = now.date()
-    if state.get("day") != day:
-        state["day"] = day
-        state["count"] = 0
-        state["sent"] = {}
-    sent = state.setdefault("sent", {})
-    if int(state.get("count", 0)) >= int(max_per_day):
-        return False, "daily cap"
-    hour_bucket = now.replace(minute=0, second=0, microsecond=0).isoformat()
-    key = (hour_bucket, tuple(sorted(kinds)))
-    if key in sent:
-        return False, "already sent this hour for these triggers"
-    sent[key] = True
-    state["count"] = int(state.get("count", 0)) + 1
-    return True, "ok"