ForexBot2/advisory_outcome.py

"""Deterministic advisory trade outcomes from stored closed candles.

This module measures the hypothetical setup reviewed by the LLM panel. It
never changes a signal, panel verdict, order, or daemon state. The model uses
the same next-open reanchoring and close-only trailing rules as the backtester,
but its evidence is the cleaned mid-candle snapshots stored in signals.db.
"""

from __future__ import annotations

import logging
import math
from datetime import datetime, timedelta
from typing import Any, Dict, Iterable, List, Optional, Tuple
from zoneinfo import ZoneInfo

logger = logging.getLogger(__name__)

MODEL = "stored-closed-mid-next-open-trail-v1"
MAX_HOLD_BARS = 300
REFERENCE_COST = 0.30
REALISTIC_COST = 0.65

_TF_ALIASES = {
    "M1": "1min", "1MIN": "1min", "1min": "1min",
    "M5": "5min", "5MIN": "5min", "5min": "5min",
    "M15": "15min", "15MIN": "15min", "15min": "15min",
    "M30": "30min", "30MIN": "30min", "30min": "30min",
    "H1": "1h", "1H": "1h", "1h": "1h",
    "H4": "4h", "4H": "4h", "4h": "4h",
    "D1": "1day", "1D": "1day", "1d": "1day",
    "1DAY": "1day", "1day": "1day",
}

_TF_SECONDS = {
    "1min": 60,
    "5min": 5 * 60,
    "15min": 15 * 60,
    "30min": 30 * 60,
    "1h": 60 * 60,
    "4h": 4 * 60 * 60,
    "1day": 24 * 60 * 60,
}

_NY_TZ = ZoneInfo("America/New_York")


def _is_forex_weekend(now: datetime) -> bool:
    """NY-anchored XAU weekend closure; test-locked to the signal copies."""
    ny = now.astimezone(_NY_TZ)
    weekday = ny.weekday()
    if weekday == 4:
        return ny.hour >= 17
    if weekday == 5:
        return True
    if weekday == 6:
        return ny.hour < 18
    return False


def normalize_timeframe(value: Any) -> str:
    text = str(getattr(value, "value", value) or "")
    return _TF_ALIASES.get(text, _TF_ALIASES.get(text.upper(), text))


def timeframe_value(mapping: Any, timeframe: Any) -> Any:
    """Read a timeframe-keyed snapshot regardless of enum/storage alias."""
    if not isinstance(mapping, dict):
        return None
    wanted = normalize_timeframe(timeframe)
    for key, value in mapping.items():
        if normalize_timeframe(key) == wanted:
            return value
    return None


def _finite(value: Any) -> Optional[float]:
    try:
        number = float(value)
    except (TypeError, ValueError):
        return None
    return number if math.isfinite(number) else None


def _timestamp(value: Any) -> Optional[datetime]:
    try:
        return datetime.fromisoformat(str(value).replace(" ", "T"))
    except (TypeError, ValueError):
        return None


def initial_outcome(
    setup: Optional[Dict[str, Any]],
    indicators: Optional[Dict[str, Any]],
    timeframe: Any,
) -> Tuple[str, Dict[str, Any]]:
    """Freeze the setup and deterministic evaluation policy."""
    if not setup:
        return "NOT_APPLICABLE", {
            "model": MODEL,
            "status": "NOT_APPLICABLE",
            "reason": "panel row has no trade setup",
        }

    tf = normalize_timeframe(timeframe)
    analysis = timeframe_value(indicators, tf)
    signal_bar = analysis.get("timestamp") if isinstance(analysis, dict) else None
    direction = str(setup.get("direction", ""))
    entry = _finite(setup.get("entry"))
    stop = _finite(setup.get("stop_loss"))
    tp1 = _finite(setup.get("take_profit_1"))
    arm = _finite(setup.get("trail_arm_r"))
    distance = _finite(setup.get("trail_dist_r"))
    risk = abs(entry - stop) if entry is not None and stop is not None else 0.0
    if (direction not in ("BUY", "SELL") or not signal_bar
            or _timestamp(signal_bar) is None or risk <= 0 or tp1 is None
            or arm is None or distance is None or arm < 0 or distance < 0):
        return "UNRESOLVABLE", {
            "model": MODEL,
            "status": "UNRESOLVABLE",
            "reason": "invalid or incomplete setup/outcome seed",
        }

    state = {
        "model": MODEL,
        "status": "PENDING",
        "timeframe": tf,
        "signal_bar_at": str(signal_bar),
        "setup": {
            "direction": direction,
            "reported_entry": entry,
            "stop_distance": risk,
            "tp1_distance": abs(tp1 - entry),
            "strategy": setup.get("strategy"),
            "setup_pattern": setup.get("setup_pattern"),
        },
        "policy": {
            "entry": "next_primary_bar_open_reanchored",
            "max_hold_bars": MAX_HOLD_BARS,
            "trail_arm_r": arm,
            "trail_dist_r": distance,
            "tp1_is_advisory": arm > 0,
            "reference_cost": REFERENCE_COST,
            "realistic_cost": REALISTIC_COST,
        },
        "progress": {
            "last_processed_bar": str(signal_bar),
            "entry_at": None,
            "entry_price": None,
            "initial_stop": None,
            "current_stop": None,
            "peak": None,
            "bars_held": 0,
            "mfe_r": 0.0,
            "mae_r": 0.0,
        },
        "result": None,
    }
    return "PENDING", state


def _valid_bars(bars: Iterable[Dict[str, Any]]) -> List[Dict[str, Any]]:
    valid = []
    for bar in bars:
        stamp = _timestamp(bar.get("timestamp")) if isinstance(bar, dict) else None
        values = [_finite(bar.get(key)) for key in ("open", "high", "low", "close")]
        if stamp is None or any(value is None for value in values):
            continue
        valid.append({
            "timestamp": str(bar["timestamp"]),
            "_timestamp": stamp,
            "open": values[0], "high": values[1],
            "low": values[2], "close": values[3],
        })
    valid.sort(key=lambda item: item["_timestamp"])
    return valid


def _next_bar_follows(
    previous: datetime, stamp: datetime, step: timedelta
) -> bool:
    """True when ``stamp`` is the next tradable bar after ``previous``: every
    bar slot between the two lies inside a scheduled XAU closure (the weekend
    or the daily 17:00 New York maintenance hour) and ``stamp`` is on the bar
    grid. ``step`` must be positive."""
    expected = previous + step
    while expected < stamp:
        ny = expected.astimezone(_NY_TZ)
        daily_maintenance = ny.weekday() < 4 and ny.hour == 17
        if not (_is_forex_weekend(expected) or daily_maintenance):
            return False
        expected += step
    return expected == stamp


def _has_continuity(
    bars: List[Dict[str, Any]], last: datetime, timeframe: str
) -> bool:
    """Reject missing tradable bars while allowing scheduled XAU closures.

    All-or-nothing over the whole of ``bars`` (the scan-outcome labeller
    calls it one bar at a time). advance_outcome does not use it: it checks
    each step as it walks, so a hole AFTER a trade's exit cannot void it."""
    seconds = _TF_SECONDS.get(normalize_timeframe(timeframe))
    if seconds is None:
        return False
    step = timedelta(seconds=seconds)
    previous = last
    seen = {last}
    for bar in bars:
        stamp = bar["_timestamp"]
        if stamp <= previous or stamp in seen:
            continue
        if not _next_bar_follows(previous, stamp, step):
            return False
        previous = stamp
        seen.add(stamp)
    return True


def _finish(
    state: Dict[str, Any], reason: str, exit_at: str, exit_price: float
) -> None:
    setup = state["setup"]
    progress = state["progress"]
    entry = float(progress["entry_price"])
    risk = float(setup["stop_distance"])
    pnl = ((exit_price - entry) if setup["direction"] == "BUY"
           else (entry - exit_price))
    gross_r = pnl / risk
    state["status"] = "RESOLVED"
    state["result"] = {
        "exit_at": exit_at,
        "exit_price": exit_price,
        "exit_reason": reason,
        "bars_held": progress["bars_held"],
        "gross_r": gross_r,
        "net_r_030": (pnl - REFERENCE_COST) / risk,
        "net_r_065": (pnl - REALISTIC_COST) / risk,
        "mfe_r": progress["mfe_r"],
        "mae_r": progress["mae_r"],
    }


def advance_outcome(
    state: Dict[str, Any], bars: Iterable[Dict[str, Any]]
) -> Dict[str, Any]:
    """Advance one pending outcome with overlapping closed-bar snapshots."""
    if state.get("status") != "PENDING":
        return state
    try:
        progress = state["progress"]
        setup = state["setup"]
        policy = state["policy"]
        last = _timestamp(progress["last_processed_bar"])
        signal_bar = _timestamp(state["signal_bar_at"])
        if last is None or signal_bar is None:
            raise ValueError("invalid progress timestamp")

        buy = setup["direction"] == "BUY"
        risk = float(setup["stop_distance"])
        trail_arm = float(policy["trail_arm_r"])
        trail_distance = float(policy["trail_dist_r"])

        valid_bars = _valid_bars(bars)
        has_future = any(bar["_timestamp"] > last for bar in valid_bars)
        has_overlap = any(bar["_timestamp"] == last for bar in valid_bars)
        if has_future and not has_overlap:
            state["status"] = "UNRESOLVABLE"
            state["result"] = None
            state["reason"] = "stored candle snapshots lost continuity"
            return state
        if not has_future:
            return state
        seconds = _TF_SECONDS.get(normalize_timeframe(state["timeframe"]))
        if seconds is None:
            state["status"] = "UNRESOLVABLE"
            state["result"] = None
            state["reason"] = "stored candle snapshots contain an internal gap"
            return state
        step = timedelta(seconds=seconds)

        for bar in valid_bars:
            stamp = bar["_timestamp"]
            if stamp <= signal_bar or stamp <= last:
                continue

            # Continuity is judged ONE STEP AT A TIME, only as far as the
            # trade is still open. Until 2026-10-01 the whole stream was
            # checked before the walk, so a trade that had exited days
            # before a later hole (a holiday, an outage) was thrown away:
            # 9 of 111 setups in the September log, 7 of them with an exit
            # the live resolver had recorded. A hole reached while the
            # trade is open still ends the walk; the progress made up to
            # the bar before it stays, so resolve_locked_floor floors from
            # the latest stop.
            if not _next_bar_follows(last, stamp, step):
                state["status"] = "UNRESOLVABLE"
                state["result"] = None
                state["reason"] = "stored candle snapshots contain an internal gap"
                return state

            if progress["entry_price"] is None:
                entry = bar["open"]
                initial_stop = entry - risk if buy else entry + risk
                progress.update({
                    "entry_at": bar["timestamp"],
                    "entry_price": entry,
                    "initial_stop": initial_stop,
                    "current_stop": initial_stop,
                    "peak": entry,
                })

            entry = float(progress["entry_price"])
            current_stop = float(progress["current_stop"])
            initial_stop = float(progress["initial_stop"])
            progress["bars_held"] += 1
            progress["last_processed_bar"] = bar["timestamp"]
            last = stamp

            if buy:
                mfe = (bar["high"] - entry) / risk
                mae = (entry - bar["low"]) / risk
                stop_hit = bar["low"] <= current_stop
            else:
                mfe = (entry - bar["low"]) / risk
                mae = (bar["high"] - entry) / risk
                stop_hit = bar["high"] >= current_stop
            progress["mfe_r"] = max(float(progress["mfe_r"]), max(0.0, mfe))
            progress["mae_r"] = max(float(progress["mae_r"]), max(0.0, mae))

            if stop_hit:
                exit_price = current_stop
                if (buy and bar["open"] < current_stop) or \
                        (not buy and bar["open"] > current_stop):
                    exit_price = bar["open"]
                reason = "SL" if current_stop == initial_stop else "TRAIL"
                _finish(state, reason, bar["timestamp"], exit_price)
                return state

            if trail_arm <= 0:
                target = entry + setup["tp1_distance"] if buy \
                    else entry - setup["tp1_distance"]
                target_hit = bar["high"] >= target if buy else bar["low"] <= target
                if target_hit:
                    exit_price = target
                    if (buy and bar["open"] > target) or \
                            (not buy and bar["open"] < target):
                        exit_price = bar["open"]
                    _finish(state, "TP1", bar["timestamp"], exit_price)
                    return state
            else:
                favorable = bar["high"] if buy else bar["low"]
                peak = max(float(progress["peak"]), favorable) if buy \
                    else min(float(progress["peak"]), favorable)
                progress["peak"] = peak
                excursion = ((peak - entry) if buy else (entry - peak)) / risk
                if excursion >= trail_arm:
                    level = peak - trail_distance * risk if buy \
                        else peak + trail_distance * risk
                    progress["current_stop"] = max(current_stop, level) if buy \
                        else min(current_stop, level)

            if progress["bars_held"] >= int(policy["max_hold_bars"]):
                _finish(state, "TIMEOUT", bar["timestamp"], bar["close"])
                return state
        return state
    except Exception as exc:
        # Still fail-safe (never raises into a scan), but no longer
        # SILENT: a real defect here used to be indistinguishable from a
        # legitimate weekend/downtime gap, discoverable only by grepping
        # advisory_outcome_json. Callers persist the state either way.
        logger.exception(
            "advisory outcome evaluator failed for the %s outcome seeded "
            "at %s — recorded UNRESOLVABLE",
            state.get("timeframe"), state.get("signal_bar_at"))
        state["status"] = "UNRESOLVABLE"
        state["result"] = None
        state["reason"] = f"outcome evaluator error: {type(exc).__name__}"
        return state


def resolve_locked_floor(state: Dict[str, Any]) -> Dict[str, Any]:
    """If an UNRESOLVABLE gap outcome had already trailed its stop past the
    entry, finish it at that stop as a FLOOR (the true outcome is at least
    this good) instead of discarding it. Returns the state (mutated)."""
    if state.get("status") != "UNRESOLVABLE":
        return state
    reason = str(state.get("reason", ""))
    # Both gap reasons ("lost continuity" / "contain an internal gap")
    # share this prefix; an invalid seed or evaluator error never floors.
    if not reason.startswith("stored candle snapshots"):
        return state
    progress = state.get("progress")
    setup = state.get("setup")
    if not isinstance(progress, dict) or not isinstance(setup, dict):
        return state
    entry = _finite(progress.get("entry_price"))
    stop = _finite(progress.get("current_stop"))
    risk = _finite(setup.get("stop_distance"))
    exit_at = progress.get("last_processed_bar")
    if (entry is None or stop is None or risk is None or risk <= 0
            or not exit_at):
        return state
    buy = setup.get("direction") == "BUY"
    locked = stop >= entry if buy else stop <= entry
    if not locked:
        return state
    _finish(state, "GAP_LOCKED", str(exit_at), stop)
    state["result"]["floor"] = True
    state["reason"] = (
        f"{reason} — resolved at the locked trailing stop (floor)")
    return state


def canonical_r(state: Dict[str, Any]) -> Optional[float]:
    result = state.get("result")
    value = result.get("net_r_065") if isinstance(result, dict) else None
    return _finite(value)