ForexBot2/telegram_bot.py

#!/usr/bin/env python3
"""
ForexBot2 — Telegram Bot
========================
Async Telegram interface with candle-boundary auto-scanner and
real-time WebSocket level-breach monitor.

Manual scan: XAU/USD × 3 timeframes (M15/H1/H4) via inline buttons.
Auto-scan:   Starts automatically on boot if TELEGRAM_CHAT_ID is set.
             /autoscan toggles for additional chats or to stop.
             M5  fires at :00/:05/:10/… + 10s buffer.
             M15 fires at :00/:15/:30/:45 + 10s buffer.
             H1  fires at :00 + 10s buffer.
             H4  fires at :00 every 4 hours + 10s buffer.
WebSocket:   Streams live ticks from TwelveData for XAU/USD.
             When price crosses a key structure level (swing S/R, BB,
             EMA200) cached from the last boundary scan, triggers an
             immediate M15 scan and sends a Level Break Alert.
             5-minute cooldown prevents spam.

Usage:
    python telegram_bot.py
    TELEGRAM_TOKEN=<token> TELEGRAM_CHAT_ID=<id> python telegram_bot.py
"""

import asyncio
import json
import logging
import os
import signal
import sys
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from typing import Dict, List, Optional, Set

import websockets

from telegram import InlineKeyboardButton, InlineKeyboardMarkup, Update
from telegram.constants import ParseMode
from telegram.error import BadRequest
from telegram.ext import (
    Application,
    CallbackQueryHandler,
    CommandHandler,
    ContextTypes,
)

from config import (
    PlatformConfig, SignalStrength, Timeframe,
    XAUUSD, PAIR_INFO,
)
from data_provider import ForexDataProvider
from signal_aggregator import SignalAggregator, SignalReport

# ── Force UTF-8 on Windows ────────────────────────────────────────────
if hasattr(sys.stdout, "reconfigure"):
    sys.stdout.reconfigure(encoding="utf-8")

logging.basicConfig(
    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
    level=logging.INFO,
    datefmt="%H:%M:%S",
)
logging.getLogger("httpx").setLevel(logging.WARNING)
logging.getLogger("httpcore").setLevel(logging.WARNING)
logger = logging.getLogger("telegram_bot")

TOKEN = os.environ.get("TELEGRAM_TOKEN", "8228251480:AAFW2OAyQ2idMicql2t_CIXk1r9-qfRTfNE")
CHAT_ID = os.environ.get("TELEGRAM_CHAT_ID", "6025899022")

# ── Pairs & buttons ───────────────────────────────────────────────────

SCAN_PAIRS = [XAUUSD]

_WS_TO_PAIR = {
    "XAU/USD": XAUUSD,
}
_WS_SYMBOLS = list(_WS_TO_PAIR.keys())

_LEVEL_KEYS = (
    "swing_high_1", "swing_high_2", "swing_high_3",
    "swing_low_1", "swing_low_2", "swing_low_3",
    "ema_trend", "bb_upper", "bb_lower",
    "kc_upper", "kc_lower",
)

SCAN_BUTTONS = [
    {"cb": "scan_xauusd_m5",  "symbol": XAUUSD, "tf": Timeframe.M5,  "label": "5 Min"},
    {"cb": "scan_xauusd_m15", "symbol": XAUUSD, "tf": Timeframe.M15, "label": "15 Min"},
    {"cb": "scan_xauusd_h1",  "symbol": XAUUSD, "tf": Timeframe.H1,  "label": "1 Hour"},
    {"cb": "scan_xauusd_h4",  "symbol": XAUUSD, "tf": Timeframe.H4,  "label": "4 Hour"},
    {"cb": "scan_xauusd_all", "symbol": XAUUSD, "tf": None,          "label": "Scan All TFs"},
]

SCAN_LOOKUP = {b["cb"]: b for b in SCAN_BUTTONS}

TF_CONFIRMATIONS = {
    Timeframe.M5:  [Timeframe.M15, Timeframe.H1],
    Timeframe.M15: [Timeframe.M5, Timeframe.H1],
    Timeframe.H1:  [Timeframe.M15, Timeframe.H4],
    Timeframe.H4:  [Timeframe.H1, Timeframe.D1],
}

AUTOSCAN_CONFIRMATIONS = {
    Timeframe.M5:  [Timeframe.M15, Timeframe.H1],
    Timeframe.M15: [Timeframe.M5, Timeframe.H1],
    Timeframe.H1:  [Timeframe.M15, Timeframe.H4],
    Timeframe.H4:  [Timeframe.H1, Timeframe.D1],
}

_executor = ThreadPoolExecutor(max_workers=4)

# ── Markdown escaping ─────────────────────────────────────────────────

def _escape_md(text: str) -> str:
    for ch in ("*", "_", "`", "[", "]"):
        text = text.replace(ch, "\\" + ch)
    return text


def _fmt_price(price: float) -> str:
    return f"{price:.2f}" if price >= 10 else f"{price:.5f}"


# ── Shared data provider (single cache across all scans) ─────────────

_shared_dp: Optional[ForexDataProvider] = None


def _get_shared_dp() -> ForexDataProvider:
    global _shared_dp
    if _shared_dp is None:
        _shared_dp = ForexDataProvider(PlatformConfig())
    return _shared_dp


# ── Signal formatting ─────────────────────────────────────────────────

_SIGNAL_EMOJI = {
    SignalStrength.STRONG_BUY:  "🟢",
    SignalStrength.BUY:         "🟩",
    SignalStrength.NEUTRAL:     "⬜",
    SignalStrength.SELL:        "🟥",
    SignalStrength.STRONG_SELL: "🔴",
}

_QUALITY_EMOJI = {
    "A+": "🏆", "A": "🥇", "B": "🥈", "C": "🥉", "D": "⚠️", "F": "❌",
}

_SIGNAL_TAG = {
    SignalStrength.STRONG_BUY:  "Strong Buy",
    SignalStrength.BUY:         "Bullish",
    SignalStrength.NEUTRAL:     "Neutral",
    SignalStrength.SELL:        "Bearish",
    SignalStrength.STRONG_SELL: "Strong Sell",
}


def _signal_label(sig: SignalStrength) -> str:
    return sig.name.replace("_", " ")


def _mtf_line(report: SignalReport) -> str:
    if not report.mtf_signals:
        return ""
    parts = []
    for tf, sig in report.mtf_signals.items():
        score = report.mtf_scores.get(tf, 0)
        parts.append(f"{tf}: {_signal_label(sig)} {score:+.1f}")
    return " | ".join(parts)


def _indicator_line(name: str, ind) -> str:
    tag = _SIGNAL_TAG.get(ind.signal, "Neutral")
    short = ind.description.split(" | ")[0] if " | " in ind.description else ind.description
    return f"[{tag}] *{_escape_md(ind.name)}*: {_escape_md(short)}"


def _live_price_line(symbol: str) -> Optional[str]:
    """Fresh WebSocket tick for this symbol, if we have one (<2 min old)."""
    price = _last_prices.get(symbol)
    ts = _last_price_ts.get(symbol, 0)
    if price is None:
        return None
    age = time.time() - ts
    if age > 120:
        return None
    return f"📡 Live: *{_fmt_price(price)}*  ({age:.0f}s tick)"


def format_signal_message(report: SignalReport) -> str:
    sig = report.final_signal
    emoji = _SIGNAL_EMOJI.get(sig, "⬜")
    qe = _QUALITY_EMOJI.get(report.signal_quality, "")
    label = _signal_label(sig)
    mtf_tag = "✅ MTF aligned" if report.mtf_agreement else "❌ MTF diverged"

    lines: list[str] = []

    now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
    regime_str = _escape_md(report.market_regime.replace("_", " ").title())
    vol_str = _escape_md(str(report.volatility_regime))
    lines += [
        f"🔍 *{report.display_name} Signal Analysis*  ({report.primary_analysis.timeframe if report.primary_analysis else ''})",
        f"🕐 {now}",
        "",
        f"💰 Close: *{_fmt_price(report.current_price)}*",
    ]
    live = _live_price_line(report.symbol)
    if live:
        lines.append(live)
    lines += [
        f"📊 Regime: {regime_str} | Vol: {vol_str}",
        "",
        "━━━━━━━━━━━━━━━━━━━━",
        f"{emoji} *{label}*  {qe}{report.signal_quality}  |  Score: {report.final_score:+.1f}  |  {report.final_confidence:.0f}% conf",
        f"_{_escape_md(report.action_recommendation)}_",
        f"{mtf_tag}",
    ]

    mtf = _mtf_line(report)
    if mtf:
        lines += ["", f"📈 *MTF*: {mtf}"]

    if report.primary_analysis:
        lines += ["", "━━━━━━━━━━━━━━━━━━━━", "📋 *Indicators*"]
        for name, ind in report.primary_analysis.indicators.items():
            if name.startswith("_") or ind.weight == 0:
                continue
            lines.append(_indicator_line(name, ind))

    if report.warnings:
        lines.append("")
        for w in report.warnings:
            lines.append(f"⚠️ {_escape_md(w)}")

    if report.trade_setup and sig not in (SignalStrength.NEUTRAL,):
        ts = report.trade_setup
        dir_emoji = "📈" if ts.direction == "BUY" else "📉"
        risk_usd = ts.risk_pips * ts.pip_value
        reward_usd = ts.reward_pips * ts.pip_value
        lines += [
            "",
            "━━━━━━━━━━━━━━━━━━━━",
            f"🚀 *TRADE SETUP*",
            f"{dir_emoji} Direction: *{ts.direction}*",
            f"▶ Entry:  `{_fmt_price(ts.entry_price)}`",
            f"🛑 SL:     `{_fmt_price(ts.stop_loss)}`  (${risk_usd:.2f} / {ts.risk_pips:.0f} pips)",
            f"🎯 TP1:    `{_fmt_price(ts.take_profit_1)}`  (${reward_usd:.2f} / {ts.reward_pips:.0f} pips)",
        ]
        if ts.take_profit_2:
            lines.append(f"🎯 TP2:    `{_fmt_price(ts.take_profit_2)}`")
        lines.append(f"⚖️  RR:     1:{ts.risk_reward:.1f}")

        # Trailing-stop management plan (matches backtested exit model:
        # TP1 is advisory, the trail is the exit).
        sig_cfg = PlatformConfig().signals
        if sig_cfg.trail_arm_r > 0:
            risk_dist = abs(ts.entry_price - ts.stop_loss)
            if ts.direction == "BUY":
                arm_price = ts.entry_price + sig_cfg.trail_arm_r * risk_dist
            else:
                arm_price = ts.entry_price - sig_cfg.trail_arm_r * risk_dist
            trail_dist = sig_cfg.trail_dist_r * risk_dist
            lines.append(
                f"🔄 Trail:  once price hits `{_fmt_price(arm_price)}` "
                f"(+{sig_cfg.trail_arm_r:.1f}R), trail SL ${trail_dist:.2f} "
                f"behind best price — TP1 is a guide, let it run"
            )

    lines += ["", "━━━━━━━━━━━━━━━━━━━━",
              "_Educational / research use only._"]

    return "\n".join(lines)


# ── Auto-scan summary format ─────────────────────────────────────────

def _format_scan_summary(reports: List[SignalReport], tf: Timeframe) -> str:
    now = datetime.now(timezone.utc).strftime("%H:%M UTC")
    lines = [
        f"📊 *{tf.value} Auto-Scan* | {now}",
        "━━━━━━━━━━━━━━━━━━━━",
    ]
    for r in reports:
        emoji = _SIGNAL_EMOJI.get(r.final_signal, "⬜")
        sig = _signal_label(r.final_signal)
        mtf = "✅" if r.mtf_agreement else "❌"
        lines.append(
            f"{emoji} *{r.display_name}*  {sig}  "
            f"{_QUALITY_EMOJI.get(r.signal_quality, '')}{r.signal_quality}  "
            f"{r.final_score:+.1f}  {r.final_confidence:.0f}%  {mtf}MTF"
        )
    lines.append("━━━━━━━━━━━━━━━━━━━━")

    actionable = [r for r in reports if r.signal_quality in ("A+", "A", "B", "C")]
    if actionable:
        names = ", ".join(r.display_name for r in actionable)
        lines.append(f"🔔 Actionable: {names}")
    else:
        lines.append("No actionable signals this cycle.")

    return "\n".join(lines)


# ── Keyboard helper ───────────────────────────────────────────────────

def scan_keyboard() -> InlineKeyboardMarkup:
    tf_buttons = [b for b in SCAN_BUTTONS if b["tf"] is not None]
    all_button = next((b for b in SCAN_BUTTONS if b["tf"] is None), None)
    rows = [[
        InlineKeyboardButton(b["label"], callback_data=b["cb"])
        for b in tf_buttons
    ]]
    if all_button:
        rows.append([InlineKeyboardButton(all_button["label"], callback_data=all_button["cb"])])
    return InlineKeyboardMarkup(rows)


def main_menu_keyboard() -> InlineKeyboardMarkup:
    """Full interactive menu: scans + settings + status in one keyboard."""
    tf_buttons = [b for b in SCAN_BUTTONS if b["tf"] is not None]
    all_button = next((b for b in SCAN_BUTTONS if b["tf"] is None), None)
    autoscan_label = "🔔 Auto-scan: ON" if _autoscan_chats else "🔕 Auto-scan: OFF"
    quality_label = f"🎚 Alerts: {_alert_min_quality}+" if _alert_min_quality != "A" else "🎚 Alerts: A only"
    rows = [
        [InlineKeyboardButton(b["label"], callback_data=b["cb"]) for b in tf_buttons],
    ]
    if all_button:
        rows.append([InlineKeyboardButton(all_button["label"], callback_data=all_button["cb"])])
    rows.append([
        InlineKeyboardButton(autoscan_label, callback_data="menu_autoscan"),
        InlineKeyboardButton(quality_label, callback_data="menu_quality"),
    ])
    rows.append([
        InlineKeyboardButton("📊 Status", callback_data="menu_status"),
        InlineKeyboardButton("ℹ️ Help", callback_data="menu_help"),
    ])
    return InlineKeyboardMarkup(rows)


async def _safe_edit(query, text: str, reply_markup=None) -> None:
    """edit_message_text that survives 'message is not modified' and
    Markdown parse failures (falls back to plain text)."""
    try:
        await query.edit_message_text(
            text, parse_mode=ParseMode.MARKDOWN, reply_markup=reply_markup
        )
    except BadRequest as exc:
        if "not modified" in str(exc).lower():
            return
        logger.warning(f"Markdown edit failed ({exc}); retrying as plain text")
        try:
            await query.edit_message_text(text, reply_markup=reply_markup)
        except BadRequest:
            pass


# ── Signal runner (shared data provider for cache reuse) ──────────────

def _run_signal_sync(
    symbol: str,
    timeframe: Timeframe,
    confirmation_tfs: Optional[List[Timeframe]] = None,
) -> SignalReport:
    if confirmation_tfs is None:
        confirmation_tfs = TF_CONFIRMATIONS.get(
            timeframe, [Timeframe.M15, Timeframe.H4]
        )
    config = PlatformConfig()
    config.primary_timeframe = timeframe
    config.confirmation_timeframes = confirmation_tfs
    aggregator = SignalAggregator(config)
    aggregator.data_provider = _get_shared_dp()
    return aggregator.analyze_pair(symbol=symbol)


async def run_signal(
    symbol: str,
    timeframe: Timeframe,
    confirmation_tfs: Optional[List[Timeframe]] = None,
) -> SignalReport:
    loop = asyncio.get_running_loop()
    return await loop.run_in_executor(
        _executor, _run_signal_sync, symbol, timeframe, confirmation_tfs
    )


# ── Alert gating (quality threshold + dedup) ─────────────────────────

_QUALITY_RANK = {"A+": 4, "A": 3, "B": 2, "C": 1}
_ALERT_QUALITY_CYCLE = ["C", "B", "A"]   # C+ → B+ → A-only
_alert_min_quality = "C"

# Fingerprint of the last alert per (symbol, timeframe) so a persisting
# signal doesn't re-send an identical message every candle.
_last_alert_fp: Dict[str, tuple] = {}
_last_alert_ts: Dict[str, float] = {}
_ALERT_REARM_SECONDS = 3600   # identical signal re-alerts after 1h


def _passes_quality(quality: str) -> bool:
    return _QUALITY_RANK.get(quality, 0) >= _QUALITY_RANK.get(_alert_min_quality, 1)


def _should_alert(key: str, report: SignalReport) -> bool:
    """True when this signal is new/changed for the key, or re-armed."""
    setup = report.trade_setup
    fp = (
        report.final_signal.name,
        report.signal_quality,
        setup.direction if setup else None,
    )
    now = time.time()
    if (_last_alert_fp.get(key) == fp
            and now - _last_alert_ts.get(key, 0) < _ALERT_REARM_SECONDS):
        return False
    _last_alert_fp[key] = fp
    _last_alert_ts[key] = now
    return True


# ── Candle-boundary scheduler ────────────────────────────────────────

_autoscan_chats: Set[int] = set()
_autoscan_tasks: list[asyncio.Task] = []


def _seconds_until_boundary(interval_minutes: int, buffer: int = 10) -> float:
    """Seconds until the next candle close + buffer."""
    now = datetime.now(timezone.utc)
    total_minutes = now.hour * 60 + now.minute
    elapsed_minutes = total_minutes % interval_minutes
    elapsed_seconds = elapsed_minutes * 60 + now.second + now.microsecond / 1e6
    interval_seconds = interval_minutes * 60
    remaining = interval_seconds - elapsed_seconds + buffer
    if remaining <= buffer:
        remaining += interval_seconds
    return remaining


async def _scan_timeframe(
    bot,
    tf: Timeframe,
    confirmation_tfs: List[Timeframe],
) -> None:
    """Scan all pairs for one timeframe and broadcast results."""
    tasks = [run_signal(symbol, tf, confirmation_tfs) for symbol in SCAN_PAIRS]
    results = await asyncio.gather(*tasks, return_exceptions=True)
    reports: List[SignalReport] = []
    for i, result in enumerate(results):
        if isinstance(result, Exception):
            logger.exception(f"Auto-scan failed for {SCAN_PAIRS[i]} {tf.value}: {result}")
        else:
            reports.append(result)

    if not reports:
        return

    _update_level_cache(reports)

    actionable = [
        r for r in reports
        if _passes_quality(r.signal_quality)
        and _should_alert(f"{r.symbol}_{tf.value}", r)
    ]
    for chat_id in list(_autoscan_chats):
        for report in actionable:
            detail = format_signal_message(report)
            try:
                await bot.send_message(chat_id, detail, parse_mode=ParseMode.MARKDOWN)
            except Exception as exc:
                logger.warning(f"Failed to send signal for chat {chat_id}: {exc}")


# ── WebSocket level-breach monitor ──────────────────────────────────

_level_cache: Dict[str, Dict[str, float]] = {}
_level_cache_ts: Dict[str, float] = {}
_LEVEL_CACHE_TTL = 3600
_last_prices: Dict[str, float] = {}
_last_price_ts: Dict[str, float] = {}
_scan_cooldowns: Dict[str, float] = {}

_cfg = PlatformConfig()
_WS_API_KEY = os.environ.get("TWELVE_DATA_API_KEY") or _cfg.twelve_data_api_key
_WS_URL = f"wss://ws.twelvedata.com/v1/quotes/price?apikey={_WS_API_KEY}"
_WS_COOLDOWN = _cfg.ws_cooldown_seconds

_breach_tasks: Set[asyncio.Task] = set()


def _update_level_cache(reports: List[SignalReport]) -> None:
    now = time.time()
    for r in reports:
        if r.primary_analysis and r.primary_analysis.key_levels:
            levels = {
                k: v for k, v in r.primary_analysis.key_levels.items()
                if k in _LEVEL_KEYS
            }
            if levels:
                _level_cache[r.symbol] = levels
                _level_cache_ts[r.symbol] = now


def _check_level_breach(symbol: str, price: float) -> Optional[str]:
    levels = _level_cache.get(symbol)
    if not levels:
        return None

    cache_age = time.time() - _level_cache_ts.get(symbol, 0)
    if cache_age > _LEVEL_CACHE_TTL:
        return None

    last = _last_prices.get(symbol)
    if last is None:
        return None

    for name, level in levels.items():
        if (last < level <= price) or (last > level >= price):
            return name

    return None


def _cleanup_cooldowns() -> None:
    now = time.time()
    expired = [k for k, v in _scan_cooldowns.items() if now - v > _WS_COOLDOWN * 2]
    for k in expired:
        del _scan_cooldowns[k]


async def _handle_breach(bot, ws_symbol: str, pair_symbol: str, price: float, level_name: str) -> None:
    now = time.time()
    if now - _scan_cooldowns.get(pair_symbol, 0) < _WS_COOLDOWN:
        return

    _scan_cooldowns[pair_symbol] = now
    display = PAIR_INFO.get(pair_symbol, {}).get("display", ws_symbol)
    label = level_name.replace("_", " ").title()
    logger.info(f"Level breach: {display} crossed {label} at {price} — rescanning M15")

    try:
        report = await run_signal(pair_symbol, Timeframe.M15)
        _update_level_cache([report])
    except Exception:
        logger.exception(f"Breach scan failed for {display}")
        return

    # Alert only when the rescan produced an actionable, changed signal —
    # same gates as the auto-scanner, so a breach can't spam the channel.
    if (report.final_signal == SignalStrength.NEUTRAL
            or not _passes_quality(report.signal_quality)
            or not _should_alert(f"{pair_symbol}_{Timeframe.M15.value}", report)):
        return

    header = (
        f"⚡ *LEVEL BREAK* — {_escape_md(display)} crossed "
        f"*{_escape_md(label)}* @ `{_fmt_price(price)}`\n\n"
    )
    detail = header + format_signal_message(report)
    for chat_id in list(_autoscan_chats):
        try:
            await bot.send_message(chat_id, detail, parse_mode=ParseMode.MARKDOWN)
        except Exception as exc:
            logger.warning(f"Failed to send breach alert to chat {chat_id}: {exc}")


async def _ws_heartbeat(ws) -> None:
    """Twelve Data drops idle WS connections; heartbeat every 10s keeps it
    alive through quiet periods (weekends, low-liquidity hours)."""
    while True:
        await asyncio.sleep(10)
        await ws.send(json.dumps({"action": "heartbeat"}))


async def _ws_monitor(bot) -> None:
    backoff = 1
    while _autoscan_chats:
        hb_task: Optional[asyncio.Task] = None
        try:
            async with websockets.connect(_WS_URL) as ws:
                subscribe = {
                    "action": "subscribe",
                    "params": {"symbols": ",".join(_WS_SYMBOLS)},
                }
                await ws.send(json.dumps(subscribe))
                logger.info(f"WebSocket connected, subscribed to {len(_WS_SYMBOLS)} symbols")
                backoff = 1
                hb_task = asyncio.create_task(_ws_heartbeat(ws))

                async for raw in ws:
                    if not _autoscan_chats:
                        break

                    try:
                        msg = json.loads(raw)
                    except json.JSONDecodeError:
                        logger.warning(f"WebSocket: malformed JSON, skipping")
                        continue

                    event = msg.get("event")
                    if event == "subscribe-status":
                        fails = msg.get("fails") or []
                        if fails:
                            logger.warning(f"WebSocket: failed to subscribe: {fails}")
                        continue
                    if event != "price":
                        continue

                    ws_symbol = msg.get("symbol")
                    pair_symbol = _WS_TO_PAIR.get(ws_symbol)
                    if not pair_symbol:
                        continue

                    try:
                        price = float(msg["price"])
                    except (KeyError, ValueError, TypeError) as e:
                        logger.warning(f"WebSocket: bad price data for {ws_symbol}: {e}")
                        continue

                    breached = _check_level_breach(pair_symbol, price)
                    _last_prices[pair_symbol] = price
                    _last_price_ts[pair_symbol] = time.time()

                    if breached:
                        task = asyncio.create_task(
                            _handle_breach(bot, ws_symbol, pair_symbol, price, breached)
                        )
                        _breach_tasks.add(task)
                        task.add_done_callback(_breach_tasks.discard)

                    _cleanup_cooldowns()

        except asyncio.CancelledError:
            break
        except Exception:
            logger.exception(f"WebSocket error, reconnecting in {backoff}s")
            await asyncio.sleep(backoff)
            backoff = min(backoff * 2, 60)
        finally:
            if hb_task is not None:
                hb_task.cancel()

    logger.info("WebSocket monitor stopped")


async def _candle_loop(bot, tf: Timeframe, interval_minutes: int) -> None:
    label = tf.value
    try:
        while _autoscan_chats:
            wait = _seconds_until_boundary(interval_minutes, buffer=10)
            logger.info(f"{label} auto-scan: next in {wait:.0f}s")
            await asyncio.sleep(wait)
            if not _autoscan_chats:
                break
            logger.info(f"{label} auto-scan: firing")
            await _scan_timeframe(
                bot, tf, AUTOSCAN_CONFIRMATIONS[tf]
            )
    except asyncio.CancelledError:
        pass
    finally:
        logger.info(f"{label} auto-scan loop stopped")


_AUTOSCAN_LOOPS = [
    (Timeframe.M5,  5),
    (Timeframe.M15, 15),
    (Timeframe.H1,  60),
    (Timeframe.H4,  240),
]


_ws_task: Optional[asyncio.Task] = None


def _start_scheduler(bot) -> None:
    global _ws_task
    alive = [t for t in _autoscan_tasks if not t.done()]
    if not alive:
        _autoscan_tasks.clear()
        for tf, interval in _AUTOSCAN_LOOPS:
            _autoscan_tasks.append(
                asyncio.create_task(_candle_loop(bot, tf, interval))
            )
        logger.info("Candle-boundary scheduler started (M5/M15/H1/H4)")

    if _ws_task is None or _ws_task.done():
        _ws_task = asyncio.create_task(_ws_monitor(bot))
        logger.info("WebSocket level-breach monitor started")


def _stop_scheduler() -> None:
    global _ws_task
    for t in _autoscan_tasks:
        t.cancel()
    _autoscan_tasks.clear()
    if _ws_task and not _ws_task.done():
        _ws_task.cancel()
    _ws_task = None
    for t in list(_breach_tasks):
        t.cancel()
    _breach_tasks.clear()
    logger.info("Scheduler and WebSocket monitor stopped")


# ── Handlers ──────────────────────────────────────────────────────────

async def cmd_start(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
    scanning = "running" if _autoscan_chats else "off"
    text = (
        "👋 *ForexBot2 — XAU/USD Signal Intelligence*\n\n"
        "Pick a timeframe to scan gold.\n"
        "Signals include a full trade setup with entry, SL and TP.\n\n"
        f"Auto-scan: *{scanning}* — /autoscan to toggle\n"
        "/scan — Quick scan buttons\n"
        "/status — Bot & WebSocket status\n"
        "/help — All commands"
    )
    await update.message.reply_text(
        text,
        parse_mode=ParseMode.MARKDOWN,
        reply_markup=main_menu_keyboard(),
    )


_HELP_TEXT = (
    "📖 *ForexBot2 Commands*\n\n"
    "/start — Welcome & interactive menu\n"
    "/scan — Quick XAU/USD scan menu\n"
    "/autoscan — Toggle auto-scan on/off\n"
    "/status — Bot status, WS connection, live prices\n"
    "/cache — Show data cache age per pair/TF\n"
    "/clearcache — Wipe data cache, force fresh fetch\n"
    "/help — This message\n\n"
    "*Auto-scan* fires at candle close: M5/M15/H1/H4.\n"
    "*Alerts* button sets the minimum quality grade that gets sent.\n"
    "*WebSocket* streams live prices; key-level breaches trigger an "
    "instant M15 rescan and alert (5-min cooldown)."
)


async def cmd_help(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
    await update.message.reply_text(_HELP_TEXT, parse_mode=ParseMode.MARKDOWN)


def _status_text() -> str:
    lines = ["📊 *ForexBot2 Status*\n"]

    if _autoscan_chats:
        lines.append(f"✅ Auto-scan: *running* ({len(_autoscan_chats)} chat)")
        tasks_alive = sum(1 for t in _autoscan_tasks if not t.done())
        lines.append(f"   Candle loops: {tasks_alive}/{len(_AUTOSCAN_LOOPS)}")
    else:
        lines.append("⏹ Auto-scan: *stopped*")

    if _ws_task and not _ws_task.done():
        lines.append("📡 WebSocket: *connected*")
    else:
        lines.append("❌ WebSocket: *disconnected*")

    if _last_prices:
        lines.append("\n*Live Prices (WebSocket):*")
        for ws_sym, pair in _WS_TO_PAIR.items():
            price = _last_prices.get(pair)
            display = PAIR_INFO.get(pair, {}).get("display", ws_sym)
            age = time.time() - _last_price_ts.get(pair, 0)
            if price is None:
                lines.append(f"  {display}: waiting...")
            else:
                lines.append(f"  {display}: `{_fmt_price(price)}` ({age:.0f}s ago)")
    else:
        lines.append("\n_No live prices yet (WS not streaming)_")

    cached_pairs = len(_level_cache)
    cached_levels = sum(len(v) for v in _level_cache.values())
    if cached_levels:
        oldest_age = time.time() - min(_level_cache_ts.values()) if _level_cache_ts else 0
        lines.append(f"\n*Level Cache:* {cached_levels} levels across {cached_pairs} pairs (oldest {oldest_age:.0f}s)")
    else:
        lines.append("\n*Level Cache:* empty — breach alerts start after first candle scan")

    active_cooldowns = sum(
        1 for v in _scan_cooldowns.values()
        if time.time() - v < _WS_COOLDOWN
    )
    if active_cooldowns:
        lines.append(f"*Cooldowns:* {active_cooldowns} pair(s) in cooldown")

    active_breach = sum(1 for t in _breach_tasks if not t.done())
    if active_breach:
        lines.append(f"*Breach tasks:* {active_breach} running")

    lines.append(f"\n*Alert quality:* {_alert_min_quality}+ "
                 f"| re-arm {_ALERT_REARM_SECONDS // 60}m")
    return "\n".join(lines)


async def cmd_status(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
    await update.message.reply_text(_status_text(), parse_mode=ParseMode.MARKDOWN)


async def cmd_cache(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
    dp = _get_shared_dp()
    if not dp._cache_timestamps:
        await update.message.reply_text("📭 Cache is empty — no data fetched yet.")
        return

    now = time.time()
    lines = ["📦 *Data Cache Status*\n"]

    for pair in SCAN_PAIRS:
        display = PAIR_INFO.get(pair, {}).get("display", pair)
        pair_lines = []
        for tf in [Timeframe.M5, Timeframe.M15, Timeframe.H1, Timeframe.H4, Timeframe.D1]:
            cache_key = f"{pair}_{tf.value}"
            ts = dp._cache_timestamps.get(cache_key)
            if ts is None:
                continue
            age_min = (now - ts) / 60
            df = dp._cache.get(cache_key)
            last_bar = df.index[-1].strftime("%H:%M") if df is not None and not df.empty else "?"
            fresh = "✅" if age_min < dp.CANDLE_SECONDS.get(tf, 3600) / 60 else "⚠️"
            pair_lines.append(f"  {tf.value:<5} {fresh} fetched `{age_min:.0f}m` ago | last bar `{last_bar}`")
        if pair_lines:
            lines.append(f"*{display}*")
            lines.extend(pair_lines)

    await update.message.reply_text("\n".join(lines), parse_mode=ParseMode.MARKDOWN)


async def cmd_clearcache(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
    dp = _get_shared_dp()
    count = dp.clear_cache()
    await update.message.reply_text(
        f"🗑 Cache cleared ({count} entries wiped).\nNext scan will fetch fresh data from API.",
        parse_mode=ParseMode.MARKDOWN,
    )


async def cmd_autoscan(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
    chat_id = update.effective_chat.id

    if chat_id in _autoscan_chats:
        _autoscan_chats.discard(chat_id)
        if not _autoscan_chats:
            _stop_scheduler()
        await update.message.reply_text(
            "⏹ Auto-scan stopped.",
            parse_mode=ParseMode.MARKDOWN,
        )
        return

    _autoscan_chats.add(chat_id)
    _start_scheduler(context.bot)

    next_m5 = _seconds_until_boundary(5, buffer=10)
    next_m15 = _seconds_until_boundary(15, buffer=10)
    next_h1 = _seconds_until_boundary(60, buffer=10)
    next_h4 = _seconds_until_boundary(240, buffer=10)
    await update.message.reply_text(
        "✅ *Auto-scan enabled*\n\n"
        f"Scanning XAU/USD on M5/M15/H1/H4 at candle close.\n"
        f"📡 Live price monitoring active.\n"
        f"Next: M5 {next_m5 / 60:.0f}m | M15 {next_m15 / 60:.0f}m | "
        f"H1 {next_h1 / 60:.0f}m | H4 {next_h4 / 60:.0f}m\n\n"
        "Use /autoscan again to stop.",
        parse_mode=ParseMode.MARKDOWN,
    )


async def cmd_scan(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
    await update.message.reply_text(
        "🔍 *XAU/USD — Pick a timeframe*",
        parse_mode=ParseMode.MARKDOWN,
        reply_markup=main_menu_keyboard(),
    )


async def cb_scan(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
    query = update.callback_query
    await query.answer()

    btn = SCAN_LOOKUP.get(query.data)
    if not btn:
        return

    if btn["tf"] is None:
        await _cb_scan_all(query)
        return

    await _safe_edit(query, f"⏳ Scanning XAU/USD {btn['label']}…")

    try:
        report = await run_signal(btn["symbol"], btn["tf"])
        text = format_signal_message(report)
    except Exception as exc:
        logger.exception("Signal scan failed")
        text = f"❌ Scan failed: {_escape_md(str(exc))}"

    await _safe_edit(query, text, reply_markup=main_menu_keyboard())


async def cb_menu(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
    """Interactive menu actions: autoscan toggle, quality cycle, status, help."""
    global _alert_min_quality
    query = update.callback_query
    action = query.data

    if action == "menu_autoscan":
        chat_id = query.message.chat.id
        if chat_id in _autoscan_chats:
            _autoscan_chats.discard(chat_id)
            if not _autoscan_chats:
                _stop_scheduler()
            await query.answer("Auto-scan stopped")
        else:
            _autoscan_chats.add(chat_id)
            _start_scheduler(context.bot)
            await query.answer("Auto-scan started")
        try:
            await query.edit_message_reply_markup(reply_markup=main_menu_keyboard())
        except BadRequest:
            pass

    elif action == "menu_quality":
        i = _ALERT_QUALITY_CYCLE.index(_alert_min_quality) if _alert_min_quality in _ALERT_QUALITY_CYCLE else 0
        _alert_min_quality = _ALERT_QUALITY_CYCLE[(i + 1) % len(_ALERT_QUALITY_CYCLE)]
        await query.answer(f"Alerting {_alert_min_quality}+ signals")
        try:
            await query.edit_message_reply_markup(reply_markup=main_menu_keyboard())
        except BadRequest:
            pass

    elif action == "menu_status":
        await query.answer()
        await query.message.reply_text(_status_text(), parse_mode=ParseMode.MARKDOWN)

    elif action == "menu_help":
        await query.answer()
        await query.message.reply_text(_HELP_TEXT, parse_mode=ParseMode.MARKDOWN)


async def _cb_scan_all(query) -> None:
    await _safe_edit(query, "⏳ Scanning XAU/USD across M5 / M15 / H1 / H4…")

    tfs = [Timeframe.M5, Timeframe.M15, Timeframe.H1, Timeframe.H4]
    tasks = [run_signal(XAUUSD, tf) for tf in tfs]
    results = await asyncio.gather(*tasks, return_exceptions=True)

    reports = []
    errors = []
    for tf, result in zip(tfs, results):
        if isinstance(result, Exception):
            errors.append(f"❌ {tf.value}: {result}")
        else:
            reports.append(result)

    if reports:
        _update_level_cache(reports)

    lines = [
        "📊 *XAU/USD — All Timeframes*",
        f"🕐 {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}",
        "",
    ]

    for r in reports:
        emoji = _SIGNAL_EMOJI.get(r.final_signal, "⬜")
        qe = _QUALITY_EMOJI.get(r.signal_quality, "")
        tf_label = r.primary_analysis.timeframe if r.primary_analysis else "?"
        sig = _signal_label(r.final_signal)
        regime = r.market_regime.replace("_", " ").title()
        lines.append(
            f"{emoji} *{tf_label}*  {sig}  {qe}{r.signal_quality}  "
            f"Score: {r.final_score:+.1f}  {r.final_confidence:.0f}%"
        )
        lines.append(f"   _{_escape_md(regime)}_ | {'✅ MTF' if r.mtf_agreement else '❌ MTF'}")
        if r.trade_setup and r.final_signal != SignalStrength.NEUTRAL:
            ts = r.trade_setup
            d = "📈" if ts.direction == "BUY" else "📉"
            lines.append(
                f"   {d} {ts.direction} `{_fmt_price(ts.entry_price)}` "
                f"SL `{_fmt_price(ts.stop_loss)}` TP `{_fmt_price(ts.take_profit_1)}` "
                f"RR 1:{ts.risk_reward:.1f}"
            )
        lines.append("")

    for err in errors:
        lines.append(err)

    lines.append("_Use buttons below for detailed single-TF view._")

    await _safe_edit(query, "\n".join(lines), reply_markup=main_menu_keyboard())


# ── Boot-time auto-start ──────────────────────────────────────────────

async def _post_init(app: Application) -> None:
    """Register command menu and start scheduler on boot."""
    await app.bot.set_my_commands([
        ("start", "Welcome & scan buttons"),
        ("scan", "Quick XAU/USD scan buttons"),
        ("autoscan", "Toggle auto-scan on/off"),
        ("status", "Bot status, WS connection, live prices"),
        ("cache", "Show data cache age per pair/TF"),
        ("clearcache", "Wipe data cache, force fresh fetch"),
        ("help", "All commands"),
    ])

    if not CHAT_ID:
        logger.info("TELEGRAM_CHAT_ID not set — auto-scan disabled on boot. "
                     "Use /autoscan in chat to enable.")
        return

    chat_id = int(CHAT_ID)
    _autoscan_chats.add(chat_id)

    logger.info("Running initial scan to populate level cache and signal stats...")
    try:
        tasks = [
            asyncio.wait_for(run_signal(sym, Timeframe.M15), timeout=30)
            for sym in SCAN_PAIRS
        ]
        results = await asyncio.gather(*tasks, return_exceptions=True)
        init_reports = [r for r in results if isinstance(r, SignalReport)]
        for i, r in enumerate(results):
            if isinstance(r, Exception):
                logger.error(f"Initial scan failed for {SCAN_PAIRS[i]}: {r}")
        if init_reports:
            _update_level_cache(init_reports)
            logger.info(f"Level cache populated: {sum(len(v) for v in _level_cache.values())} levels")
    except Exception:
        logger.exception("Initial scan failed — WS will wait for first candle scan")

    _start_scheduler(app.bot)

    next_m5 = _seconds_until_boundary(5, buffer=10)
    next_m15 = _seconds_until_boundary(15, buffer=10)
    next_h1 = _seconds_until_boundary(60, buffer=10)
    next_h4 = _seconds_until_boundary(240, buffer=10)
    logger.info(
        f"Auto-scan started for chat {chat_id} — "
        f"M5 {next_m5:.0f}s, M15 {next_m15:.0f}s, H1 {next_h1:.0f}s, H4 {next_h4:.0f}s"
    )

    try:
        await app.bot.send_message(
            chat_id,
            f"🤖 *ForexBot2 started*\n\n"
            f"Auto-scanning XAU/USD on M5/M15/H1/H4.\n"
            f"📡 Live price monitoring active.\n"
            f"Next: M5 {next_m5 / 60:.0f}m | M15 {next_m15 / 60:.0f}m | "
            f"H1 {next_h1 / 60:.0f}m | H4 {next_h4 / 60:.0f}m\n\n"
            f"/autoscan to stop",
            parse_mode=ParseMode.MARKDOWN,
        )
    except Exception:
        logger.exception("Failed to send boot message")


# ── Graceful shutdown ─────────────────────────────────────────────────

async def _post_shutdown(app: Application) -> None:
    logger.info("Shutting down ForexBot2...")
    _stop_scheduler()
    _executor.shutdown(wait=False)
    logger.info("Shutdown complete")


# ── Entry point ───────────────────────────────────────────────────────

def main() -> None:
    if not TOKEN:
        sys.exit("TELEGRAM_TOKEN not set")

    logger.info("Starting ForexBot2 Telegram bot…")

    app = (
        Application.builder()
        .token(TOKEN)
        .post_init(_post_init)
        .post_shutdown(_post_shutdown)
        .build()
    )

    app.add_handler(CommandHandler("start", cmd_start))
    app.add_handler(CommandHandler("scan", cmd_scan))
    app.add_handler(CommandHandler("help", cmd_help))
    app.add_handler(CommandHandler("status", cmd_status))
    app.add_handler(CommandHandler("autoscan", cmd_autoscan))
    app.add_handler(CommandHandler("cache", cmd_cache))
    app.add_handler(CommandHandler("clearcache", cmd_clearcache))
    app.add_handler(CallbackQueryHandler(cb_scan, pattern="^scan_"))
    app.add_handler(CallbackQueryHandler(cb_menu, pattern="^menu_"))

    logger.info("Bot running — press Ctrl+C to stop")
    app.run_polling(allowed_updates=Update.ALL_TYPES)


if __name__ == "__main__":
    main()