#!/usr/bin/env python3 """ XAU/USD static dashboard publisher. Standalone daemon that scans XAU/USD at candle closes and atomically writes `snapshot.json` + `news.json` (plus copies of the static `dashboard.html` / `.css` / `.js` and `news_summary.php`) into an Apache-served directory. Apache does all web serving — this process never opens a port, so a failure here can only ever mean a stale page. Independence rules (owner requirement — the telegram bot must be immune to anything on the web side): * Runs as its own process; never imported by, and never imports, `telegram_bot` (telegram_bot resolves credentials at import). The confirmation map and boundary math are therefore duplicated here as local constants. * The only shared-code dependency beyond the standard signal path is the additive `NewsEngine.get_week_events` accessor, which no production signal path calls. * Never writes a database. It is not always running, so the Telegram bot is the single writer of `signals.db` and `newsdata.db`; main() switches every SQLite writer off for this process (`app_config.set_db_writes_for_process`, owner decision 2026-09-17). Scheduling: every M15/H1/H4 boundary is also an M5 boundary (all intervals are multiples of 5, anchored to UTC midnight), so one sequential loop sleeps to the next M5 boundary (+10 s buffer) and scans whichever timeframes just closed. Due-classification uses the RAW wake time — flooring 12:00:10 to the 5-minute grid yields the 12:00 boundary that just closed. Do NOT subtract the buffer first: 12:00:10 − 15 s floors to 11:55 and silently skips every H1/H4 scan (v1 bug, 2026-08-02). A cycle can overrun its 5-minute slot (or the process can be down), so each wake also catches up the timeframes whose latest closed boundary is newer than their last scanned one (`catch_up_timeframes`) — without it the page served an hours-old H1/H4 report behind a fresh countdown (2026-09-13 audit). Scans run sequentially M5→M15→H1→H4 through one shared data provider so later timeframes hit the warm boundary cache — that ordering IS the duplicate-fetch dedup (providers have no in-flight dedup; Oanda has no throttle). Keep it sequential. Run: python dashboard_publisher.py [--once] [--out DIR] Out dir precedence: --out → DASHBOARD_OUT_DIR env → /var/www/trade """ import argparse import json import hashlib import logging import math import numbers import os import shutil import sys import time from datetime import datetime, timedelta, timezone from logging.handlers import TimedRotatingFileHandler from typing import Dict, List, Optional import pandas as pd from agent_panel import attach_panel from app_config import (get_app_config, set_db_writes_for_process, violent_watch_enabled) from config import PlatformConfig, Timeframe, XAUUSD from exposure import attach_exposure from main import report_to_dict from news_engine import NewsEngine from pair_signals import produce_signal from providers import make_data_provider from violent_watch import attach_watch import providers.base as _providers_base logger = logging.getLogger("dashboard") DASHBOARD_TFS = [Timeframe.M5, Timeframe.M15, Timeframe.H1, Timeframe.H4] # Local copy of the bot's confirmation map (see independence rules above). CONFIRMATIONS = { Timeframe.M5: [Timeframe.M15, Timeframe.H1], Timeframe.M15: [Timeframe.M5, Timeframe.H1], Timeframe.H1: [Timeframe.M15, Timeframe.H4], Timeframe.H4: [Timeframe.H1, Timeframe.D1], } TF_INTERVAL_MINUTES = { Timeframe.M5: 5, Timeframe.M15: 15, Timeframe.H1: 60, Timeframe.H4: 240, } DEFAULT_OUT_DIR = "/var/www/trade" SCAN_BUFFER_SECONDS = 10 SNAPSHOT_FILE = "snapshot.json" NEWS_FILE = "news.json" PAGE_FILE = "dashboard.html" SUMMARY_PHP_FILE = "news_summary.php" # Repo files copied into the out dir at daemon start (git pull + restart # updates them). The PHP endpoint reads news.json in place, so it lives # next to the page. STATIC_FILES = [PAGE_FILE, "dashboard.css", "dashboard.js", SUMMARY_PHP_FILE] # ══════════════════════════════════════════════════════════════════ # Pure layer (no network, fully covered by test_all.py) # ══════════════════════════════════════════════════════════════════ def _safe_num(v): """Coerce to a JSON-safe float; NaN/±inf become None.""" try: f = float(v) except (TypeError, ValueError): return v if math.isnan(f) or math.isinf(f): return None return f def _sanitize(obj): """Recursively make a payload safe for json.dumps(allow_nan=False): NaN/inf → None, numpy scalars → native ints/floats.""" if isinstance(obj, bool): return obj if isinstance(obj, dict): return {k: _sanitize(v) for k, v in obj.items()} if isinstance(obj, (list, tuple)): return [_sanitize(v) for v in obj] if isinstance(obj, numbers.Integral): return int(obj) if isinstance(obj, numbers.Real): return _safe_num(obj) return obj def analysis_to_dict(analysis) -> Optional[dict]: """Serialize a TechnicalAnalysis. Iterates analysis.indicators only — never vars()/asdict(), which would drag dynamic pandas attrs (_rsi_series, _macd_histogram, …) into the payload. ATR (weight 0) is surfaced separately as context, not in the directional grid.""" if analysis is None: return None indicators = {} atr_context = None for name, ind in analysis.indicators.items(): if name.startswith("_"): continue entry = { "signal": ind.signal.name, "value": _safe_num(ind.value), "confidence": _safe_num(ind.confidence), "weight": _safe_num(ind.weight), "description": ind.description, } if name == "atr": atr_context = entry elif ind.weight != 0: indicators[name] = entry return { "timeframe": analysis.timeframe, "timestamp": str(analysis.timestamp), "composite_signal": analysis.composite_signal.name, "composite_score": _safe_num(analysis.composite_score), "confidence": _safe_num(analysis.confidence), "market_regime": analysis.market_regime, "volatility_regime": analysis.volatility_regime, "pattern_signals": list(analysis.pattern_signals), "key_levels": {k: _safe_num(v) for k, v in analysis.key_levels.items()}, "indicators": indicators, "atr_context": atr_context, } def report_to_webapp_dict(report) -> dict: """main.report_to_dict + the per-TF detail the dashboard needs.""" d = report_to_dict(report) d["primary_signal"] = report.primary_signal.name d["primary_score"] = _safe_num(report.primary_score) d["mtf_scores"] = {k: _safe_num(v) for k, v in report.mtf_scores.items()} d["exhaustion_count"] = report.exhaustion_count d["primary_analysis"] = analysis_to_dict(report.primary_analysis) d["confirmation_analyses"] = [ analysis_to_dict(a) for a in report.confirmation_analyses ] return _sanitize(d) def event_to_dict(ev) -> dict: """EconomicEvent → dict. surprise is a property, so asdict() would skip it — added explicitly. actual/forecast/previous stay strings as published ('228K', '0.3%').""" return { "time": ev.time.astimezone(timezone.utc).isoformat(), "currency": ev.currency, "impact": ev.impact, "event": ev.event, "forecast": ev.forecast, "previous": ev.previous, "actual": ev.actual, "surprise": ev.surprise, } def headline_to_dict(h) -> dict: # id uses the same sha1(headline) scheme as NewsEngine._llm_sent_cache; # it is the join key the PHP summary endpoint whitelists against. return { "id": hashlib.sha1(h.headline.encode("utf-8")).hexdigest(), "headline": h.headline, "source": h.source, "link": h.link, "description": h.description, "publisher": h.publisher, "timestamp": h.timestamp.astimezone(timezone.utc).isoformat() if h.timestamp else None, "currencies": list(h.currencies_mentioned), "score": _safe_num(h.sentiment_score), "keywords": list(h.keywords), "score_by_currency": { k: _safe_num(v) for k, v in h.sentiment_by_currency.items() }, } def build_news_payload(context, week_events, now: datetime) -> dict: """news.json: full calendar week + scored headlines + LLM summary.""" headlines = sorted( context.news_headlines, key=lambda h: h.timestamp or now, reverse=True, ) return _sanitize({ "generated_at": now.isoformat(), "llm_summary": context.llm_summary, "sentiment": dict(context.news_sentiment), "high_impact_warning": context.high_impact_warning, "warning_message": context.warning_message, "calendar_available": context.calendar_available, "events": [event_to_dict(e) for e in week_events], "headlines": [headline_to_dict(h) for h in headlines], }) def due_timeframes(wake_dt: datetime) -> List[Timeframe]: """Timeframes whose candle just closed at the boundary this wake belongs to. Takes the RAW wake time (boundary + buffer): flooring to the 5-minute grid yields the boundary that just passed. Never subtract the buffer first — see module docstring (v1 bug). A timeframe whose just-closed bar would carry a label inside gold's daily settlement pause is skipped: no such bar exists, so the scan would only re-read the previous one (2026-09-13 audit).""" minutes = wake_dt.hour * 60 + wake_dt.minute floored = minutes - (minutes % 5) boundary = wake_dt.replace( hour=floored // 60, minute=floored % 60, second=0, microsecond=0) return [ tf for tf in DASHBOARD_TFS if floored % TF_INTERVAL_MINUTES[tf] == 0 and not _providers_base._is_daily_maintenance( boundary - timedelta(minutes=TF_INTERVAL_MINUTES[tf])) ] def catch_up_timeframes(wake_dt: datetime, last_boundary: Dict[str, float]) -> List[Timeframe]: """Timeframes whose most recent closed boundary (<= wake) is newer than the boundary of their last successful scan — i.e. boundaries that were missed while a cycle overran or the process was down. A TF never scanned is due. Excludes bars whose label sits in the daily settlement pause (same rule as due_timeframes). `due_timeframes` only ever names the timeframes whose boundary coincides with THIS wake, so an overrunning cycle (scan + advisory panel + news can take minutes) or a restart left the page serving the previous H1/H4 report for up to 1 h / 4 h while advertising a fresh next_close_epoch (2026-09-13 audit).""" minutes = wake_dt.hour * 60 + wake_dt.minute out = [] for tf in DASHBOARD_TFS: interval = TF_INTERVAL_MINUTES[tf] floored = minutes - (minutes % interval) last_closed = wake_dt.replace( hour=floored // 60, minute=floored % 60, second=0, microsecond=0) # No bar carries a label inside a scheduled closure (settlement # pause or weekend), so nothing can have been "missed" there — and a # weekend-blocked scan never advances the ledger, so without the # weekend half of this test every weekend wake would re-scan # M15/H1/H4 for nothing (~1,800 blocked rows per weekend). # (Whole-span rule, as in providers.base._has_latest_closed_bar: an # H4 bar labelled Sunday 20:00 UTC starts in the weekend but runs # through the reopen, so it exists.) label = last_closed - timedelta(minutes=interval) if (_providers_base._is_scheduled_closure(label) and _providers_base._is_scheduled_closure( last_closed - timedelta(seconds=1))): continue scanned = last_boundary.get(tf.value) if scanned is None or scanned < last_closed.timestamp(): out.append(tf) return out def seconds_until_boundary(interval_minutes: int, buffer: int = 10) -> float: # Same math as telegram_bot.py — kept local so the dashboard never # imports bot modules (independence rules above). now = datetime.now(timezone.utc) total_minutes = now.hour * 60 + now.minute elapsed_minutes = total_minutes % interval_minutes elapsed_seconds = elapsed_minutes * 60 + now.second + now.microsecond / 1e6 interval_seconds = interval_minutes * 60 remaining = interval_seconds - elapsed_seconds + buffer # NOTE: remaining > buffer always (elapsed < interval by construction); # a call at the exact boundary instant targets the NEXT boundary — # steady-state loops wake at boundary+buffer, so only a process # started in the boundary microsecond skips one scan (2026-08-04 # audit I42; the old `remaining <= buffer` guard was unreachable). return remaining def _empty_tf_entry() -> dict: return {"status": "warming", "scanned_at": None, "scan_seconds": None, "error": None, "error_at": None, "report": None} def build_snapshot_payload(snapshots: Dict[str, dict], now: datetime) -> dict: """snapshot.json: one entry per dashboard TF. Timeframes not yet scanned appear as status="warming" so the page can render a placeholder instead of a hole. server_time_epoch lets the client compute its clock offset once so countdowns don't drift.""" timeframes = {} for tf in DASHBOARD_TFS: interval = TF_INTERVAL_MINUTES[tf] minutes = now.hour * 60 + now.minute secs_into = (minutes % interval) * 60 + now.second + now.microsecond / 1e6 next_close = now.timestamp() + (interval * 60 - secs_into) entry = _empty_tf_entry() entry.update(snapshots.get(tf.value) or {}) entry["next_close_epoch"] = next_close entry["next_scan_epoch"] = next_close + SCAN_BUFFER_SECONDS timeframes[tf.value] = entry return _sanitize({ "symbol": XAUUSD, "server_time_epoch": now.timestamp(), "market_closed": _providers_base._is_forex_weekend(now), "timeframes": timeframes, }) def publish(obj: dict, path: str) -> None: """Atomic JSON write: temp file + os.replace, so Apache never serves a torn file. allow_nan=False — a NaN slipping through would render the page blank (JSON.parse rejects bare NaN).""" data = json.dumps(obj, allow_nan=False, separators=(",", ":")) tmp = path + ".tmp" with open(tmp, "w", encoding="utf-8") as fh: fh.write(data) os.replace(tmp, path) # ══════════════════════════════════════════════════════════════════ # Runtime layer (constructed only from main(); never at import) # ══════════════════════════════════════════════════════════════════ def _setup_logging(level: str = "INFO") -> None: log_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "logs") os.makedirs(log_dir, exist_ok=True) log_file = os.path.join(log_dir, "dashboard.log") fmt = logging.Formatter( "%(asctime)s [%(levelname)s] %(name)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S") file_handler = TimedRotatingFileHandler( log_file, when="h", interval=3, backupCount=240, utc=True) file_handler.setFormatter(fmt) console_handler = logging.StreamHandler(sys.stdout) console_handler.setFormatter(fmt) logging.basicConfig( level=getattr(logging, level.upper(), logging.INFO), handlers=[file_handler, console_handler], force=True) class DashboardPublisher: def __init__(self, out_dir: str): self.out_dir = out_dir self.snapshots: Dict[str, dict] = {} self._reports = {} # tf.value → epoch seconds of the boundary the last successful scan # belonged to; feeds catch_up_timeframes (2026-09-13 audit). self._last_boundary: Dict[str, float] = {} self.news_payload: Optional[dict] = None self._dp = None self._news_engine: Optional[NewsEngine] = None def _provider(self): if self._dp is None: self._dp = make_data_provider(PlatformConfig()) return self._dp def _news(self) -> NewsEngine: if self._news_engine is None: self._news_engine = NewsEngine(PlatformConfig().news) return self._news_engine def install_static(self) -> None: """Copy the repo's static files (page + PHP summary endpoint) into the out dir (atomic), so git pull + restart also updates them. The ai/ cache dir is NOT created here — the daemon runs as a different user than Apache; that dir is a one-time deploy step (deploy/README-dashboard.md).""" static_dir = os.path.join( os.path.dirname(os.path.abspath(__file__)), "static") for name in STATIC_FILES: src = os.path.join(static_dir, name) if not os.path.exists(src): logger.warning(f"{src} missing — not installed") continue dst = os.path.join(self.out_dir, name) shutil.copyfile(src, dst + ".tmp") os.replace(dst + ".tmp", dst) def scan_tf(self, tf: Timeframe, boundary_epoch: Optional[float] = None) -> None: """Scan one timeframe and store its snapshot entry. `boundary_epoch` is the fallback boundary recorded when the report carries no analysis (weekend/blocked or failed scan) or its bar label will not parse; normally the boundary comes from the report itself (bar label + interval).""" started = time.time() try: report = produce_signal(XAUUSD, tf, self._provider(), CONFIRMATIONS[tf]) # Display-only violent-move watch (dark by default). Never # raises and never touches the signal — only watch_line. if violent_watch_enabled(): attach_watch(report, tf, get_app_config().violent_watch_settings) # Display-only "already open" line (a read-only query on the # bot's signals.db — this process still never writes it) and, # only when config.json sets trade_size.dashboard, the lot # size: the page is public and the size shows the owner's risk. attach_exposure(report, surface="dashboard") # Which boundary this report belongs to: the bar LABEL it closed # on plus one interval. Recorded so a boundary missed by an # overrunning cycle or a restart is caught up on the next wake # instead of sitting stale (2026-09-13 audit). boundary = boundary_epoch if report.primary_analysis is not None: try: boundary = (pd.Timestamp(report.timestamp).timestamp() + TF_INTERVAL_MINUTES[tf] * 60) except Exception: boundary = boundary_epoch if boundary is not None: self._last_boundary[tf.value] = boundary self._reports[tf.value] = report self.snapshots[tf.value] = { "status": "ok", "scanned_at": datetime.now(timezone.utc).isoformat(), "scan_seconds": round(time.time() - started, 2), "error": None, "error_at": None, "report": report_to_webapp_dict(report), } logger.info(f"{tf.value}: {report.final_signal.name} " f"q={report.signal_quality or '-'} " f"({time.time() - started:.1f}s)") except Exception: # Keep the last good report; the page shows it with an # error badge instead of going blank. logger.exception(f"{tf.value} scan failed") self._reports.pop(tf.value, None) entry = self.snapshots.get(tf.value) or _empty_tf_entry() entry["status"] = "error" # The public snapshot carries a stable message; exception # detail remains in dashboard.log for the operator. entry["error"] = "Scan unavailable" entry["error_at"] = datetime.now(timezone.utc).isoformat() self.snapshots[tf.value] = entry def attach_panels(self, tfs: List[Timeframe]) -> bool: """Add advisory panels after publishing the technical snapshot. A panel is display-only and can take tens of seconds on a cold LLM request. Keeping its report object locally lets the page receive the candle-close signal first, then a second atomic snapshot only when a panel produced a result. """ updated = False for tf in tfs: report = self._reports.get(tf.value) if report is None: continue if not attach_panel(report, surface="dashboard"): continue entry = self.snapshots.get(tf.value) if entry is None or entry.get("status") != "ok": continue entry["report"] = report_to_webapp_dict(report) updated = True return updated def refresh_news(self) -> None: try: ne = self._news() context = ne.get_fundamental_context() week = ne.get_week_events() self.news_payload = build_news_payload( context, week, datetime.now(timezone.utc)) except Exception: logger.exception("news refresh failed — keeping last payload") def publish_snapshot(self) -> None: publish(build_snapshot_payload( self.snapshots, datetime.now(timezone.utc)), os.path.join(self.out_dir, SNAPSHOT_FILE)) def publish_news(self) -> None: if self.news_payload is not None: publish(self.news_payload, os.path.join(self.out_dir, NEWS_FILE)) def run_cycle(self, tfs: List[Timeframe]) -> None: for tf in tfs: # sequential — see module docstring (cache dedup) self.scan_tf(tf) # Technical signals land before any optional slow work. News and the # advisory panel are display-only and may take 20-30 seconds cold. self.publish_snapshot() if self.attach_panels(tfs): self.publish_snapshot() self.refresh_news() self.publish_news() def run_forever(self) -> None: self.run_cycle(DASHBOARD_TFS) # populate the page at boot while True: delay = seconds_until_boundary(5, buffer=SCAN_BUFFER_SECONDS) logger.info(f"next M5 boundary in {delay:.0f}s") time.sleep(delay) try: now = datetime.now(timezone.utc) due = due_timeframes(now) logger.info(f"boundary wake — scanning {[t.value for t in due]}") missed = catch_up_timeframes(now, self._last_boundary) extra = [tf for tf in missed if tf not in due] if extra: logger.info(f"catch-up — scanning " f"{[t.value for t in extra]} (missed boundary)") self.run_cycle(due + extra) except Exception: logger.exception("cycle failed") def main(argv=None) -> None: ap = argparse.ArgumentParser(description="XAU/USD static dashboard publisher") ap.add_argument("--once", action="store_true", help="one scan+publish cycle, then exit") ap.add_argument("--out", default=os.environ.get("DASHBOARD_OUT_DIR", DEFAULT_OUT_DIR), help="output directory served by Apache " f"(default: $DASHBOARD_OUT_DIR or {DEFAULT_OUT_DIR})") args = ap.parse_args(argv) _setup_logging() # This process never writes a database. It is not always running, so # the Telegram bot is the single writer of signals.db and newsdata.db # (owner decision 2026-09-17). The switch is process-wide and beats # config.json / env; set here in main(), not at import, so importing # the module (the test suite does) leaves every other writer on. set_db_writes_for_process(False) logger.info("database writes disabled for this process " "(the Telegram bot is the only writer)") os.makedirs(args.out, exist_ok=True) pub = DashboardPublisher(args.out) pub.install_static() if args.once: pub.run_cycle(DASHBOARD_TFS) print(f"Published to {args.out}") return pub.run_forever() if __name__ == "__main__": main()