From f814485615f5b74ee133eebc104d40aaf15ca2d6 Mon Sep 17 00:00:00 2001 From: Harwinder Sekhon Date: Wed, 22 Jul 2026 17:18:14 -0400 Subject: [PATCH] Tiingo bid/ask spread monitor + logging hardening MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - tiingo_provider.py: /tiingo/fx/top quotes (10s TTL cache, graceful None on any failure); spread log writer. Signals never consume bid/ask; warn-only by design (no historical bid/ask to validate a gate against — live == backtest rule). - app_config: repo-root .env loader (setdefault; shell env still wins), tiingo_api_key resolver (TIINGO_API_KEY/TIINGO → config.json), telegram.channel_spread_detail flag (default false). - telegram_bot: 5-min spread sampler → logs/spread_log.csv (weekend- aware), alert-time quote refresh, bid/ask + wide-spread warning in signal messages, /status spread line. Raw numbers to owner DM always; channel only while channel_spread_detail=true (Tiingo free tier is internal-use-only; flip before public launch). Free-tier limit math: worst case ~41 req/h vs 50/h cap, one pair. - Logging: 3h rotation x 240 backups (30-day retention), full-date stamps, force=True root handlers; candle loops + sampler survive failed cycles; breach-task exceptions logged via done-callback; PTB error handler; sys/threading excepthooks in main(). - test_all.py: Module 14 (tiingo, dotenv, spread routing, loop resilience, rotation) — 215 tests green. Claude-Session: https://claude.ai/code/session_01HVRmy6KspMs96o5PTPYFAB --- CLAUDE.md | 18 +-- app_config.py | 70 ++++++++++- config.json.example | 6 +- telegram_bot.py | 217 ++++++++++++++++++++++++++++++-- test_all.py | 292 +++++++++++++++++++++++++++++++++++++++++++- tiingo_provider.py | 157 ++++++++++++++++++++++++ 6 files changed, 733 insertions(+), 27 deletions(-) create mode 100644 tiingo_provider.py diff --git a/CLAUDE.md b/CLAUDE.md index 041247a..0117b50 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,9 +13,10 @@ This file is the authoritative developer/agent reference. The human-facing docs | Path | Lines | Purpose | |------|-------|---------| | `config.py` | ~374 | All config dataclasses, `PAIR_INFO` registry, per-pair strategy configs. **`PlatformConfig` defaults ARE the tuned XAU config.** | -| `app_config.py` | ~244 | config.json loader/validation, env-precedence resolvers, `apply_pair_overrides` deployment overlay | -| `config.json.example` | 24 | committed schema template; real gitignored config.json holds secrets + deployment settings | +| `app_config.py` | ~290 | config.json loader/validation, env-precedence resolvers, repo-root `.env` loader (setdefault at import), `apply_pair_overrides` deployment overlay | +| `config.json.example` | 27 | committed schema template; real gitignored config.json holds secrets + deployment settings | | `data_provider.py` | ~347 | Twelve Data SDK wrapper: fetch, rate-limit (55 credits/min), in-memory cache, validation/cleaning, freshness checks | +| `tiingo_provider.py` | ~160 | Tiingo `/tiingo/fx/top` bid/ask quotes (supplementary, NOT an OHLCV source): spread visibility on alerts (warn-only) + 5-min sampler → `logs/spread_log.csv`. No key → feature off | | `engine.py` | ~1173 | `TechnicalEngine`: computes all indicators on one DataFrame → `TechnicalAnalysis` (composite score, regime, key levels) | | `chart_patterns.py` | ~280 | Chart-pattern detectors for the XAU_CP archetype (pure functions, ATR-unit geometry, first-close-beyond triggers) | | `signal_aggregator.py` | ~1447 | `SignalAggregator`: MTF analysis, news integration, gates/penalties, quality grade, trade-setup generation → `SignalReport` | @@ -24,13 +25,13 @@ This file is the authoritative developer/agent reference. The human-facing docs | `pair_signals/__init__.py` | 52 | `PAIR_PRODUCERS` registry + `produce_signal()` | | `pair_signals/xauusd.py` | 71 | XAU config factory + per-timeframe overrides (docstring = validation record) | | `main.py` | ~348 | CLI: `python main.py --pair XAUUSD --timeframe H1 --detail/--json` | -| `test_all.py` | ~2800 | QA suite: **200 tests in 13 modules**, no live API needed. Must stay green. | +| `test_all.py` | ~3100 | QA suite: **215 tests in 14 modules**, no live API needed. Must stay green. | | `backtest/gpu_backtest.py` | ~2231 | Bar-replay backtester (CuPy→NumPy fallback) using the real `SignalAggregator` | | `backtest/diagnose_filters.py` | 609 | Filter-diagnosis tooling | | `backtest/research/` | — | Calendar research infra (`ff_calendar_scraper.py`, `calendar_store.py`) + result CSVs | | `backtest/*.csv` | — | Verification trade books from past tuning campaigns (`xau_*.csv`, …). **Gitignored, local-only** — used for manual before/after comparison, not by tests | | `backtest/cache/` | — | Parquet OHLCV cache (raw bars) | -| `logs/telegram.log` | — | Bot log (midnight rotation, 30 backups) | +| `logs/telegram.log` | — | Bot log (3-hour rotation × 240 backups = 30-day retention); `logs/spread_log.csv` = Tiingo spread samples | --- @@ -272,8 +273,10 @@ Run: `python telegram_bot.py` (credentials resolve env `TELEGRAM_TOKEN`/`TELEGRA - **Alert gating**: quality threshold (`_alert_min_quality`, default "C", menu cycles C→B→A) + fingerprint dedup per (symbol, TF) on (signal, quality, direction, entry bucket) with 1-hour re-arm — a persisting identical signal doesn't re-send every candle, but a setup that moved ≥ half its risk distance (new pullback/swing anchor) re-alerts. The fingerprint is committed only **after** a message reaches ≥1 target (`_commit_alert`), so a failed Telegram send doesn't consume the alert. - **WebSocket monitor**: subscribes every `scan_pairs` symbol to TD quotes WS; 10 s heartbeat; exponential reconnect (≤60 s). On tick crossing any cached key level (`swing_high/low_1..3`, `ema_trend`, `bb_upper/lower`, `kc_upper/lower`; cache TTL 1 h) → immediate M15 rescan → "LEVEL BREAK" alert if actionable+changed (same gates as auto-scan), 5-min per-pair cooldown. - **Commands**: `/start` (menu), `/scan` (buttons: per pair — M5/M15/H1/H4 + Scan All TFs), `/autoscan` (toggle DM copy), `/status` (loops, WS, live prices, level cache, cooldowns), `/cache` (age per pair/TF), `/clearcache`, `/help`. +- **Tiingo spread monitor** (optional; on only when a Tiingo key resolves — see Environment variables): a 5-min sampler task fetches top-of-book bid/ask per scanned pair (`tiingo_provider.py`), logs to `logs/spread_log.csv` (skipping the weekend-closed window), and caches the latest quote; alert sends refresh the quote first, and signal messages then show live bid/ask + a warning when spread ≥ 2× the pair's assumed round-trip cost (`WIDE_SPREAD_MULTIPLE`). **Warn-only BY DESIGN** — no historical bid/ask exists, so a suppressing gate could never be validated (live == backtest rule); signals never consume bid/ask. `/status` shows the live spread per pair. **License (free tier is internal-use-only):** raw bid/ask/spread numbers go to owner DMs always; the channel gets them ONLY while config.json `telegram.channel_spread_detail=true` (the channel is internal/testing as of 2026-07-22 — **flip to false before the channel goes public**), otherwise a number-free "spread unusually wide" caution via `format_signal_message(report, spread_detail=False)` — routing in `_spread_detail_for`, test-enforced, committed default false. **Rate limits (free tier 50 req/h, 1000/day, checked 2026-07-22):** sampler 12/h + alert-batch refreshes ≤17/h + breach refreshes ≤12/h = worst case ≈41/h with ONE pair; adding pairs multiplies this — recheck before growing `scan_pairs`. - Signal messages include the full indicator breakdown, warnings, trade setup, **trailing-stop management plan** (arm price + trail distance, from Signal/pb params by `strategy`), and a risk disclaimer on every outbound message. -- Blocking scan work runs in `ThreadPoolExecutor(max_workers=4)`; one shared `ForexDataProvider` across all scans (cache reuse). Logs to `logs/telegram.log`. +- Blocking scan work runs in `ThreadPoolExecutor(max_workers=4)`; one shared `ForexDataProvider` across all scans (cache reuse). +- **Logging (hardened 2026-07-22)**: root-logger handlers (`force=True`) → console + `logs/telegram.log` (3-hour rotation, 240 backups = 30 days), full-date timestamps; every module logger (data_provider, tiingo_provider, news_engine, websockets, telegram.ext) lands in the same file. Failure surfaces all covered: candle loops and the spread sampler catch-and-continue per cycle (one bad scan can't kill a loop), breach tasks log exceptions via done-callback, PTB `add_error_handler` registered, and `sys.excepthook`/`threading.excepthook` (installed in `main()`) route uncaught crashes to the log. WS monitor keeps its exponential-backoff reconnect + `logger.exception`. --- @@ -318,13 +321,13 @@ Data: paginated TD fetches (5000-bar chunks) cached as parquet in `backtest/cach ## test_all.py — QA suite -`python test_all.py` — **200 tests, 13 modules** (config, data_provider, engine, news_engine, signal_aggregator, main, telegram_bot, integration, pair_signals registry + engine-bug regressions, 2026-07-16 audit-fix regressions, chart patterns, app_config). No network needed (mocked). Uses a custom `@test("name")` decorator that runs immediately (NOT pytest — don't grep for `def test_`). Exit code non-zero on failure. Includes: the xauusd-factory ≡ inline-config equivalence test, engine bug-fix regression tests (Module 9b), a live-vs-GPU engine parity sweep incl. the EMA-ribbon path (Module 11), and config.json loading/precedence/validation coverage (Module 13). **Any signal-logic change must keep this green and should add tests.** +`python test_all.py` — **215 tests, 14 modules** (config, data_provider, engine, news_engine, signal_aggregator, main, telegram_bot, integration, pair_signals registry + engine-bug regressions, 2026-07-16 audit-fix regressions, chart patterns, app_config, tiingo_provider). No network needed (mocked). Uses a custom `@test("name")` decorator that runs immediately (NOT pytest — don't grep for `def test_`). Exit code non-zero on failure. Includes: the xauusd-factory ≡ inline-config equivalence test, engine bug-fix regression tests (Module 9b), a live-vs-GPU engine parity sweep incl. the EMA-ribbon path (Module 11), config.json loading/precedence/validation coverage (Module 13), and Tiingo quote/spread-monitor + `.env` loader coverage (Module 14). **Any signal-logic change must keep this green and should add tests.** --- ## Environment variables -Precedence for every credential/deployment setting: **env var → config.json → none**. Hardcoded fallbacks (previously in `config.py` / `telegram_bot.py`) were **REMOVED 2026-07-21**; the bot now refuses to boot (`telegram_bot.main`) and data fetches fail with a named error (`data_provider`) if a secret resolves to neither source. See `app_config.py` and `config.json.example`. +Precedence for every credential/deployment setting: **env var → config.json → none**. A repo-root `.env` file (gitignored) is loaded at `app_config` import via `os.environ.setdefault` — it behaves like real env vars except a real shell export always wins. Hardcoded fallbacks (previously in `config.py` / `telegram_bot.py`) were **REMOVED 2026-07-21**; the bot now refuses to boot (`telegram_bot.main`) and data fetches fail with a named error (`data_provider`) if a secret resolves to neither source. See `app_config.py` and `config.json.example`. | Variable | Purpose | Default | |---|---|---| @@ -332,6 +335,7 @@ Precedence for every credential/deployment setting: **env var → config.json | `TELEGRAM_TOKEN` | bot token | none — falls through to config.json `telegram.token`, then refuses to boot | | `TELEGRAM_CHAT_ID` | owner chat id (boot auto-scan + owner lock) | none — falls through to config.json `telegram.chat_id`, then refuses to boot | | `TELEGRAM_CHANNEL_ID` | subscriber channel (`-100…`); `""` disables the feed | none — falls through to config.json `telegram.channel_id` | +| `TIINGO_API_KEY` (or `TIINGO`) | Tiingo bid/ask quote key for the spread monitor — optional | none — falls through to config.json `data_providers.tiingo.api_key`; absent = spread monitor off (never fatal) | | `FB2_EMA_LO` | EMA200 "near" penalty threshold in % | `0.5` | Dependencies (`requirements.txt`): numpy, pandas, requests, twelvedata, python-telegram-bot ≥20, websockets. Python 3.12. A `venv/` exists in-repo. diff --git a/app_config.py b/app_config.py index 1cb9e19..c9052e4 100644 --- a/app_config.py +++ b/app_config.py @@ -9,9 +9,12 @@ alert-quality boot default, log level. Signal logic, weights, thresholds and gates stay in config.py / pair_signals. Precedence for every setting: env var → config.json → code default. -Secrets have NO code default — with neither env nor config.json the -bot refuses to start (telegram_bot.main) and data fetches fail with a -named error (data_provider). +A repo-root .env file (gitignored) is loaded at import via setdefault, +so it behaves exactly like real environment variables except that a +real shell export always wins. Secrets have NO code default — with +neither env nor config.json the bot refuses to start +(telegram_bot.main) and data fetches fail with a named error +(data_provider). Parity contract (test-enforced): with no config.json, or one holding the config.json.example defaults, every produced PlatformConfig is @@ -34,6 +37,40 @@ logger = logging.getLogger(__name__) CONFIG_PATH = os.path.join( os.path.dirname(os.path.abspath(__file__)), "config.json") +ENV_PATH = os.path.join( + os.path.dirname(os.path.abspath(__file__)), ".env") + + +def load_dotenv(path: str = ENV_PATH) -> None: + """Minimal .env loader (no python-dotenv dependency). + + KEY=VALUE lines land in os.environ via setdefault — a variable + already exported in the shell always wins, keeping the + env → config.json precedence intact (.env is just a convenient + place to put env vars). Comments, blank lines, ``export `` prefixes + and single/double quotes are handled; anything else is skipped. + """ + try: + with open(path, encoding="utf-8") as fh: + lines = fh.read().splitlines() + except OSError: + return + for line in lines: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + if line.startswith("export "): + line = line[len("export "):] + key, _, value = line.partition("=") + key, value = key.strip(), value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + value = value[1:-1] + if key: + os.environ.setdefault(key, value) + + +# Module import == process boot, before any resolver can run. +load_dotenv() _VALID_QUALITIES = ("A+", "A", "B", "C") _VALID_LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL") @@ -42,7 +79,8 @@ _VALID_LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL") _SCANNABLE_TFS = ("M5", "M15", "H1", "H4") _KNOWN_TOP = {"telegram", "data_providers", "scanning", "pairs", "log_level"} -_KNOWN_TELEGRAM = {"token", "chat_id", "channel_id", "alert_min_quality"} +_KNOWN_TELEGRAM = {"token", "chat_id", "channel_id", "alert_min_quality", + "channel_spread_detail"} _KNOWN_SCANNING = {"pairs", "timeframes"} _KNOWN_PAIR_KEYS = {"blocked_hours_utc"} @@ -57,6 +95,11 @@ class AppConfig: telegram_chat_id: str = "" telegram_channel_id: str = "" # "" = channel feed disabled alert_min_quality: str = "C" + # True = the channel is INTERNAL (testing) and may receive raw Tiingo + # bid/ask numbers. Default False: Tiingo's free tier is licensed + # internal-use-only, so a public/subscriber channel must only ever + # get the number-free wide-spread caution. Flip back before launch. + channel_spread_detail: bool = False active_provider: str = "twelve_data" provider_settings: Dict[str, Dict[str, Any]] = field(default_factory=dict) scan_pairs: List[str] = field(default_factory=lambda: [XAUUSD]) @@ -105,6 +148,11 @@ def load_app_config(path: str = CONFIG_PATH) -> AppConfig: f'config.json: telegram.alert_min_quality "{quality}" ' f"not in {_VALID_QUALITIES}") app.alert_min_quality = quality + spread_detail = tg.get("channel_spread_detail", app.channel_spread_detail) + if not isinstance(spread_detail, bool): + raise AppConfigError( + "config.json: telegram.channel_spread_detail must be a boolean") + app.channel_spread_detail = spread_detail dp = raw.get("data_providers") if dp is not None: @@ -227,6 +275,20 @@ def provider_api_key(app: Optional[AppConfig] = None) -> str: return str(block.get("api_key", "")) +def tiingo_api_key(app: Optional[AppConfig] = None) -> str: + """Key for the SUPPLEMENTARY Tiingo bid/ask quote provider (spread + monitor) — never a data_providers.active candidate; Twelve Data + stays the OHLCV source. Empty string = feature disabled. + Env TIINGO_API_KEY (or short TIINGO, e.g. from .env) → + config.json data_providers.tiingo.api_key → "".""" + app = app or get_app_config() + env = os.environ.get("TIINGO_API_KEY") or os.environ.get("TIINGO") + if env: + return env + block = app.provider_settings.get("tiingo", {}) + return str(block.get("api_key", "")) + + # ── PlatformConfig overlay ──────────────────────────────────────────── def apply_pair_overrides( diff --git a/config.json.example b/config.json.example index 11fdbf4..9177e31 100644 --- a/config.json.example +++ b/config.json.example @@ -3,12 +3,16 @@ "token": "123456789:REPLACE_WITH_YOUR_BOT_TOKEN", "chat_id": "REPLACE_WITH_OWNER_CHAT_ID", "channel_id": "-100REPLACE_WITH_CHANNEL_ID_OR_EMPTY_STRING", - "alert_min_quality": "C" + "alert_min_quality": "C", + "channel_spread_detail": false }, "data_providers": { "active": "twelve_data", "twelve_data": { "api_key": "REPLACE_WITH_YOUR_TWELVE_DATA_API_KEY" + }, + "tiingo": { + "api_key": "" } }, "scanning": { diff --git a/telegram_bot.py b/telegram_bot.py index d12c65d..cd4ae62 100755 --- a/telegram_bot.py +++ b/telegram_bot.py @@ -43,6 +43,7 @@ import logging import logging.handlers import os import sys +import threading import time from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone @@ -66,9 +67,12 @@ from config import ( ) from app_config import ( get_app_config, provider_api_key, telegram_token, telegram_chat_id, - telegram_channel_id, + telegram_channel_id, tiingo_api_key, ) from data_provider import ForexDataProvider, make_data_provider +from tiingo_provider import ( + Quote, TiingoQuoteProvider, WIDE_SPREAD_MULTIPLE, log_quote, +) from pair_signals import PAIR_PRODUCERS from signal_aggregator import SignalAggregator, SignalReport @@ -81,17 +85,24 @@ if hasattr(sys.stdout, "reconfigure"): _LOG_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "logs") os.makedirs(_LOG_DIR, exist_ok=True) _log_formatter = logging.Formatter( - "%(asctime)s [%(levelname)s] %(name)s: %(message)s", datefmt="%H:%M:%S" + "%(asctime)s [%(levelname)s] %(name)s: %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", ) _console_handler = logging.StreamHandler() _console_handler.setFormatter(_log_formatter) +# 3-hour rotation × 240 backups = 30 days retention. Handlers sit on the +# ROOT logger (force=True guarantees ownership even if something logged +# first), so every module logger — data_provider, tiingo_provider, +# news_engine, signal_aggregator, websockets, telegram.ext — lands in +# the same file. _file_handler = logging.handlers.TimedRotatingFileHandler( os.path.join(_LOG_DIR, "telegram.log"), - when="midnight", backupCount=30, encoding="utf-8", + when="h", interval=3, backupCount=240, encoding="utf-8", ) _file_handler.setFormatter(_log_formatter) logging.basicConfig(level=getattr(logging, _APP.log_level, logging.INFO), - handlers=[_console_handler, _file_handler]) + handlers=[_console_handler, _file_handler], + force=True) logging.getLogger("httpx").setLevel(logging.WARNING) logging.getLogger("httpcore").setLevel(logging.WARNING) logger = logging.getLogger("telegram_bot") @@ -202,6 +213,32 @@ def _get_shared_dp() -> ForexDataProvider: return _shared_dp +# ── Tiingo bid/ask spread monitor (optional; no key → feature off) ─── +# Warn-only BY DESIGN: there is no historical bid/ask to replay, so a +# suppressing gate could never be validated (live == backtest rule). +# Signals never consume bid/ask — this is execution-cost visibility. + +_TIINGO_KEY = tiingo_api_key() +_tiingo: Optional[TiingoQuoteProvider] = ( + TiingoQuoteProvider(_TIINGO_KEY) if _TIINGO_KEY else None +) +_last_quotes: Dict[str, Quote] = {} +_QUOTE_MAX_AGE = 600 # cached quote stays displayable this long +_SPREAD_SAMPLE_SECONDS = 300 # sampler cadence (→ logs/spread_log.csv) + +# config.json telegram.channel_spread_detail: True while the channel is +# internal/testing → it gets raw bid/ask like the owner DM. MUST be +# flipped back to false before the channel goes public (Tiingo free +# tier is internal-use-only). +_CHANNEL_SPREAD_DETAIL = _APP.channel_spread_detail + + +def _spread_detail_for(chat_id: int) -> bool: + """Owner DMs always get raw quote numbers; the channel only while + config.json marks it internal.""" + return chat_id != CHANNEL_ID or _CHANNEL_SPREAD_DETAIL + + # ── Signal formatting ───────────────────────────────────────────────── _SIGNAL_EMOJI = { @@ -257,6 +294,37 @@ def _live_price_line(symbol: str) -> Optional[str]: return f"📡 Live: *{_fmt_price(price)}* ({age:.0f}s tick)" +def _spread_lines(symbol: str, include_quote: bool = True) -> List[str]: + """Live bid/ask + wide-spread warning for signal messages. + Empty when Tiingo is off or the cached quote is stale. + + include_quote=False is the subscriber-channel variant: Tiingo's + free tier is licensed internal-use-only (no redistributing the + data), so raw bid/ask/spread numbers stay in the owner DM — the + channel gets only a number-free wide-spread caution.""" + q = _last_quotes.get(symbol) + if q is None or q.age_seconds() > _QUOTE_MAX_AGE: + return [] + assumed = PAIR_INFO.get(symbol, {}).get("default_spread", 0.0) + wide = assumed > 0 and q.spread >= WIDE_SPREAD_MULTIPLE * assumed + if not include_quote: + return [ + "⚠️ Spread unusually wide right now — execution cost is " + "elevated, consider waiting for it to normalize" + ] if wide else [] + lines = [ + f"💱 Bid/Ask: {_fmt_price(q.bid)} / {_fmt_price(q.ask)}" + f" (spread ${q.spread:.2f})" + ] + if wide: + lines.append( + f"⚠️ Wide spread: ${q.spread:.2f} ≈ {q.spread / assumed:.1f}× the " + f"${assumed:.2f} cost the strategy was validated at — " + f"execution is expensive right now" + ) + return lines + + _DISCLAIMER = ( "⚠️ _Not financial advice. For educational and research purposes only. " "Trading forex/CFDs carries a high risk of loss — do your own analysis " @@ -264,7 +332,7 @@ _DISCLAIMER = ( ) -def format_signal_message(report: SignalReport) -> str: +def format_signal_message(report: SignalReport, spread_detail: bool = True) -> str: sig = report.final_signal emoji = _SIGNAL_EMOJI.get(sig, "⬜") qe = _QUALITY_EMOJI.get(report.signal_quality, "") @@ -285,6 +353,7 @@ def format_signal_message(report: SignalReport) -> str: live = _live_price_line(report.symbol) if live: lines.append(live) + lines += _spread_lines(report.symbol, include_quote=spread_detail) lines += [ f"📊 Regime: {regime_str} | Vol: {vol_str}", "", @@ -477,6 +546,24 @@ async def run_signal( ) +async def _refresh_quotes(symbols: List[str]) -> None: + """Fetch fresh Tiingo quotes (blocking HTTP) off the event loop — + called just before alerts go out so the spread block reflects the + moment of entry, and by the 5-min sampler.""" + if _tiingo is None: + return + loop = asyncio.get_running_loop() + for symbol in set(symbols): + try: + quote = await loop.run_in_executor( + _executor, _tiingo.get_quote, symbol) + except Exception as exc: + logger.warning(f"Tiingo quote refresh failed for {symbol}: {exc}") + continue + if quote is not None: + _last_quotes[symbol] = quote + + # ── Alert gating (quality threshold + dedup) ───────────────────────── _QUALITY_RANK = {"A+": 4, "A": 3, "B": 2, "C": 1} @@ -589,10 +676,13 @@ async def _scan_timeframe( if _passes_quality(r.signal_quality) and _should_alert(f"{r.symbol}_{tf.value}", r) ] + if actionable: + await _refresh_quotes([r.symbol for r in actionable]) sent_ok: Set[int] = set() for chat_id in _signal_targets(): for i, report in enumerate(actionable): - detail = format_signal_message(report) + detail = format_signal_message( + report, spread_detail=_spread_detail_for(chat_id)) try: await bot.send_message(chat_id, detail, parse_mode=ParseMode.MARKDOWN) sent_ok.add(i) @@ -690,9 +780,11 @@ async def _handle_breach(bot, ws_symbol: str, pair_symbol: str, price: float, le f"⚡ *LEVEL BREAK* — {_escape_md(display)} crossed " f"*{_escape_md(label)}* @ `{_fmt_price(price)}`\n\n" ) - detail = header + format_signal_message(report) + await _refresh_quotes([pair_symbol]) sent = False for chat_id in _signal_targets(): + detail = header + format_signal_message( + report, spread_detail=_spread_detail_for(chat_id)) try: await bot.send_message(chat_id, detail, parse_mode=ParseMode.MARKDOWN) sent = True @@ -702,6 +794,15 @@ async def _handle_breach(bot, ws_symbol: str, pair_symbol: str, price: float, le _commit_alert(f"{pair_symbol}_{Timeframe.M15.value}", report) +def _breach_task_done(task: asyncio.Task) -> None: + """Fire-and-forget breach tasks must still surface their failures — + without this, an exception sits unretrieved until GC and never + reaches the log file.""" + _breach_tasks.discard(task) + if not task.cancelled() and task.exception() is not None: + logger.error("Breach alert task failed", exc_info=task.exception()) + + async def _ws_heartbeat(ws) -> None: """Twelve Data drops idle WS connections; heartbeat every 10s keeps it alive through quiet periods (weekends, low-liquidity hours).""" @@ -764,7 +865,7 @@ async def _ws_monitor(bot) -> None: _handle_breach(bot, ws_symbol, pair_symbol, price, breached) ) _breach_tasks.add(task) - task.add_done_callback(_breach_tasks.discard) + task.add_done_callback(_breach_task_done) _cleanup_cooldowns() @@ -791,15 +892,51 @@ async def _candle_loop(bot, tf: Timeframe, interval_minutes: int) -> None: if not _feed_active(): break logger.info(f"{label} auto-scan: firing") - await _scan_timeframe( - bot, tf, AUTOSCAN_CONFIRMATIONS[tf] - ) + try: + await _scan_timeframe( + bot, tf, AUTOSCAN_CONFIRMATIONS[tf] + ) + except Exception: + # One failed cycle must not kill the loop for good — + # log it and wait for the next candle boundary. + logger.exception(f"{label} auto-scan: cycle failed") except asyncio.CancelledError: pass finally: logger.info(f"{label} auto-scan loop stopped") +def _weekend_closed(now: datetime) -> bool: + """Fri 22:00 → Sun 22:00 UTC — the window the platform treats as + market-closed (same as the weekend filter).""" + wd, hr = now.weekday(), now.hour + return (wd == 4 and hr >= 22) or wd == 5 or (wd == 6 and hr < 22) + + +async def _spread_sampler_loop() -> None: + """Every 5 min: fetch bid/ask for each scanned pair, cache it for + the alert formatter, and append to logs/spread_log.csv — the + dataset that validates (or refutes) the assumed $0.30 round-trip + cost every backtest charges.""" + try: + while True: + try: + if not _weekend_closed(datetime.now(timezone.utc)): + await _refresh_quotes(SCAN_PAIRS) + for symbol in SCAN_PAIRS: + q = _last_quotes.get(symbol) + # Age guard: never log a stale row after a failed fetch. + if q is not None and q.age_seconds() < _SPREAD_SAMPLE_SECONDS: + log_quote(q) + except Exception: + logger.exception("Spread sampler: cycle failed") + await asyncio.sleep(_SPREAD_SAMPLE_SECONDS) + except asyncio.CancelledError: + pass + finally: + logger.info("Spread sampler stopped") + + _TF_INTERVAL_MINUTES = { Timeframe.M5: 5, Timeframe.M15: 15, Timeframe.M30: 30, Timeframe.H1: 60, Timeframe.H4: 240, Timeframe.D1: 1440, @@ -810,10 +947,11 @@ _AUTOSCAN_LOOPS = [ _ws_task: Optional[asyncio.Task] = None +_spread_task: Optional[asyncio.Task] = None def _start_scheduler(bot) -> None: - global _ws_task + global _ws_task, _spread_task alive = [t for t in _autoscan_tasks if not t.done()] if not alive: _autoscan_tasks.clear() @@ -828,15 +966,23 @@ def _start_scheduler(bot) -> None: _ws_task = asyncio.create_task(_ws_monitor(bot)) logger.info("WebSocket level-breach monitor started") + if _tiingo is not None and (_spread_task is None or _spread_task.done()): + _spread_task = asyncio.create_task(_spread_sampler_loop()) + logger.info("Tiingo spread sampler started (every %ss)", + _SPREAD_SAMPLE_SECONDS) + def _stop_scheduler() -> None: - global _ws_task + global _ws_task, _spread_task for t in _autoscan_tasks: t.cancel() _autoscan_tasks.clear() if _ws_task and not _ws_task.done(): _ws_task.cancel() _ws_task = None + if _spread_task and not _spread_task.done(): + _spread_task.cancel() + _spread_task = None for t in list(_breach_tasks): t.cancel() _breach_tasks.clear() @@ -934,6 +1080,17 @@ def _status_text() -> str: else: lines.append("❌ WebSocket: *disconnected*") + if _tiingo is not None: + for pair in SCAN_PAIRS: + q = _last_quotes.get(pair) + display = PAIR_INFO.get(pair, {}).get("display", pair) + if q is None: + lines.append(f"💱 Spread ({display}): waiting for first quote") + else: + lines.append( + f"💱 Spread ({display}): `${q.spread:.2f}` " + f"({q.age_seconds():.0f}s ago)") + if _last_prices: lines.append("\n*Live Prices (WebSocket):*") for ws_sym, pair in _WS_TO_PAIR.items(): @@ -1266,6 +1423,38 @@ async def _post_shutdown(app: Application) -> None: # ── Entry point ─────────────────────────────────────────────────────── +async def _on_error(update: object, context: ContextTypes.DEFAULT_TYPE) -> None: + """python-telegram-bot routes every unhandled handler exception here; + without a registered error handler PTB only half-logs them.""" + logger.error( + "Unhandled error in Telegram handler (update=%s)", + getattr(update, "update_id", update), + exc_info=context.error, + ) + + +def _install_excepthooks() -> None: + """Route uncaught exceptions from the main thread and worker threads + into the rotating log file instead of bare stderr, so a crash right + before exit is still recorded.""" + def hook(exc_type, exc, tb): + if issubclass(exc_type, KeyboardInterrupt): + sys.__excepthook__(exc_type, exc, tb) + return + logger.critical("Uncaught exception", exc_info=(exc_type, exc, tb)) + + def thread_hook(args): + if issubclass(args.exc_type, SystemExit): + return + name = args.thread.name if args.thread else "?" + logger.critical( + f"Uncaught exception in thread {name}", + exc_info=(args.exc_type, args.exc_value, args.exc_traceback)) + + sys.excepthook = hook + threading.excepthook = thread_hook + + def main() -> None: if not TOKEN or not CHAT_ID: raise SystemExit( @@ -1273,6 +1462,7 @@ def main() -> None: "and telegram.chat_id in config.json, or the TELEGRAM_TOKEN and " "TELEGRAM_CHAT_ID environment variables.") + _install_excepthooks() logger.info("Starting ForexBot2 Telegram bot…") app = ( @@ -1292,6 +1482,7 @@ def main() -> None: app.add_handler(CommandHandler("clearcache", cmd_clearcache)) app.add_handler(CallbackQueryHandler(cb_scan, pattern="^scan_")) app.add_handler(CallbackQueryHandler(cb_menu, pattern="^menu_")) + app.add_error_handler(_on_error) logger.info("Bot running — press Ctrl+C to stop") app.run_polling(allowed_updates=Update.ALL_TYPES) diff --git a/test_all.py b/test_all.py index 503c623..36c65b3 100644 --- a/test_all.py +++ b/test_all.py @@ -25,7 +25,7 @@ sys.path.insert(0, os.path.dirname(__file__)) import json import app_config for _k in ("TELEGRAM_TOKEN", "TELEGRAM_CHAT_ID", "TELEGRAM_CHANNEL_ID", - "TWELVE_DATA_API_KEY"): + "TWELVE_DATA_API_KEY", "TIINGO_API_KEY", "TIINGO"): os.environ.pop(_k, None) _TEST_APP = app_config.AppConfig( provider_settings={"twelve_data": {"api_key": "test-key"}} @@ -2607,7 +2607,7 @@ def _write_json(payload) -> str: _FULL_CFG = { "telegram": {"token": "t0k", "chat_id": "111", "channel_id": "-100222", - "alert_min_quality": "B"}, + "alert_min_quality": "B", "channel_spread_detail": True}, "data_providers": {"active": "twelve_data", "twelve_data": {"api_key": "json-key"}}, "scanning": {"pairs": ["XAU/USD"], "timeframes": ["M15", "H1"]}, @@ -2623,6 +2623,7 @@ def _(): assert app.telegram_chat_id == "111" assert app.telegram_channel_id == "-100222" assert app.alert_min_quality == "B" + assert app.channel_spread_detail is True assert app.active_provider == "twelve_data" assert app.provider_settings["twelve_data"]["api_key"] == "json-key" assert app.scan_pairs == [XAUUSD] @@ -2676,6 +2677,7 @@ def _(): {"data_providers": {"active": "polygon", "twelve_data": {"api_key": "k"}}}, # no block {"telegram": {"alert_min_quality": "Z"}}, # bad quality + {"telegram": {"channel_spread_detail": "yes"}}, # not a bool {"log_level": "CHATTY"}, # bad level ] for payload in bad_payloads: @@ -2857,6 +2859,292 @@ def _(): Timeframe.H1, Timeframe.H4] +# ══════════════════════════════════════════════════════════════════════ +# MODULE 14: tiingo_provider.py (bid/ask spread monitor) +# ══════════════════════════════════════════════════════════════════════ +print("\n═══ Module 14: tiingo_provider.py ═══") + +from tiingo_provider import ( + Quote, TiingoQuoteProvider, WIDE_SPREAD_MULTIPLE, log_quote, + tiingo_ticker, +) +from app_config import load_dotenv, tiingo_api_key + + +def _tiingo_response(payload): + resp = MagicMock() + resp.json.return_value = payload + resp.raise_for_status.return_value = None + return resp + + +_TIINGO_ROW = [{ + "ticker": "xauusd", + "quoteTimestamp": "2026-07-22T12:00:00+00:00", + "bidPrice": 2400.10, "bidSize": 1000000.0, + "askPrice": 2400.40, "askSize": 1000000.0, + "midPrice": 2400.25, +}] + + +@test("tiingo: ticker mapping XAU/USD → xauusd") +def _(): + assert tiingo_ticker("XAU/USD") == "xauusd" + assert tiingo_ticker("EUR/USD") == "eurusd" + + +@test("tiingo: quote parsed — spread = ask − bid, canonical symbol") +def _(): + provider = TiingoQuoteProvider("k") + with patch("tiingo_provider.requests.get", + return_value=_tiingo_response(_TIINGO_ROW)) as mock_get: + q = provider.get_quote("XAU/USD") + assert q is not None + assert q.symbol == "XAU/USD" + assert abs(q.bid - 2400.10) < 1e-9 and abs(q.ask - 2400.40) < 1e-9 + assert abs(q.spread - 0.30) < 1e-9 + assert abs(q.mid - 2400.25) < 1e-9 + assert q.quote_time is not None and q.quote_time.tzinfo is not None + assert q.age_seconds() < 5 + params = mock_get.call_args.kwargs["params"] + assert params["tickers"] == "xauusd" and params["token"] == "k" + + +@test("tiingo: mid falls back to (bid+ask)/2 when midPrice missing") +def _(): + row = [dict(_TIINGO_ROW[0], midPrice=None)] + provider = TiingoQuoteProvider("k") + with patch("tiingo_provider.requests.get", + return_value=_tiingo_response(row)): + q = provider.get_quote("XAU/USD") + assert q is not None and abs(q.mid - 2400.25) < 1e-9 + + +@test("tiingo: TTL cache dedupes calls; ttl=0 refetches") +def _(): + provider = TiingoQuoteProvider("k") + with patch("tiingo_provider.requests.get", + return_value=_tiingo_response(_TIINGO_ROW)) as mock_get: + provider.get_quote("XAU/USD") + provider.get_quote("XAU/USD") + assert mock_get.call_count == 1, "second call within TTL must hit cache" + provider = TiingoQuoteProvider("k", quote_ttl=0.0) + with patch("tiingo_provider.requests.get", + return_value=_tiingo_response(_TIINGO_ROW)) as mock_get: + provider.get_quote("XAU/USD") + provider.get_quote("XAU/USD") + assert mock_get.call_count == 2 + + +@test("tiingo: every failure path returns None (never raises)") +def _(): + provider = TiingoQuoteProvider("k", quote_ttl=0.0) + # network error + with patch("tiingo_provider.requests.get", + side_effect=Exception("boom")): + assert provider.get_quote("XAU/USD") is None + # HTTP error + resp = _tiingo_response(_TIINGO_ROW) + resp.raise_for_status.side_effect = Exception("401") + with patch("tiingo_provider.requests.get", return_value=resp): + assert provider.get_quote("XAU/USD") is None + # empty / non-list / null-price payloads + for payload in ([], {"detail": "unknown ticker"}, + [dict(_TIINGO_ROW[0], bidPrice=None)], + [dict(_TIINGO_ROW[0], askPrice=None)]): + with patch("tiingo_provider.requests.get", + return_value=_tiingo_response(payload)): + assert provider.get_quote("XAU/USD") is None, \ + f"payload {payload!r} must yield None" + + +@test("tiingo: log_quote writes header once then appends rows") +def _(): + q = Quote(symbol=XAUUSD, bid=2400.10, ask=2400.40, mid=2400.25, + spread=0.30, quote_time=None, fetched_at=time.time()) + with tempfile.TemporaryDirectory() as td: + path = os.path.join(td, "sub", "spread_log.csv") + log_quote(q, path) + log_quote(q, path) + with open(path, encoding="utf-8") as fh: + lines = fh.read().strip().splitlines() + assert lines[0] == "ts_utc,symbol,bid,ask,mid,spread" + assert len(lines) == 3, "header + one row per log_quote call" + cells = lines[1].split(",") + assert cells[1] == XAUUSD and float(cells[5]) == 0.30 + + +@test("app_config: tiingo_api_key precedence env → env-short → json → ''") +def _(): + app = AppConfig(provider_settings={ + "twelve_data": {"api_key": "td"}, + "tiingo": {"api_key": "json-tiingo"}, + }) + with patch.dict(os.environ, {}, clear=True): + assert tiingo_api_key(app) == "json-tiingo" + with patch.dict(os.environ, {"TIINGO": "short-env"}, clear=True): + assert tiingo_api_key(app) == "short-env" + with patch.dict(os.environ, {"TIINGO": "short-env", + "TIINGO_API_KEY": "long-env"}, clear=True): + assert tiingo_api_key(app) == "long-env" + with patch.dict(os.environ, {}, clear=True): + assert tiingo_api_key(AppConfig()) == "", "no key anywhere = disabled" + + +@test("app_config: load_dotenv parses lines, never overrides real env") +def _(): + fh = tempfile.NamedTemporaryFile( + "w", suffix=".env", delete=False, encoding="utf-8") + fh.write("# comment\n\nPLAIN=abc\nexport EXPORTED=def\n" + 'QUOTED="with spaces"\nSINGLE=\'sq\'\nEXISTING=from-dotenv\n' + "NOT A KV LINE\n") + fh.close() + with patch.dict(os.environ, {"EXISTING": "from-shell"}, clear=True): + load_dotenv(fh.name) + assert os.environ["PLAIN"] == "abc" + assert os.environ["EXPORTED"] == "def" + assert os.environ["QUOTED"] == "with spaces" + assert os.environ["SINGLE"] == "sq" + assert os.environ["EXISTING"] == "from-shell", \ + "a real shell export must beat .env" + load_dotenv("/nonexistent/.env") # missing file is a silent no-op + + +@test("telegram_bot: _spread_lines fresh/wide/stale/absent") +def _(): + import telegram_bot as tb + try: + # fresh, normal spread → bid/ask line only + tb._last_quotes[XAUUSD] = Quote( + symbol=XAUUSD, bid=2400.10, ask=2400.40, mid=2400.25, + spread=0.30, quote_time=None, fetched_at=time.time()) + lines = tb._spread_lines(XAUUSD) + assert len(lines) == 1 and "Bid/Ask" in lines[0] + assert "spread $0.30" in lines[0] + # wide spread (≥ 2× the $0.30 assumption) → warning appended + tb._last_quotes[XAUUSD] = Quote( + symbol=XAUUSD, bid=2400.10, ask=2401.00, mid=2400.55, + spread=0.90, quote_time=None, fetched_at=time.time()) + lines = tb._spread_lines(XAUUSD) + assert len(lines) == 2 and "Wide spread" in lines[1] + assert "3.0×" in lines[1] + # stale quote → nothing + tb._last_quotes[XAUUSD] = Quote( + symbol=XAUUSD, bid=2400.10, ask=2400.40, mid=2400.25, + spread=0.30, quote_time=None, + fetched_at=time.time() - tb._QUOTE_MAX_AGE - 1) + assert tb._spread_lines(XAUUSD) == [] + # no quote at all → nothing + tb._last_quotes.pop(XAUUSD, None) + assert tb._spread_lines(XAUUSD) == [] + finally: + tb._last_quotes.pop(XAUUSD, None) + + +@test("telegram_bot: channel variant never leaks Tiingo numbers") +def _(): + # Tiingo free tier is internal-use-only: the subscriber channel may + # see a number-free caution, never bid/ask/spread values. + import telegram_bot as tb + try: + # normal spread → channel variant shows nothing at all + tb._last_quotes[XAUUSD] = Quote( + symbol=XAUUSD, bid=2400.10, ask=2400.40, mid=2400.25, + spread=0.30, quote_time=None, fetched_at=time.time()) + assert tb._spread_lines(XAUUSD, include_quote=False) == [] + # wide spread → one caution line with no prices in it + tb._last_quotes[XAUUSD] = Quote( + symbol=XAUUSD, bid=2400.10, ask=2401.00, mid=2400.55, + spread=0.90, quote_time=None, fetched_at=time.time()) + lines = tb._spread_lines(XAUUSD, include_quote=False) + assert len(lines) == 1 and "unusually wide" in lines[0] + assert "2400" not in lines[0] and "0.90" not in lines[0] \ + and "$" not in lines[0], "channel line must carry no data values" + # format_signal_message plumbs the flag through + config = PlatformConfig() + agg = SignalAggregator(config) + dfs = { + Timeframe.H1: make_ohlcv(300, seed=42), + Timeframe.M15: make_ohlcv(300, seed=43), + Timeframe.H4: make_ohlcv(300, seed=44), + } + report = agg.analyze_from_dataframes("XAU/USD", dfs, use_news=False) + owner_msg = tb.format_signal_message(report, spread_detail=True) + channel_msg = tb.format_signal_message(report, spread_detail=False) + assert "Bid/Ask" in owner_msg + assert "Bid/Ask" not in channel_msg + finally: + tb._last_quotes.pop(XAUUSD, None) + + +@test("telegram_bot: _spread_detail_for routes by chat + internal flag") +def _(): + import telegram_bot as tb + saved_channel, saved_flag = tb.CHANNEL_ID, tb._CHANNEL_SPREAD_DETAIL + try: + tb.CHANNEL_ID = -100222 + tb._CHANNEL_SPREAD_DETAIL = False # default: channel is public + assert tb._spread_detail_for(111) is True, "owner DM gets raw quotes" + assert tb._spread_detail_for(-100222) is False, \ + "public channel must get the number-free variant" + tb._CHANNEL_SPREAD_DETAIL = True # internal-testing channel + assert tb._spread_detail_for(-100222) is True + assert tb._spread_detail_for(111) is True + finally: + tb.CHANNEL_ID, tb._CHANNEL_SPREAD_DETAIL = saved_channel, saved_flag + + +@test("telegram_bot: log rotation is 3-hourly with 30-day retention") +def _(): + import logging.handlers + import telegram_bot as tb + h = tb._file_handler + assert isinstance(h, logging.handlers.TimedRotatingFileHandler) + assert h.when.upper() == "H" and h.interval == 3 * 3600, \ + "rotation must fire every 3 hours" + assert h.backupCount == 240, "240 × 3h = 30-day retention" + + +@test("telegram_bot: candle loop survives a failing scan cycle") +def _(): + import asyncio + import telegram_bot as tb + calls = {"n": 0} + + async def boom(bot, tf, confirm): + calls["n"] += 1 + if calls["n"] == 1: + raise RuntimeError("scan blew up") + tb._autoscan_chats.clear() # let the loop exit after call 2 + + tb._autoscan_chats.add(1) + try: + with patch.object(tb, "_scan_timeframe", boom), \ + patch.object(tb, "_seconds_until_boundary", + lambda interval, buffer=10: 0.0): + asyncio.run(tb._candle_loop(None, Timeframe.M15, 15)) + assert calls["n"] == 2, \ + "loop must keep firing after an exception, not die on it" + finally: + tb._autoscan_chats.clear() + + +@test("telegram_bot: no Tiingo key → provider off, message unchanged") +def _(): + import telegram_bot as tb + assert tb._tiingo is None, "hermetic env has no Tiingo key" + config = PlatformConfig() + agg = SignalAggregator(config) + dfs = { + Timeframe.H1: make_ohlcv(300, seed=42), + Timeframe.M15: make_ohlcv(300, seed=43), + Timeframe.H4: make_ohlcv(300, seed=44), + } + report = agg.analyze_from_dataframes("XAU/USD", dfs, use_news=False) + msg = tb.format_signal_message(report) + assert "Bid/Ask" not in msg and "Wide spread" not in msg + + # ══════════════════════════════════════════════════════════════════════ # SUMMARY # ══════════════════════════════════════════════════════════════════════ diff --git a/tiingo_provider.py b/tiingo_provider.py new file mode 100644 index 0000000..0c65ba0 --- /dev/null +++ b/tiingo_provider.py @@ -0,0 +1,157 @@ +""" +tiingo_provider.py — Tiingo top-of-book bid/ask quotes +====================================================== + +Supplementary LIVE-QUOTE provider, NOT an OHLCV source: Twelve Data +remains the signal/backtest data provider. Tiingo's forex endpoint +(GET /tiingo/fx/top) returns real bid/ask, which mid-price OHLCV +cannot provide. Two uses: + +1. Spread visibility on alerts — signal messages show the live + bid/ask and warn when the spread is ≥ WIDE_SPREAD_MULTIPLE × the + assumed round-trip cost (PAIR_INFO ``default_spread``, $0.30 on + XAU) that every backtest charges. + + Warn-only BY DESIGN: no historical bid/ask exists to replay, so a + suppressing gate could never be validated (live == backtest rule). + This sits in the same safety category as the staleness check. + +2. Spread logging (logs/spread_log.csv, 5-min sampler in the bot) — + builds the dataset to validate or refute the $0.30 round-trip + assumption against reality. + +Signals NEVER consume bid/ask — indicators stay on mid OHLCV. Every +failure path returns None (alerts go out without the spread block +rather than being delayed or dropped). + +Key resolution: env TIINGO_API_KEY (or TIINGO, e.g. from .env) → +config.json data_providers.tiingo.api_key → "" (feature disabled). +See app_config.tiingo_api_key. +""" + +import logging +import os +import threading +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Dict, Optional + +import requests + +logger = logging.getLogger(__name__) + +_TIINGO_FX_TOP_URL = "https://api.tiingo.com/tiingo/fx/top" + +# Spread ≥ this multiple of the pair's assumed round-trip cost is +# flagged as "wide" on alerts (2× the $0.30 XAU assumption = $0.60). +WIDE_SPREAD_MULTIPLE = 2.0 + +# A quote younger than this is served from cache — dedupes bursts +# (alert refresh right after a sampler tick) without hiding real moves. +QUOTE_TTL_SECONDS = 10.0 + +SPREAD_LOG_PATH = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "logs", "spread_log.csv") +_CSV_HEADER = "ts_utc,symbol,bid,ask,mid,spread\n" + + +def tiingo_ticker(symbol: str) -> str: + """Canonical '"XAU/USD"' form → Tiingo's 'xauusd'.""" + return symbol.replace("/", "").lower() + + +@dataclass +class Quote: + """One top-of-book snapshot. ``spread`` (ask − bid) IS the + round-trip cost a market order pays.""" + symbol: str # canonical form, e.g. "XAU/USD" + bid: float + ask: float + mid: float + spread: float + quote_time: Optional[datetime] # venue timestamp (UTC) if provided + fetched_at: float # time.time() at fetch + + def age_seconds(self) -> float: + return time.time() - self.fetched_at + + +class TiingoQuoteProvider: + """Minimal /tiingo/fx/top client with a short TTL cache. + + Lock held across the fetch (news_engine convention) so concurrent + scan threads share one request instead of racing the API. + """ + + def __init__(self, api_key: str, quote_ttl: float = QUOTE_TTL_SECONDS): + self.api_key = api_key + self.quote_ttl = quote_ttl + self._cache: Dict[str, Quote] = {} + self._lock = threading.Lock() + + def get_quote(self, symbol: str) -> Optional[Quote]: + with self._lock: + cached = self._cache.get(symbol) + if cached is not None and cached.age_seconds() < self.quote_ttl: + return cached + quote = self._fetch(symbol) + if quote is not None: + self._cache[symbol] = quote + return quote + + def _fetch(self, symbol: str) -> Optional[Quote]: + try: + resp = requests.get( + _TIINGO_FX_TOP_URL, + params={"tickers": tiingo_ticker(symbol), + "token": self.api_key}, + timeout=10, + ) + resp.raise_for_status() + payload = resp.json() + if not isinstance(payload, list) or not payload: + logger.warning( + f"Tiingo: unexpected payload for {symbol}: {payload!r:.200}") + return None + row = payload[0] + bid, ask = row.get("bidPrice"), row.get("askPrice") + if not isinstance(bid, (int, float)) or not isinstance(ask, (int, float)): + logger.warning(f"Tiingo: no bid/ask for {symbol}: {row!r:.200}") + return None + mid = row.get("midPrice") + if not isinstance(mid, (int, float)): + mid = (bid + ask) / 2 + quote_time: Optional[datetime] = None + raw_ts = row.get("quoteTimestamp") + if isinstance(raw_ts, str): + try: + quote_time = datetime.fromisoformat( + raw_ts.replace("Z", "+00:00")) + except ValueError: + pass + return Quote( + symbol=symbol, bid=float(bid), ask=float(ask), + mid=float(mid), spread=float(ask) - float(bid), + quote_time=quote_time, fetched_at=time.time(), + ) + except Exception as exc: + logger.warning(f"Tiingo quote fetch failed for {symbol}: {exc}") + return None + + +def log_quote(quote: Quote, path: str = SPREAD_LOG_PATH) -> None: + """Append one row to the spread log (creates dir + header on first + write). ~15 KB/day at 5-min sampling — no rotation needed.""" + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + is_new = not os.path.exists(path) + with open(path, "a", encoding="utf-8") as fh: + if is_new: + fh.write(_CSV_HEADER) + ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + fh.write( + f"{ts},{quote.symbol},{quote.bid:.5f},{quote.ask:.5f}," + f"{quote.mid:.5f},{quote.spread:.5f}\n") + except OSError as exc: + logger.warning(f"Spread log write failed: {exc}")