Tiingo bid/ask spread monitor + logging hardening
- tiingo_provider.py: /tiingo/fx/top quotes (10s TTL cache, graceful None on any failure); spread log writer. Signals never consume bid/ask; warn-only by design (no historical bid/ask to validate a gate against — live == backtest rule). - app_config: repo-root .env loader (setdefault; shell env still wins), tiingo_api_key resolver (TIINGO_API_KEY/TIINGO → config.json), telegram.channel_spread_detail flag (default false). - telegram_bot: 5-min spread sampler → logs/spread_log.csv (weekend- aware), alert-time quote refresh, bid/ask + wide-spread warning in signal messages, /status spread line. Raw numbers to owner DM always; channel only while channel_spread_detail=true (Tiingo free tier is internal-use-only; flip before public launch). Free-tier limit math: worst case ~41 req/h vs 50/h cap, one pair. - Logging: 3h rotation x 240 backups (30-day retention), full-date stamps, force=True root handlers; candle loops + sampler survive failed cycles; breach-task exceptions logged via done-callback; PTB error handler; sys/threading excepthooks in main(). - test_all.py: Module 14 (tiingo, dotenv, spread routing, loop resilience, rotation) — 215 tests green. Claude-Session: https://claude.ai/code/session_01HVRmy6KspMs96o5PTPYFAB
This commit is contained in:
Harwinder Sekhon
@@ -0,0 +1,157 @@
|
||||
"""
|
||||
tiingo_provider.py — Tiingo top-of-book bid/ask quotes
|
||||
======================================================
|
||||
|
||||
Supplementary LIVE-QUOTE provider, NOT an OHLCV source: Twelve Data
|
||||
remains the signal/backtest data provider. Tiingo's forex endpoint
|
||||
(GET /tiingo/fx/top) returns real bid/ask, which mid-price OHLCV
|
||||
cannot provide. Two uses:
|
||||
|
||||
1. Spread visibility on alerts — signal messages show the live
|
||||
bid/ask and warn when the spread is ≥ WIDE_SPREAD_MULTIPLE × the
|
||||
assumed round-trip cost (PAIR_INFO ``default_spread``, $0.30 on
|
||||
XAU) that every backtest charges.
|
||||
|
||||
Warn-only BY DESIGN: no historical bid/ask exists to replay, so a
|
||||
suppressing gate could never be validated (live == backtest rule).
|
||||
This sits in the same safety category as the staleness check.
|
||||
|
||||
2. Spread logging (logs/spread_log.csv, 5-min sampler in the bot) —
|
||||
builds the dataset to validate or refute the $0.30 round-trip
|
||||
assumption against reality.
|
||||
|
||||
Signals NEVER consume bid/ask — indicators stay on mid OHLCV. Every
|
||||
failure path returns None (alerts go out without the spread block
|
||||
rather than being delayed or dropped).
|
||||
|
||||
Key resolution: env TIINGO_API_KEY (or TIINGO, e.g. from .env) →
|
||||
config.json data_providers.tiingo.api_key → "" (feature disabled).
|
||||
See app_config.tiingo_api_key.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, Optional
|
||||
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_TIINGO_FX_TOP_URL = "https://api.tiingo.com/tiingo/fx/top"
|
||||
|
||||
# Spread ≥ this multiple of the pair's assumed round-trip cost is
|
||||
# flagged as "wide" on alerts (2× the $0.30 XAU assumption = $0.60).
|
||||
WIDE_SPREAD_MULTIPLE = 2.0
|
||||
|
||||
# A quote younger than this is served from cache — dedupes bursts
|
||||
# (alert refresh right after a sampler tick) without hiding real moves.
|
||||
QUOTE_TTL_SECONDS = 10.0
|
||||
|
||||
SPREAD_LOG_PATH = os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)), "logs", "spread_log.csv")
|
||||
_CSV_HEADER = "ts_utc,symbol,bid,ask,mid,spread\n"
|
||||
|
||||
|
||||
def tiingo_ticker(symbol: str) -> str:
|
||||
"""Canonical '"XAU/USD"' form → Tiingo's 'xauusd'."""
|
||||
return symbol.replace("/", "").lower()
|
||||
|
||||
|
||||
@dataclass
|
||||
class Quote:
|
||||
"""One top-of-book snapshot. ``spread`` (ask − bid) IS the
|
||||
round-trip cost a market order pays."""
|
||||
symbol: str # canonical form, e.g. "XAU/USD"
|
||||
bid: float
|
||||
ask: float
|
||||
mid: float
|
||||
spread: float
|
||||
quote_time: Optional[datetime] # venue timestamp (UTC) if provided
|
||||
fetched_at: float # time.time() at fetch
|
||||
|
||||
def age_seconds(self) -> float:
|
||||
return time.time() - self.fetched_at
|
||||
|
||||
|
||||
class TiingoQuoteProvider:
|
||||
"""Minimal /tiingo/fx/top client with a short TTL cache.
|
||||
|
||||
Lock held across the fetch (news_engine convention) so concurrent
|
||||
scan threads share one request instead of racing the API.
|
||||
"""
|
||||
|
||||
def __init__(self, api_key: str, quote_ttl: float = QUOTE_TTL_SECONDS):
|
||||
self.api_key = api_key
|
||||
self.quote_ttl = quote_ttl
|
||||
self._cache: Dict[str, Quote] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def get_quote(self, symbol: str) -> Optional[Quote]:
|
||||
with self._lock:
|
||||
cached = self._cache.get(symbol)
|
||||
if cached is not None and cached.age_seconds() < self.quote_ttl:
|
||||
return cached
|
||||
quote = self._fetch(symbol)
|
||||
if quote is not None:
|
||||
self._cache[symbol] = quote
|
||||
return quote
|
||||
|
||||
def _fetch(self, symbol: str) -> Optional[Quote]:
|
||||
try:
|
||||
resp = requests.get(
|
||||
_TIINGO_FX_TOP_URL,
|
||||
params={"tickers": tiingo_ticker(symbol),
|
||||
"token": self.api_key},
|
||||
timeout=10,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
payload = resp.json()
|
||||
if not isinstance(payload, list) or not payload:
|
||||
logger.warning(
|
||||
f"Tiingo: unexpected payload for {symbol}: {payload!r:.200}")
|
||||
return None
|
||||
row = payload[0]
|
||||
bid, ask = row.get("bidPrice"), row.get("askPrice")
|
||||
if not isinstance(bid, (int, float)) or not isinstance(ask, (int, float)):
|
||||
logger.warning(f"Tiingo: no bid/ask for {symbol}: {row!r:.200}")
|
||||
return None
|
||||
mid = row.get("midPrice")
|
||||
if not isinstance(mid, (int, float)):
|
||||
mid = (bid + ask) / 2
|
||||
quote_time: Optional[datetime] = None
|
||||
raw_ts = row.get("quoteTimestamp")
|
||||
if isinstance(raw_ts, str):
|
||||
try:
|
||||
quote_time = datetime.fromisoformat(
|
||||
raw_ts.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
pass
|
||||
return Quote(
|
||||
symbol=symbol, bid=float(bid), ask=float(ask),
|
||||
mid=float(mid), spread=float(ask) - float(bid),
|
||||
quote_time=quote_time, fetched_at=time.time(),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(f"Tiingo quote fetch failed for {symbol}: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def log_quote(quote: Quote, path: str = SPREAD_LOG_PATH) -> None:
|
||||
"""Append one row to the spread log (creates dir + header on first
|
||||
write). ~15 KB/day at 5-min sampling — no rotation needed."""
|
||||
try:
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
is_new = not os.path.exists(path)
|
||||
with open(path, "a", encoding="utf-8") as fh:
|
||||
if is_new:
|
||||
fh.write(_CSV_HEADER)
|
||||
ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
fh.write(
|
||||
f"{ts},{quote.symbol},{quote.bid:.5f},{quote.ask:.5f},"
|
||||
f"{quote.mid:.5f},{quote.spread:.5f}\n")
|
||||
except OSError as exc:
|
||||
logger.warning(f"Spread log write failed: {exc}")
|
||||
Reference in New Issue
Block a user