Tiingo bid/ask spread monitor + logging hardening
- tiingo_provider.py: /tiingo/fx/top quotes (10s TTL cache, graceful None on any failure); spread log writer. Signals never consume bid/ask; warn-only by design (no historical bid/ask to validate a gate against — live == backtest rule). - app_config: repo-root .env loader (setdefault; shell env still wins), tiingo_api_key resolver (TIINGO_API_KEY/TIINGO → config.json), telegram.channel_spread_detail flag (default false). - telegram_bot: 5-min spread sampler → logs/spread_log.csv (weekend- aware), alert-time quote refresh, bid/ask + wide-spread warning in signal messages, /status spread line. Raw numbers to owner DM always; channel only while channel_spread_detail=true (Tiingo free tier is internal-use-only; flip before public launch). Free-tier limit math: worst case ~41 req/h vs 50/h cap, one pair. - Logging: 3h rotation x 240 backups (30-day retention), full-date stamps, force=True root handlers; candle loops + sampler survive failed cycles; breach-task exceptions logged via done-callback; PTB error handler; sys/threading excepthooks in main(). - test_all.py: Module 14 (tiingo, dotenv, spread routing, loop resilience, rotation) — 215 tests green. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HVRmy6KspMs96o5PTPYFAB
This commit is contained in:
Harwinder Sekhon
+290
-2
@@ -25,7 +25,7 @@ sys.path.insert(0, os.path.dirname(__file__))
|
||||
import json
|
||||
import app_config
|
||||
for _k in ("TELEGRAM_TOKEN", "TELEGRAM_CHAT_ID", "TELEGRAM_CHANNEL_ID",
|
||||
"TWELVE_DATA_API_KEY"):
|
||||
"TWELVE_DATA_API_KEY", "TIINGO_API_KEY", "TIINGO"):
|
||||
os.environ.pop(_k, None)
|
||||
_TEST_APP = app_config.AppConfig(
|
||||
provider_settings={"twelve_data": {"api_key": "test-key"}}
|
||||
@@ -2607,7 +2607,7 @@ def _write_json(payload) -> str:
|
||||
|
||||
_FULL_CFG = {
|
||||
"telegram": {"token": "t0k", "chat_id": "111", "channel_id": "-100222",
|
||||
"alert_min_quality": "B"},
|
||||
"alert_min_quality": "B", "channel_spread_detail": True},
|
||||
"data_providers": {"active": "twelve_data",
|
||||
"twelve_data": {"api_key": "json-key"}},
|
||||
"scanning": {"pairs": ["XAU/USD"], "timeframes": ["M15", "H1"]},
|
||||
@@ -2623,6 +2623,7 @@ def _():
|
||||
assert app.telegram_chat_id == "111"
|
||||
assert app.telegram_channel_id == "-100222"
|
||||
assert app.alert_min_quality == "B"
|
||||
assert app.channel_spread_detail is True
|
||||
assert app.active_provider == "twelve_data"
|
||||
assert app.provider_settings["twelve_data"]["api_key"] == "json-key"
|
||||
assert app.scan_pairs == [XAUUSD]
|
||||
@@ -2676,6 +2677,7 @@ def _():
|
||||
{"data_providers": {"active": "polygon",
|
||||
"twelve_data": {"api_key": "k"}}}, # no block
|
||||
{"telegram": {"alert_min_quality": "Z"}}, # bad quality
|
||||
{"telegram": {"channel_spread_detail": "yes"}}, # not a bool
|
||||
{"log_level": "CHATTY"}, # bad level
|
||||
]
|
||||
for payload in bad_payloads:
|
||||
@@ -2857,6 +2859,292 @@ def _():
|
||||
Timeframe.H1, Timeframe.H4]
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# MODULE 14: tiingo_provider.py (bid/ask spread monitor)
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
print("\n═══ Module 14: tiingo_provider.py ═══")
|
||||
|
||||
from tiingo_provider import (
|
||||
Quote, TiingoQuoteProvider, WIDE_SPREAD_MULTIPLE, log_quote,
|
||||
tiingo_ticker,
|
||||
)
|
||||
from app_config import load_dotenv, tiingo_api_key
|
||||
|
||||
|
||||
def _tiingo_response(payload):
|
||||
resp = MagicMock()
|
||||
resp.json.return_value = payload
|
||||
resp.raise_for_status.return_value = None
|
||||
return resp
|
||||
|
||||
|
||||
_TIINGO_ROW = [{
|
||||
"ticker": "xauusd",
|
||||
"quoteTimestamp": "2026-07-22T12:00:00+00:00",
|
||||
"bidPrice": 2400.10, "bidSize": 1000000.0,
|
||||
"askPrice": 2400.40, "askSize": 1000000.0,
|
||||
"midPrice": 2400.25,
|
||||
}]
|
||||
|
||||
|
||||
@test("tiingo: ticker mapping XAU/USD → xauusd")
|
||||
def _():
|
||||
assert tiingo_ticker("XAU/USD") == "xauusd"
|
||||
assert tiingo_ticker("EUR/USD") == "eurusd"
|
||||
|
||||
|
||||
@test("tiingo: quote parsed — spread = ask − bid, canonical symbol")
|
||||
def _():
|
||||
provider = TiingoQuoteProvider("k")
|
||||
with patch("tiingo_provider.requests.get",
|
||||
return_value=_tiingo_response(_TIINGO_ROW)) as mock_get:
|
||||
q = provider.get_quote("XAU/USD")
|
||||
assert q is not None
|
||||
assert q.symbol == "XAU/USD"
|
||||
assert abs(q.bid - 2400.10) < 1e-9 and abs(q.ask - 2400.40) < 1e-9
|
||||
assert abs(q.spread - 0.30) < 1e-9
|
||||
assert abs(q.mid - 2400.25) < 1e-9
|
||||
assert q.quote_time is not None and q.quote_time.tzinfo is not None
|
||||
assert q.age_seconds() < 5
|
||||
params = mock_get.call_args.kwargs["params"]
|
||||
assert params["tickers"] == "xauusd" and params["token"] == "k"
|
||||
|
||||
|
||||
@test("tiingo: mid falls back to (bid+ask)/2 when midPrice missing")
|
||||
def _():
|
||||
row = [dict(_TIINGO_ROW[0], midPrice=None)]
|
||||
provider = TiingoQuoteProvider("k")
|
||||
with patch("tiingo_provider.requests.get",
|
||||
return_value=_tiingo_response(row)):
|
||||
q = provider.get_quote("XAU/USD")
|
||||
assert q is not None and abs(q.mid - 2400.25) < 1e-9
|
||||
|
||||
|
||||
@test("tiingo: TTL cache dedupes calls; ttl=0 refetches")
|
||||
def _():
|
||||
provider = TiingoQuoteProvider("k")
|
||||
with patch("tiingo_provider.requests.get",
|
||||
return_value=_tiingo_response(_TIINGO_ROW)) as mock_get:
|
||||
provider.get_quote("XAU/USD")
|
||||
provider.get_quote("XAU/USD")
|
||||
assert mock_get.call_count == 1, "second call within TTL must hit cache"
|
||||
provider = TiingoQuoteProvider("k", quote_ttl=0.0)
|
||||
with patch("tiingo_provider.requests.get",
|
||||
return_value=_tiingo_response(_TIINGO_ROW)) as mock_get:
|
||||
provider.get_quote("XAU/USD")
|
||||
provider.get_quote("XAU/USD")
|
||||
assert mock_get.call_count == 2
|
||||
|
||||
|
||||
@test("tiingo: every failure path returns None (never raises)")
|
||||
def _():
|
||||
provider = TiingoQuoteProvider("k", quote_ttl=0.0)
|
||||
# network error
|
||||
with patch("tiingo_provider.requests.get",
|
||||
side_effect=Exception("boom")):
|
||||
assert provider.get_quote("XAU/USD") is None
|
||||
# HTTP error
|
||||
resp = _tiingo_response(_TIINGO_ROW)
|
||||
resp.raise_for_status.side_effect = Exception("401")
|
||||
with patch("tiingo_provider.requests.get", return_value=resp):
|
||||
assert provider.get_quote("XAU/USD") is None
|
||||
# empty / non-list / null-price payloads
|
||||
for payload in ([], {"detail": "unknown ticker"},
|
||||
[dict(_TIINGO_ROW[0], bidPrice=None)],
|
||||
[dict(_TIINGO_ROW[0], askPrice=None)]):
|
||||
with patch("tiingo_provider.requests.get",
|
||||
return_value=_tiingo_response(payload)):
|
||||
assert provider.get_quote("XAU/USD") is None, \
|
||||
f"payload {payload!r} must yield None"
|
||||
|
||||
|
||||
@test("tiingo: log_quote writes header once then appends rows")
|
||||
def _():
|
||||
q = Quote(symbol=XAUUSD, bid=2400.10, ask=2400.40, mid=2400.25,
|
||||
spread=0.30, quote_time=None, fetched_at=time.time())
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
path = os.path.join(td, "sub", "spread_log.csv")
|
||||
log_quote(q, path)
|
||||
log_quote(q, path)
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
lines = fh.read().strip().splitlines()
|
||||
assert lines[0] == "ts_utc,symbol,bid,ask,mid,spread"
|
||||
assert len(lines) == 3, "header + one row per log_quote call"
|
||||
cells = lines[1].split(",")
|
||||
assert cells[1] == XAUUSD and float(cells[5]) == 0.30
|
||||
|
||||
|
||||
@test("app_config: tiingo_api_key precedence env → env-short → json → ''")
|
||||
def _():
|
||||
app = AppConfig(provider_settings={
|
||||
"twelve_data": {"api_key": "td"},
|
||||
"tiingo": {"api_key": "json-tiingo"},
|
||||
})
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
assert tiingo_api_key(app) == "json-tiingo"
|
||||
with patch.dict(os.environ, {"TIINGO": "short-env"}, clear=True):
|
||||
assert tiingo_api_key(app) == "short-env"
|
||||
with patch.dict(os.environ, {"TIINGO": "short-env",
|
||||
"TIINGO_API_KEY": "long-env"}, clear=True):
|
||||
assert tiingo_api_key(app) == "long-env"
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
assert tiingo_api_key(AppConfig()) == "", "no key anywhere = disabled"
|
||||
|
||||
|
||||
@test("app_config: load_dotenv parses lines, never overrides real env")
|
||||
def _():
|
||||
fh = tempfile.NamedTemporaryFile(
|
||||
"w", suffix=".env", delete=False, encoding="utf-8")
|
||||
fh.write("# comment\n\nPLAIN=abc\nexport EXPORTED=def\n"
|
||||
'QUOTED="with spaces"\nSINGLE=\'sq\'\nEXISTING=from-dotenv\n'
|
||||
"NOT A KV LINE\n")
|
||||
fh.close()
|
||||
with patch.dict(os.environ, {"EXISTING": "from-shell"}, clear=True):
|
||||
load_dotenv(fh.name)
|
||||
assert os.environ["PLAIN"] == "abc"
|
||||
assert os.environ["EXPORTED"] == "def"
|
||||
assert os.environ["QUOTED"] == "with spaces"
|
||||
assert os.environ["SINGLE"] == "sq"
|
||||
assert os.environ["EXISTING"] == "from-shell", \
|
||||
"a real shell export must beat .env"
|
||||
load_dotenv("/nonexistent/.env") # missing file is a silent no-op
|
||||
|
||||
|
||||
@test("telegram_bot: _spread_lines fresh/wide/stale/absent")
|
||||
def _():
|
||||
import telegram_bot as tb
|
||||
try:
|
||||
# fresh, normal spread → bid/ask line only
|
||||
tb._last_quotes[XAUUSD] = Quote(
|
||||
symbol=XAUUSD, bid=2400.10, ask=2400.40, mid=2400.25,
|
||||
spread=0.30, quote_time=None, fetched_at=time.time())
|
||||
lines = tb._spread_lines(XAUUSD)
|
||||
assert len(lines) == 1 and "Bid/Ask" in lines[0]
|
||||
assert "spread $0.30" in lines[0]
|
||||
# wide spread (≥ 2× the $0.30 assumption) → warning appended
|
||||
tb._last_quotes[XAUUSD] = Quote(
|
||||
symbol=XAUUSD, bid=2400.10, ask=2401.00, mid=2400.55,
|
||||
spread=0.90, quote_time=None, fetched_at=time.time())
|
||||
lines = tb._spread_lines(XAUUSD)
|
||||
assert len(lines) == 2 and "Wide spread" in lines[1]
|
||||
assert "3.0×" in lines[1]
|
||||
# stale quote → nothing
|
||||
tb._last_quotes[XAUUSD] = Quote(
|
||||
symbol=XAUUSD, bid=2400.10, ask=2400.40, mid=2400.25,
|
||||
spread=0.30, quote_time=None,
|
||||
fetched_at=time.time() - tb._QUOTE_MAX_AGE - 1)
|
||||
assert tb._spread_lines(XAUUSD) == []
|
||||
# no quote at all → nothing
|
||||
tb._last_quotes.pop(XAUUSD, None)
|
||||
assert tb._spread_lines(XAUUSD) == []
|
||||
finally:
|
||||
tb._last_quotes.pop(XAUUSD, None)
|
||||
|
||||
|
||||
@test("telegram_bot: channel variant never leaks Tiingo numbers")
|
||||
def _():
|
||||
# Tiingo free tier is internal-use-only: the subscriber channel may
|
||||
# see a number-free caution, never bid/ask/spread values.
|
||||
import telegram_bot as tb
|
||||
try:
|
||||
# normal spread → channel variant shows nothing at all
|
||||
tb._last_quotes[XAUUSD] = Quote(
|
||||
symbol=XAUUSD, bid=2400.10, ask=2400.40, mid=2400.25,
|
||||
spread=0.30, quote_time=None, fetched_at=time.time())
|
||||
assert tb._spread_lines(XAUUSD, include_quote=False) == []
|
||||
# wide spread → one caution line with no prices in it
|
||||
tb._last_quotes[XAUUSD] = Quote(
|
||||
symbol=XAUUSD, bid=2400.10, ask=2401.00, mid=2400.55,
|
||||
spread=0.90, quote_time=None, fetched_at=time.time())
|
||||
lines = tb._spread_lines(XAUUSD, include_quote=False)
|
||||
assert len(lines) == 1 and "unusually wide" in lines[0]
|
||||
assert "2400" not in lines[0] and "0.90" not in lines[0] \
|
||||
and "$" not in lines[0], "channel line must carry no data values"
|
||||
# format_signal_message plumbs the flag through
|
||||
config = PlatformConfig()
|
||||
agg = SignalAggregator(config)
|
||||
dfs = {
|
||||
Timeframe.H1: make_ohlcv(300, seed=42),
|
||||
Timeframe.M15: make_ohlcv(300, seed=43),
|
||||
Timeframe.H4: make_ohlcv(300, seed=44),
|
||||
}
|
||||
report = agg.analyze_from_dataframes("XAU/USD", dfs, use_news=False)
|
||||
owner_msg = tb.format_signal_message(report, spread_detail=True)
|
||||
channel_msg = tb.format_signal_message(report, spread_detail=False)
|
||||
assert "Bid/Ask" in owner_msg
|
||||
assert "Bid/Ask" not in channel_msg
|
||||
finally:
|
||||
tb._last_quotes.pop(XAUUSD, None)
|
||||
|
||||
|
||||
@test("telegram_bot: _spread_detail_for routes by chat + internal flag")
|
||||
def _():
|
||||
import telegram_bot as tb
|
||||
saved_channel, saved_flag = tb.CHANNEL_ID, tb._CHANNEL_SPREAD_DETAIL
|
||||
try:
|
||||
tb.CHANNEL_ID = -100222
|
||||
tb._CHANNEL_SPREAD_DETAIL = False # default: channel is public
|
||||
assert tb._spread_detail_for(111) is True, "owner DM gets raw quotes"
|
||||
assert tb._spread_detail_for(-100222) is False, \
|
||||
"public channel must get the number-free variant"
|
||||
tb._CHANNEL_SPREAD_DETAIL = True # internal-testing channel
|
||||
assert tb._spread_detail_for(-100222) is True
|
||||
assert tb._spread_detail_for(111) is True
|
||||
finally:
|
||||
tb.CHANNEL_ID, tb._CHANNEL_SPREAD_DETAIL = saved_channel, saved_flag
|
||||
|
||||
|
||||
@test("telegram_bot: log rotation is 3-hourly with 30-day retention")
|
||||
def _():
|
||||
import logging.handlers
|
||||
import telegram_bot as tb
|
||||
h = tb._file_handler
|
||||
assert isinstance(h, logging.handlers.TimedRotatingFileHandler)
|
||||
assert h.when.upper() == "H" and h.interval == 3 * 3600, \
|
||||
"rotation must fire every 3 hours"
|
||||
assert h.backupCount == 240, "240 × 3h = 30-day retention"
|
||||
|
||||
|
||||
@test("telegram_bot: candle loop survives a failing scan cycle")
|
||||
def _():
|
||||
import asyncio
|
||||
import telegram_bot as tb
|
||||
calls = {"n": 0}
|
||||
|
||||
async def boom(bot, tf, confirm):
|
||||
calls["n"] += 1
|
||||
if calls["n"] == 1:
|
||||
raise RuntimeError("scan blew up")
|
||||
tb._autoscan_chats.clear() # let the loop exit after call 2
|
||||
|
||||
tb._autoscan_chats.add(1)
|
||||
try:
|
||||
with patch.object(tb, "_scan_timeframe", boom), \
|
||||
patch.object(tb, "_seconds_until_boundary",
|
||||
lambda interval, buffer=10: 0.0):
|
||||
asyncio.run(tb._candle_loop(None, Timeframe.M15, 15))
|
||||
assert calls["n"] == 2, \
|
||||
"loop must keep firing after an exception, not die on it"
|
||||
finally:
|
||||
tb._autoscan_chats.clear()
|
||||
|
||||
|
||||
@test("telegram_bot: no Tiingo key → provider off, message unchanged")
|
||||
def _():
|
||||
import telegram_bot as tb
|
||||
assert tb._tiingo is None, "hermetic env has no Tiingo key"
|
||||
config = PlatformConfig()
|
||||
agg = SignalAggregator(config)
|
||||
dfs = {
|
||||
Timeframe.H1: make_ohlcv(300, seed=42),
|
||||
Timeframe.M15: make_ohlcv(300, seed=43),
|
||||
Timeframe.H4: make_ohlcv(300, seed=44),
|
||||
}
|
||||
report = agg.analyze_from_dataframes("XAU/USD", dfs, use_news=False)
|
||||
msg = tb.format_signal_message(report)
|
||||
assert "Bid/Ask" not in msg and "Wide spread" not in msg
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# SUMMARY
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
Reference in New Issue
Block a user