From 11ff49befba13b43ad48afc959605dc5b2443dd1 Mon Sep 17 00:00:00 2001 From: Harwinder Sekhon Date: Tue, 21 Jul 2026 14:06:55 -0400 Subject: [PATCH] Remove dead code: legacy config fields, unused helpers/imports, stray files - Delete test_ws_dual.py (standalone probe, unused) and a stale tracked log - config.py: drop MarketSession enum, is_jpy_pair, candle_buffer_seconds, and the never-read legacy SignalConfig/XauStrategyConfig fields - data_provider.py: drop uncalled get_latest_price/get_spread_estimate - Remove unused imports (engine, telegram_bot, main, news_engine) - test_all.py: drop/trim the corresponding defaults assertions (199 tests) - CLAUDE.md updated to match Zero behavior change: no production code read any removed field (verified reference-by-reference); suite green at 199/199. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01VPNF37MP9Kuv62B1VBrZ1b --- CLAUDE.md | 11 +--- config.py | 41 +------------ data_provider.py | 19 ------ engine.py | 2 +- logs/telegram.log.2026-07-14 | 21 ------- main.py | 1 - news_engine.py | 2 +- telegram_bot.py | 1 - test_all.py | 23 +------- test_ws_dual.py | 110 ----------------------------------- 10 files changed, 9 insertions(+), 222 deletions(-) delete mode 100644 logs/telegram.log.2026-07-14 delete mode 100644 test_ws_dual.py diff --git a/CLAUDE.md b/CLAUDE.md index 9ae17a0..de53abb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,13 +24,12 @@ This file is the authoritative developer/agent reference. The human-facing docs | `pair_signals/__init__.py` | 52 | `PAIR_PRODUCERS` registry + `produce_signal()` | | `pair_signals/xauusd.py` | 71 | XAU config factory + per-timeframe overrides (docstring = validation record) | | `main.py` | ~348 | CLI: `python main.py --pair XAUUSD --timeframe H1 --detail/--json` | -| `test_all.py` | ~2100 | QA suite: **200 tests in 13 modules**, no live API needed. Must stay green. | +| `test_all.py` | ~2800 | QA suite: **199 tests in 13 modules**, no live API needed. Must stay green. | | `backtest/gpu_backtest.py` | ~2231 | Bar-replay backtester (CuPy→NumPy fallback) using the real `SignalAggregator` | | `backtest/diagnose_filters.py` | 609 | Filter-diagnosis tooling | | `backtest/research/` | — | Calendar research infra (`ff_calendar_scraper.py`, `calendar_store.py`) + result CSVs | | `backtest/*.csv` | — | Verification trade books from past tuning campaigns (`xau_*.csv`, …). **Gitignored, local-only** — used for manual before/after comparison, not by tests | | `backtest/cache/` | — | Parquet OHLCV cache (raw bars) | -| `test_ws_dual.py` | 108 | Standalone WebSocket dual-connection probe | | `logs/telegram.log` | — | Bot log (midnight rotation, 30 backups) | --- @@ -83,7 +82,6 @@ EMA 9/21/50/200 (`ema_mode`: `"4ema"` default or `"2ema"` = 50/200 only); RSI 14 | `tp_floor_rr` | 0.5 | TP1 floor as fraction of SL distance (M15 factory overrides to 1.0) | | `trail_arm_r` / `trail_dist_r` | 0.6 / 0.75 | trailing exit: once favorable excursion ≥ arm×risk, stop trails dist×risk behind best price (bar-close updates). TP1 becomes advisory. Validated on 12mo M5: +105R vs +48R static. 0 disables | | `pb_trail_arm_r` / `pb_trail_dist_r` | 0.8 / 0.75 | trailing params for XAU_PB pullback trades (tighter stops → arm later) | -| `weights`, `min_confidence`, `cooldown_bars`, `default_risk_pct`, `default_rr_ratio`, `atr_sl_multiplier` | — | **LEGACY / UNUSED** outside tests. The engine has its own weights; backtest cooldown lives in `BacktestConfig` | ### NewsConfig Tracked currencies `["USD","XAU"]`; pre-event buffer 30 min, post-event 15 min (45 min after a *surprise* print); `event_risk_caps_signals=False` — **XAU default: event risk warns but does NOT cap confidence** (2026-07-11 audit: the caps suppressed trades worth +4.6..+7.2R/yr; XAU trades near USD releases outperform); `sentiment_affects_signals=False` — **headline sentiment is informational only** (warnings still emitted; the +5 alignment bonus and 10-pt `news_conflict` penalty are gated off, so live decisions are pure technical, matching backtests). Context cache TTL 600 s (must stay < pre-event buffer). 4 RSS feeds (investinglive, FXStreet, Google-News gold query, CNBC economy). @@ -104,13 +102,11 @@ Tracked currencies `["USD","XAU"]`; pre-event buffer 30 min, post-event 15 min ( | M5 pullback (XAU_PB) | | `m5_pullback_enabled=True`; min score 15 (lower than gate 1 BY DESIGN — mid-pullback scores weaker); ADX ≥ 20; RSI zones BUY (35,62) / SELL (38,65); retrace depth 0.8–6.0 ATR from the 40-bar swing extreme; band touch within 3 bars. Validated +38.3R/254 trades on 12mo | | Chart patterns (XAU_CP) | `cp_enabled=False` | `cp_*` fields: pattern list, lookback 72, swing window 3, recency 20, double tolerance 0.5 ATR, height 1.5–8 ATR, H&S head ≥0.5 ATR / shoulder tol 1.0 ATR, flag pole ≥3 ATR in ≤12 bars, flag 3–12 bars retracing 10–50%. Textbook defaults, NOT tuned. **Default OFF** — M5 factory enables BUY-only (`inv_hs`, `double_bottom`, `bull_flag`); M15/H1/H4 remain disabled (all failed OOS validation 2026-07-16). Bearish patterns (`hs_top`, `double_top`, `bear_flag`) dropped: SELL CP lost −3.21R on OOS, −4.52R on M15 5yr | -Legacy fields kept for config-shape compatibility: `rsi_entry_threshold`, `min_ema200_tfs`, `allow_generic_trades`, `buy_threshold_boost`, `m5_confirmation_bars`, `h4_rsi_gate=False`, `h1_macd_gate=False`, `rsi_zone` — not wired to current gates. - ### FilterConfig (`config.backtest` — despite the name it applies LIVE too) `session_filter=True`, window 03:00–23:00 UTC, `blocked_hours_utc` mirrors XAU's list (non-XAU pairs read this copy), `allowed_weekdays` Mon–Fri+Sun. `regime_filter=False` (optional H4-ADX≥25 gate, off). ### PlatformConfig (top level) -`symbol="XAU/USD"`, primary TF H1, confirmations [M15, H4], `data_lookback_days=60`, `scan_pairs=[XAUUSD]` (mirrors config.json `scanning.pairs` — see below), `scan_timeframes=[M5,M15,H1,H4]`, `candle_buffer_seconds=10`, `ws_cooldown_seconds=300`. +`symbol="XAU/USD"`, primary TF H1, confirmations [M15, H4], `data_lookback_days=60`, `scan_pairs=[XAUUSD]` (mirrors config.json `scanning.pairs` — see below), `scan_timeframes=[M5,M15,H1,H4]`, `ws_cooldown_seconds=300`. Per-pair base blocked hours plus `scan_pairs`/`scan_timeframes` overlay from config.json via `app_config.apply_pair_overrides`, applied in the pair factory BEFORE per-TF deltas. Credentials, alert quality, and log level never touch `PlatformConfig` — `telegram_bot.py` and `data_provider.py` read them directly from `app_config.get_app_config()` and its resolvers. `scanning.pairs` in config.json is the deployment switch for what the bot scans — `PlatformConfig.scan_pairs` just carries the resolved value. @@ -243,7 +239,6 @@ Two entry points: - **Freshness** (`_is_data_fresh`): last bar older than 3× candle duration → stale (warns; live path suppresses signal if the PRIMARY TF is stale). Weekends exempt (Fri 22:00→Sun 22:00 UTC). - **`_validate_and_clean`**: drop all-NaN rows → ffill (limit 3) → drop NaN closes → sort + dedupe (keep last) → drop future timestamps (>now+5 min) → repair OHLC inconsistencies (High=max, Low=min of O/C/existing) → remove >10σ return outliers (spike bar only — the opposite-direction snap-back bar right after it is kept, else every spike punched a two-bar hole) → warn if < ~210 bars. - `outputsize = max(300, min(days×bars_per_day+20, 5000))`; TD caps at 5000/call (the backtester paginates by date). -- `get_latest_price` / `get_spread_estimate` (median M5 High−Low) helpers. --- @@ -362,4 +357,4 @@ Dependencies (`requirements.txt`): numpy, pandas, requests, twelvedata, python-t 7. **Don't zero/disable indicators via config in production paths** — `disabled_indicators` exists for experiments; composite thresholds assume the full set. 8. **The forming candle must stay dropped** and analysis must only ever see closed bars, or live signals repaint. 9. Keep `test_all.py` green (`python test_all.py`); extend it for every new gate/knob. -10. Legacy/no-op config fields exist (flagged above) — do not build new logic on them; delete only with test updates. +10. Dead/legacy config fields were removed 2026-07-21 (`MarketSession`, `is_jpy_pair`, `candle_buffer_seconds`, the unused `SignalConfig`/`XauStrategyConfig` legacy knobs). Do not add config fields that nothing reads; delete dead fields only with test updates. diff --git a/config.py b/config.py index 336ee95..2d810be 100755 --- a/config.py +++ b/config.py @@ -27,14 +27,6 @@ class SignalStrength(Enum): STRONG_SELL = -2 -class MarketSession(Enum): - SYDNEY = "sydney" - TOKYO = "tokyo" - LONDON = "london" - NEW_YORK = "new_york" - OVERLAP_LONDON_NY = "london_ny_overlap" - - # ── Supported pairs ────────────────────────────────────────────── XAUUSD = "XAU/USD" @@ -42,7 +34,7 @@ XAUUSD = "XAU/USD" # default_spread: round-trip cost per trade in price units, charged by # the backtester when --spread is not given. PAIR_INFO = { - XAUUSD: {"display": "XAU/USD", "pip_value": 0.10, "is_jpy_pair": False, + XAUUSD: {"display": "XAU/USD", "pip_value": 0.10, "price_precision": 2, "default_spread": 0.30}, } @@ -102,25 +94,8 @@ class TechnicalConfig: @dataclass class SignalConfig: """Signal scoring & filtering.""" - min_confidence: float = 55.0 - - weights: Dict[str, float] = field(default_factory=lambda: { - "ema_trend": 1.0, - "rsi": 1.5, - "bollinger": 1.5, - "macd": 1.5, - "keltner": 1.5, - "roc": 1.5, - "candlestick": 1.0, - "divergence": 1.5, - }) - mtf_agreement_threshold: int = 2 # of 3 timeframes - cooldown_bars: int = 5 - default_risk_pct: float = 1.0 - default_rr_ratio: float = 2.0 - atr_sl_multiplier: float = 1.5 min_rr_ratio: float = 0.5 # TP tightening (May 2026) was tuned on the pre-spread backtester. # Re-tested 2026-07-08 with $0.30 spread + gap fills: 0.8 nearly @@ -199,25 +174,14 @@ class NewsConfig: @dataclass class XauStrategyConfig: - """XAU/USD strategy config. - Legacy gates (H4 RSI, H1 MACD, M15 score) removed in favor of - new momentum indicators (ROC, Keltner squeeze, ADX slope). - """ + """XAU/USD strategy config.""" enabled: bool = True - rsi_entry_threshold: float = 45.0 tp_atr_multiplier: float = 1.0 sl_atr_multiplier: float = 2.0 - min_ema200_tfs: int = 2 - allow_generic_trades: bool = True blocked_hours_utc: List[int] = field(default_factory=lambda: [0, 1, 2, 10, 11, 13, 16, 21]) weekend_filter: bool = True min_atr_threshold: float = 1.0 v2_sl_atr_multiplier: float = 2.0 - buy_threshold_boost: int = 0 - m5_confirmation_bars: int = 0 - h4_rsi_gate: bool = False - h1_macd_gate: bool = False - rsi_zone: tuple = (40.0, 65.0) allowed_weekdays: List[int] = field(default_factory=lambda: [0, 1, 2, 3, 4, 6]) # Mon-Fri + Sun (Sydney open) sell_min_quality: List[str] = field(default_factory=lambda: ["A+", "A", "B"]) # Volatility regimes in which SELL setups are allowed (low/normal-vol @@ -351,7 +315,6 @@ class PlatformConfig: scan_timeframes: List[Timeframe] = field( default_factory=lambda: [Timeframe.M5, Timeframe.M15, Timeframe.H1, Timeframe.H4] ) - candle_buffer_seconds: int = 10 ws_cooldown_seconds: int = 300 log_level: str = "INFO" diff --git a/data_provider.py b/data_provider.py index 018c1b0..250ed7b 100755 --- a/data_provider.py +++ b/data_provider.py @@ -204,25 +204,6 @@ class ForexDataProvider: logger.info(f"Cache cleared: {count} entries wiped") return count - def get_latest_price(self, symbol: str = XAUUSD) -> Optional[float]: - try: - df = self.fetch_ohlcv(symbol, Timeframe.M5, lookback_days=1) - if df is not None and not df.empty: - return float(df["Close"].iloc[-1]) - except Exception: - pass - return None - - def get_spread_estimate(self, symbol: str = XAUUSD) -> float: - try: - df = self.fetch_ohlcv(symbol, Timeframe.M5, lookback_days=1) - if df is not None and not df.empty: - return float((df["High"] - df["Low"]).median()) - except Exception: - pass - info = PAIR_INFO.get(symbol, PAIR_INFO[XAUUSD]) - return info["pip_value"] - def get_pair_info(self, symbol: str = XAUUSD) -> Dict: info = PAIR_INFO.get(symbol, PAIR_INFO[XAUUSD]) return {"symbol": symbol, **info} diff --git a/engine.py b/engine.py index 1287658..5eb85ec 100755 --- a/engine.py +++ b/engine.py @@ -13,7 +13,7 @@ Design philosophy: import logging from dataclasses import dataclass, field -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Tuple import numpy as np import pandas as pd diff --git a/logs/telegram.log.2026-07-14 b/logs/telegram.log.2026-07-14 deleted file mode 100644 index fe40ece..0000000 --- a/logs/telegram.log.2026-07-14 +++ /dev/null @@ -1,21 +0,0 @@ -01:51:29 [WARNING] data_provider: BTC/USD 1h: last bar 2026-07-11 05:51 is 4320m old (limit: 180m) -01:52:02 [WARNING] data_provider: BTC/USD 1h: last bar 2026-07-11 05:52 is 4320m old (limit: 180m) -02:42:59 [WARNING] data_provider: BTC/USD 1h: last bar 2026-07-11 06:42 is 4320m old (limit: 180m) -03:30:24 [WARNING] data_provider: BTC/USD 1h: last bar 2026-07-11 07:30 is 4320m old (limit: 180m) -03:31:32 [WARNING] data_provider: BTC/USD 1h: last bar 2026-07-11 07:31 is 4320m old (limit: 180m) -11:30:20 [WARNING] data_provider: BTC/USD 1h: last bar 2026-07-11 15:30 is 4320m old (limit: 180m) -11:41:34 [WARNING] data_provider: BTC/USD 1h: last bar 2026-07-11 15:41 is 4320m old (limit: 180m) -11:45:46 [WARNING] data_provider: BTC/USD 1h: last bar 2026-07-11 15:45 is 4320m old (limit: 180m) -11:56:38 [WARNING] data_provider: BTC/USD 1h: last bar 2026-07-11 15:56 is 4320m old (limit: 180m) -11:57:11 [WARNING] data_provider: BTC/USD 1h: last bar 2026-07-11 15:57 is 4320m old (limit: 180m) -11:59:52 [WARNING] data_provider: BTC/USD 1h: last bar 2026-07-11 15:59 is 4320m old (limit: 180m) -12:10:05 [WARNING] data_provider: BTC/USD 1h: last bar 2026-07-11 16:10 is 4320m old (limit: 180m) -13:05:37 [WARNING] data_provider: BTC/USD 1h: last bar 2026-07-11 17:05 is 4320m old (limit: 180m) -13:14:25 [WARNING] data_provider: BTC/USD 1h: last bar 2026-07-11 17:14 is 4320m old (limit: 180m) -13:38:50 [WARNING] data_provider: BTC/USD 1h: last bar 2026-07-11 17:38 is 4320m old (limit: 180m) -13:45:14 [WARNING] data_provider: BTC/USD 1h: last bar 2026-07-11 17:45 is 4320m old (limit: 180m) -13:51:33 [WARNING] data_provider: BTC/USD 1h: last bar 2026-07-11 17:51 is 4320m old (limit: 180m) -15:04:41 [WARNING] data_provider: BTC/USD 1h: last bar 2026-07-11 19:04 is 4320m old (limit: 180m) -15:04:50 [WARNING] data_provider: BTC/USD 1h: last bar 2026-07-11 19:04 is 4320m old (limit: 180m) -15:05:17 [WARNING] data_provider: BTC/USD 1h: last bar 2026-07-11 19:05 is 4320m old (limit: 180m) -15:06:10 [WARNING] data_provider: BTC/USD 1h: last bar 2026-07-11 19:06 is 4320m old (limit: 180m) diff --git a/main.py b/main.py index a74d3ec..f2cfc0c 100755 --- a/main.py +++ b/main.py @@ -13,7 +13,6 @@ import json import logging import sys from datetime import datetime, timezone -from typing import Dict, List # Force UTF-8 output on Windows so box-drawing chars render if hasattr(sys.stdout, "reconfigure"): diff --git a/news_engine.py b/news_engine.py index 1b7c4f0..769ab9d 100755 --- a/news_engine.py +++ b/news_engine.py @@ -14,7 +14,7 @@ import threading import time import xml.etree.ElementTree as ET from dataclasses import dataclass, field -from datetime import datetime, timedelta, timezone +from datetime import datetime, timezone from typing import Dict, List, Optional, Tuple import requests diff --git a/telegram_bot.py b/telegram_bot.py index 7f3aa46..d12c65d 100755 --- a/telegram_bot.py +++ b/telegram_bot.py @@ -42,7 +42,6 @@ import json import logging import logging.handlers import os -import signal import sys import time from concurrent.futures import ThreadPoolExecutor diff --git a/test_all.py b/test_all.py index 841c791..d24579a 100644 --- a/test_all.py +++ b/test_all.py @@ -100,7 +100,7 @@ def make_xau_ohlcv(n=300, start_price=2400.0, seed=42): print("\n═══ Module 1: config.py ═══") from config import ( - Timeframe, SignalStrength, MarketSession, + Timeframe, SignalStrength, PlatformConfig, TechnicalConfig, SignalConfig, NewsConfig, FilterConfig, XauStrategyConfig, XAUUSD, PAIR_INFO, @@ -130,24 +130,13 @@ def _(): assert SignalStrength.SELL.value > SignalStrength.STRONG_SELL.value -@test("MarketSession enum values") -def _(): - assert MarketSession.SYDNEY.value == "sydney" - assert MarketSession.TOKYO.value == "tokyo" - assert MarketSession.LONDON.value == "london" - assert MarketSession.NEW_YORK.value == "new_york" - assert MarketSession.OVERLAP_LONDON_NY.value == "london_ny_overlap" - - @test("PAIR_INFO completeness and correctness") def _(): assert XAUUSD in PAIR_INFO for pair, info in PAIR_INFO.items(): assert "display" in info, f"Missing 'display' for {pair}" assert "pip_value" in info, f"Missing 'pip_value' for {pair}" - assert "is_jpy_pair" in info, f"Missing 'is_jpy_pair' for {pair}" assert info["pip_value"] > 0, f"pip_value must be > 0 for {pair}" - assert PAIR_INFO[XAUUSD]["is_jpy_pair"] is False assert PAIR_INFO[XAUUSD]["pip_value"] == 0.10 @@ -180,14 +169,9 @@ def _(): @test("SignalConfig defaults") def _(): sc = SignalConfig() - assert sc.min_confidence == 55.0 + assert sc.mtf_agreement_threshold == 2 assert sc.min_rr_ratio == 0.5 assert sc.tp_tightening_factor == 1.0 - assert sc.atr_sl_multiplier == 1.5 - assert sc.default_rr_ratio == 2.0 - assert sc.cooldown_bars == 5 - assert "ema_trend" in sc.weights - assert "rsi" in sc.weights @test("PlatformConfig defaults") @@ -206,9 +190,6 @@ def _(): def _(): xc = XauStrategyConfig() assert xc.enabled is True - assert xc.rsi_entry_threshold == 45.0 - assert xc.min_ema200_tfs == 2 - assert xc.allow_generic_trades is True assert xc.tp_atr_multiplier == 1.0 assert xc.sl_atr_multiplier == 2.0 assert isinstance(xc.blocked_hours_utc, list) diff --git a/test_ws_dual.py b/test_ws_dual.py deleted file mode 100644 index 0e97674..0000000 --- a/test_ws_dual.py +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env python3 -"""Test whether Twelve Data allows 2 simultaneous WebSocket connections for XAU/USD. - -Uses the twelvedata SDK which handles the WS protocol correctly. -Runs connection 1, then adds connection 2 after 10s, monitors for 30s total. -""" - -import threading -import time - -from twelvedata import TDClient - -from app_config import provider_api_key - -API_KEY = provider_api_key() -SYMBOL = "XAU/USD" -DURATION = 30 - -results = { - "WS-1": {"ticks": 0, "errors": [], "alive": False}, - "WS-2": {"ticks": 0, "errors": [], "alive": False}, -} - - -def make_handlers(label): - def on_event(event): - if event.get("event") == "price": - results[label]["ticks"] += 1 - price = event.get("price", "?") - print(f"[{label}] tick #{results[label]['ticks']:3d} price={price}") - - def on_error(error): - results[label]["errors"].append(str(error)) - print(f"[{label}] ERROR: {error}") - - def on_close(status): - results[label]["alive"] = False - print(f"[{label}] CLOSED (status={status})") - - def on_connect(): - results[label]["alive"] = True - print(f"[{label}] CONNECTED — subscribed to {SYMBOL}") - - return on_event, on_error, on_close, on_connect - - -def run_ws(label): - td = TDClient(apikey=API_KEY) - on_event, on_error, on_close, on_connect = make_handlers(label) - ws = td.websocket( - symbols=SYMBOL, - on_event=on_event, - on_error=on_error, - on_close=on_close, - on_connect=on_connect, - ) - ws.connect() - return ws - - -def main(): - print(f"Testing dual WebSocket for {SYMBOL} ({DURATION}s)\n") - - print("Starting WS-1...") - ws1 = run_ws("WS-1") - - time.sleep(10) - print(f"\nWS-1 ticks so far: {results['WS-1']['ticks']}") - print("Starting WS-2...\n") - ws2 = run_ws("WS-2") - - remaining = DURATION - 10 - time.sleep(remaining) - - print("\nStopping connections...") - try: - ws1.disconnect() - except Exception: - pass - try: - ws2.disconnect() - except Exception: - pass - - time.sleep(1) - - print("\n" + "=" * 50) - print("RESULTS") - print("=" * 50) - for label, info in sorted(results.items()): - status = "alive" if info["alive"] else "dead" - errs = info["errors"] or ["none"] - print(f" {label}: {status} ticks={info['ticks']} errors={errs[0]}") - - ws1_ok = results["WS-1"]["ticks"] > 0 and not results["WS-1"]["errors"] - ws2_ok = results["WS-2"]["ticks"] > 0 and not results["WS-2"]["errors"] - ws1_died = results["WS-1"]["alive"] is False and results["WS-1"]["ticks"] > 0 - - if ws1_ok and ws2_ok: - print("\n✓ Both connections worked — dual WS is supported") - elif ws2_ok and ws1_died: - print("\n✗ WS-2 replaced WS-1 — only 1 connection allowed at a time") - elif ws1_ok and not ws2_ok: - print("\n✗ WS-2 was rejected — only 1 connection allowed") - else: - print("\n? Inconclusive — check errors above") - - -if __name__ == "__main__": - main()