diff --git a/.gitignore b/.gitignore index b568178..3baeced 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ Thumbs.db # Backtesting data (keep scripts, ignore generated files) backtest/cache/ backtest/*.csv +backtest/trailgrid/ backtest/__pycache__/ .env.* logs/ diff --git a/CLAUDE.md b/CLAUDE.md index de53abb..ce11831 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,7 @@ This file is the authoritative developer/agent reference. The human-facing docs | `pair_signals/__init__.py` | 52 | `PAIR_PRODUCERS` registry + `produce_signal()` | | `pair_signals/xauusd.py` | 71 | XAU config factory + per-timeframe overrides (docstring = validation record) | | `main.py` | ~348 | CLI: `python main.py --pair XAUUSD --timeframe H1 --detail/--json` | -| `test_all.py` | ~2800 | QA suite: **199 tests in 13 modules**, no live API needed. Must stay green. | +| `test_all.py` | ~2800 | QA suite: **200 tests in 13 modules**, no live API needed. Must stay green. | | `backtest/gpu_backtest.py` | ~2231 | Bar-replay backtester (CuPy→NumPy fallback) using the real `SignalAggregator` | | `backtest/diagnose_filters.py` | 609 | Filter-diagnosis tooling | | `backtest/research/` | — | Calendar research infra (`ff_calendar_scraper.py`, `calendar_store.py`) + result CSVs | @@ -80,7 +80,7 @@ EMA 9/21/50/200 (`ema_mode`: `"4ema"` default or `"2ema"` = 50/200 only); RSI 14 | `min_tp_cost_multiple` | 30.0 | spread gate: TP1 distance must be ≥ cost × this (= $9 on XAU). Validated: sub-$10 targets lose | | `min_sl_pips` | 80.0 | SL floor in pips ($8 on XAU) | | `tp_floor_rr` | 0.5 | TP1 floor as fraction of SL distance (M15 factory overrides to 1.0) | -| `trail_arm_r` / `trail_dist_r` | 0.6 / 0.75 | trailing exit: once favorable excursion ≥ arm×risk, stop trails dist×risk behind best price (bar-close updates). TP1 becomes advisory. Validated on 12mo M5: +105R vs +48R static. 0 disables | +| `trail_arm_r` / `trail_dist_r` | 0.6 / 0.75 | trailing exit: once favorable excursion ≥ arm×risk, stop trails dist×risk behind best price (bar-close updates). TP1 becomes advisory. Validated on 12mo M5: +105R vs +48R static. 0 disables. **M15 factory overrides arm to 0.70** (2026-07-21 grid: +210.7R vs +175.7R/5yr, OOS +57.4 vs +45.4; DD clause waived by owner — see factory docstring) | | `pb_trail_arm_r` / `pb_trail_dist_r` | 0.8 / 0.75 | trailing params for XAU_PB pullback trades (tighter stops → arm later) | ### NewsConfig @@ -256,6 +256,7 @@ Default confirmation map (same in the factory, the bot, and backtest usage): M5 - M5: append hour 9 UTC to blocked hours (6-yr replay: +207.1R vs +178.3R; M15-and-up keep hour 9 — it's positive there). Enable BUY-only chart patterns (`cp_enabled=True`, `cp_patterns=["inv_hs","double_bottom","bull_flag"]`): 12mo +41.8R PF 1.60 (train/val both positive), 5yr +65.2R PF 1.48; bearish CP dropped (OOS −3.21R). M15/H1 CP remain disabled (all failed OOS). - M15: `signals.tp_floor_rr = 1.0` (1R minimum targets: +116.6R vs +105.4R, better in train/val and 2024 OOS; **hurts M5** — per-TF only) and `xau_strategy.blocked_regimes = ["trending"]` (mixed regime block: +65.7R vs +59.0R). A transitioning-SELL block was tested and REJECTED (bull-market artifact — failed 2024 OOS). - M15: `xau_strategy.sell_blocked_hours_utc = [15, 17]` (SELL-only hour block, validated 2026-07-16 on the 5-yr replay: +176.7R PF 1.31 vs +144.9R PF 1.22, better in all 6 years, both 12-mo halves, and 2024-06→2025-03 OOS; M5 cross-TF pocket confirms; BUYs at those hours are +34.3R so `blocked_hours_utc` would be wrong). +- M15: `signals.trail_arm_r = 0.70` (trail arms later; validated 2026-07-21, 5-yr grid arm 0.3-0.9 × dist 0.5-1.0: +210.7R PF 1.35 WR 57.8% vs +175.7R PF 1.30 WR 53.9% at the 0.6 default; both halves and OOS better; 5yr max-DD clause 33.9R vs 29.1R waived by owner. Earlier-arm cells uniformly worse. M5 keeps 0.6). - The factory must stay equivalent to inline `PlatformConfig(symbol=XAUUSD, …)` + overrides — **enforced by a test in test_all.py**. --- @@ -292,6 +293,7 @@ Replays history bar-by-bar through the **real `SignalAggregator.analyze_from_dat - Same-bar SL+TP tie → SL first (conservative). Gap through SL/TP → filled at the bar open. - **Trailing exits** (default on, from `signals.trail_arm_r/trail_dist_r`; XAU_PB trades carry per-trade 0.8/0.75 overrides): stop ratchets at bar close only, so a new stop can only fill from the next bar (conservative). Exit reasons: SL / TRAIL / TP1 (static mode) / TIMEOUT / NO_DATA. - Session + weekend + weekday filters evaluate the **decision time = bar close** (`ts + primary_duration`) — exactly matching the live bot's wall-clock check. +- The optional H4-ADX regime filter (`config.backtest.regime_filter`, default off) is applied in the replay by calling the live `_apply_regime_filter_live` method itself under the same guard — live and backtest cannot diverge if it's enabled (2026-07-21 parity fix; the unregistered-pair config fallback also applies the config.json overlay now). - Confirmation TFs are sliced to bars **fully closed** by decision time (no look-ahead). **Performance:** `precompute_indicators` builds every indicator series once per TF (CuPy on GPU or NumPy — check the log line `Backend: CuPy GPU` vs `NumPy CPU fallback`; CuPy needs the `nvidia-*-cu12` pip wheels or it silently falls back). `GPUTechnicalEngine` subclasses the live engine and overrides only the `_compute_*` methods with cache lookups — outputs are byte-identical to the live engine (regression-tested). Bar replay: pure per-bar evaluation fans out over forked worker processes (`--workers`, 0 = auto: cores−2 capped 12; <5000 bars forces 1); the entry **cooldown (5 bars) is the only sequential rule**, applied in an ordered pass afterwards — parallel and sequential books are identical. ~300 bars/s/core, ~1700 bars/s at 10 workers. diff --git a/backtest/gpu_backtest.py b/backtest/gpu_backtest.py index bee59aa..8f530d6 100644 --- a/backtest/gpu_backtest.py +++ b/backtest/gpu_backtest.py @@ -1530,6 +1530,13 @@ def _replay_span(span: Tuple[int, int]): use_news=False, ) + # Live parity: analyze_pair applies the optional higher-TF ADX + # regime filter after delegation, under this exact guard. Calling + # the SAME method on the SAME sliced dfs keeps the two paths + # identical if the (default-off) filter is ever enabled. + if platform_cfg.backtest.regime_filter: + aggregator._apply_regime_filter_live(report, slice_dfs) + if report.trade_setup is None: continue if report.signal_quality not in bt_cfg.quality_filter: @@ -2141,6 +2148,10 @@ def _parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace: p.add_argument("--max-hold-bars", type=int, default=300) p.add_argument("--swing-sr-lookback", type=int, default=0, help="Override TechnicalConfig.swing_sr_lookback (0 = use config default)") + p.add_argument("--trail-arm", type=float, default=None, + help="Override signals.trail_arm_r for this run (research knob)") + p.add_argument("--trail-dist", type=float, default=None, + help="Override signals.trail_dist_r for this run (research knob)") p.add_argument("--quality", default="A+,A,B,C", help="Comma-separated allowed grades") p.add_argument("--strategies", default="", help="Comma-separated strategy tags to include (XAU,XAU_PB,XAU_CP,GENERIC). Empty = all.") @@ -2214,8 +2225,16 @@ def main(argv: Optional[List[str]] = None) -> int: primary_timeframe=primary_tf, confirmation_timeframes=confirm_tfs or [Timeframe.H1, Timeframe.H4], ) + # The factory path applies the config.json overlay inside + # make_config; the fallback must match (live == backtest). + from app_config import apply_pair_overrides # noqa: E402 + apply_pair_overrides(platform_cfg) if args.swing_sr_lookback > 0: platform_cfg.technical.swing_sr_lookback = args.swing_sr_lookback + if args.trail_arm is not None: + platform_cfg.signals.trail_arm_r = args.trail_arm + if args.trail_dist is not None: + platform_cfg.signals.trail_dist_r = args.trail_dist if args.cp: platform_cfg.xau_strategy.cp_enabled = True if args.cp_patterns: diff --git a/pair_signals/xauusd.py b/pair_signals/xauusd.py index 942b7e1..15f1f73 100644 --- a/pair_signals/xauusd.py +++ b/pair_signals/xauusd.py @@ -56,6 +56,22 @@ positive, val half -1.89R — marginal but sign-correct in aggregate). Gold's structural uptrend makes bullish patterns fundamentally more reliable. M15/H1 fail on all windows; M5-only enablement. +M15 override 4 (validated 2026-07-21, 5-year replay 2021-07-21 -> +2026-07-21, $0.30 spread, next-open fills): trailing-stop arm raised to +0.70R (trail_arm_r 0.70, dist unchanged 0.75). Full grid arm 0.3-0.9 x +dist 0.5-1.0 (30 cells): every earlier-arm cell (0.3/0.45) is WORSE — +the "arm earlier to save fading trades" hypothesis is refuted; the +plateau is arm 0.70-0.80 x dist 0.70-0.85 (all >= +190R). Chosen +plateau center 0.70/0.75: +210.7R/1629 PF 1.35 WR 57.8% vs baseline +0.60/0.75 +175.7R PF 1.30 WR 53.9%; both last-12mo halves improve +(+49.7/+50.0 vs +36.4/+46.0); per-year better 4/6 (2022 -0.9R, 2023 +-1.4R, both within tolerance); OOS 2024-06-01 -> 2025-03-30 +57.4R +PF 1.57 DD 17.0R vs +45.4R PF 1.47 DD 17.8R — better on every OOS +metric. One clause waived BY OWNER DECISION 2026-07-21: 5yr max DD is +33.9R vs 29.1R baseline (+4.8R deeper worst episode, 2022-23 chop), +accepted in exchange for +20% total R. M5 keeps the global 0.6 arm +(same per-TF divergence pattern as tp_floor_rr and the hour-9 block). + M5 override (validated 2026-07-11, full-history replay 2020-03-17 -> 2026-07-11 + 12mo book, $0.30 spread): block hour 9 UTC (post-London- open whipsaw). Hour-9 entries lost -28.9R/143 over 6 years, negative @@ -106,4 +122,7 @@ def make_config( config.xau_strategy.blocked_regimes = ["trending"] # Block SELL entries at 15:00/17:00 UTC on M15 only — see docstring. config.xau_strategy.sell_blocked_hours_utc = [15, 17] + # Later trailing-stop arm on M15 only — see module docstring. + config.signals.trail_arm_r = 0.70 + config.signals.trail_dist_r = 0.75 return config diff --git a/test_all.py b/test_all.py index d24579a..714e053 100644 --- a/test_all.py +++ b/test_all.py @@ -1868,6 +1868,8 @@ def _(): inline_cfg.signals.tp_floor_rr = 1.0 inline_cfg.xau_strategy.blocked_regimes = ["trending"] inline_cfg.xau_strategy.sell_blocked_hours_utc = [15, 17] + inline_cfg.signals.trail_arm_r = 0.70 + inline_cfg.signals.trail_dist_r = 0.75 if tf == Timeframe.M5: # Documented per-TF overrides (see xauusd.py docstring): # hour 9 UTC blocked + BUY-only chart patterns on M5 only. @@ -2060,6 +2062,13 @@ def _(): bl = r_live.indicators["bollinger"] bg = r_gpu.indicators["bollinger"] assert bl.signal == bg.signal and bl.description == bg.description, f"bollinger @ {i}" + al = r_live.indicators["atr"] + ag = r_gpu.indicators["atr"] + assert abs(al.value - ag.value) < 1e-9, f"atr value @ {i}" + assert al.description == ag.description, f"atr description @ {i}" + assert r_live.volatility_regime == r_gpu.volatility_regime, f"vol regime @ {i}" + assert abs(r_live.key_levels["atr"] - r_gpu.key_levels["atr"]) < 1e-9, f"key_levels atr @ {i}" + assert abs(r_live.key_levels["atr_pct"] - r_gpu.key_levels["atr_pct"]) < 1e-9, f"key_levels atr_pct @ {i}" if "Ribbon COMPRESSED" in el.description: seen_compressed = True if "Ribbon EXPANDING" in el.description: @@ -2068,6 +2077,45 @@ def _(): assert seen_expanding, "sweep never saw a ribbon expansion (+10 conf path) — no teeth" +@test("Regime filter: live gate semantics + backtest calls the same method") +def _(): + from signal_aggregator import SignalAggregator + from pair_signals import PAIR_PRODUCERS + cfg = PAIR_PRODUCERS[XAUUSD](Timeframe.M15, [Timeframe.M5, Timeframe.H1]) + cfg.backtest.regime_filter = True + agg = SignalAggregator(cfg) + + df = make_ohlcv(n=300, start_price=3400.0, trend=0.05, seed=3) + flat = make_ohlcv(n=300, start_price=3400.0, trend=0.0, seed=7) + adx = agg._latest_adx(flat, cfg.technical.adx_period) + assert adx is not None and adx < cfg.backtest.regime_filter_min_adx, \ + f"fixture must be choppy (ADX {adx}) or the test has no teeth" + + # Low higher-TF ADX → signal downgraded to NEUTRAL/F, setup killed. + report = agg.analyze_from_dataframes(XAUUSD, {Timeframe.M15: df}, + context=None, use_news=False) + agg._apply_regime_filter_live(report, {Timeframe.H4: flat}) + assert report.final_signal == SignalStrength.NEUTRAL + assert report.signal_quality == "F" + assert report.trade_setup is None + assert any("REGIME FILTER" in w for w in report.warnings) + + # Regime TF absent from dfs → warning only, report untouched. + report2 = agg.analyze_from_dataframes(XAUUSD, {Timeframe.M15: df}, + context=None, use_news=False) + quality_before = report2.signal_quality + agg._apply_regime_filter_live(report2, {}) + assert report2.signal_quality == quality_before + assert any("data unavailable" in w for w in report2.warnings) + + # Backtest wiring: the replay applies the SAME method under the SAME + # guard as live analyze_pair (function-identity parity). + import gpu_backtest as _gb + src = open(_gb.__file__, encoding="utf-8").read() + assert "_apply_regime_filter_live(report, slice_dfs)" in src + assert "backtest.regime_filter" in src + + @test("disabled_indicators=['macd'] removes the pillar but keeps the histogram") def _(): from engine import TechnicalEngine @@ -2521,6 +2569,16 @@ def _(): assert args2.cp is False +@test("Backtester: --trail-arm/--trail-dist research knobs parse, default None") +def _(): + from gpu_backtest import _parse_args + args = _parse_args(["--pair", "XAUUSD", + "--trail-arm", "0.45", "--trail-dist", "0.75"]) + assert args.trail_arm == 0.45 and args.trail_dist == 0.75 + args2 = _parse_args(["--pair", "XAUUSD"]) + assert args2.trail_arm is None and args2.trail_dist is None + + # ══════════════════════════════════════════════════════════════════════ # MODULE 13: app_config.py (config.json overlay) # ══════════════════════════════════════════════════════════════════════