From 042b079861628feddbd5858e283fa6891ed77f71 Mon Sep 17 00:00:00 2001 From: Harwinder Sekhon Date: Thu, 30 Jul 2026 11:30:20 -0400 Subject: [PATCH] Big-candle gate (default-OFF) + M5 relaxation campaign + --set tooling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gate 6h (big-candle entry gate): blocks all XAU entries (core, XAU_PB, XAU_CP) when the primary TF's last 24h contains a bar with range >= K x prior-bar ATR. Ships default-OFF: validation rejected every threshold (K 3-6 on M5/M15 12mo) — removed trades were net-POSITIVE at every cell; M5 quiet-window trades are the only losing bucket. Config knobs big_candle_atr_multiple/lookback_hours, --big-candle backtest flag, Module 17 tests. M5 relaxation SHIPPED (validated 2026-07-30, Oanda bars, $0.30 spread): m5_min_abs_score 30->25, m5_min_confirmation_score 10->5, hour-21 UTC unblocked M5-only. Beats control in ALL 6 calendar years of the 5yr replay (+230.9R/2319 PF 1.26 DD 21.3 vs +168.0R/2048 PF 1.21 DD 27.0), 12mo +107.4R PF 1.36 vs +84.6R PF 1.33, OOS +26.1R vs +17.2R, $0.70 spread +86.9R vs +66.4R. M15: every relaxation retested negative — config unchanged. Declined-but-open: M5 hour-22 block, M15 hour-16 unblock (both fail 5yr/OOS). Evidence in the factory docstring. Tooling: generic --set PATH=JSON research override in gpu_backtest (typo-proof, logged); diagnose_filters filter_tags refreshed for gates 6d-6i with a staleness-guard test (Module 18); research scripts hour_analysis/burst_profile/book_diff. Suite: 268 tests. Claude-Session: https://claude.ai/code/session_014jqHpmNYmwYMowF6bPZyeD --- backtest/diagnose_filters.py | 68 +++++--- backtest/gpu_backtest.py | 53 ++++++ backtest/research/book_diff.py | 86 ++++++++++ backtest/research/burst_profile.py | 42 +++++ backtest/research/hour_analysis.py | 40 +++++ config.py | 20 ++- pair_signals/xauusd.py | 27 +++- signal_aggregator.py | 69 +++++++- test_all.py | 252 ++++++++++++++++++++++++++++- 9 files changed, 621 insertions(+), 36 deletions(-) create mode 100644 backtest/research/book_diff.py create mode 100644 backtest/research/burst_profile.py create mode 100644 backtest/research/hour_analysis.py diff --git a/backtest/diagnose_filters.py b/backtest/diagnose_filters.py index a2d82a3..3b026c7 100644 --- a/backtest/diagnose_filters.py +++ b/backtest/diagnose_filters.py @@ -79,34 +79,54 @@ def classify_bar( """ filters_hit: List[str] = [] - # Scan warnings for known filter tags - filter_tags = { - "H4 RSI GATE": "H4_RSI_GATE", - "H1 MACD GATE": "H1_MACD_GATE", - "RSI ZONE GATE": "RSI_ZONE_GATE", - "XAU BUY FILTER": "XAU_BUY_FILTER", - "SESSION FILTER": "SESSION_FILTER", - "DAY FILTER": "DAY_FILTER", - "LOW ATR FILTER": "LOW_ATR_FILTER", - "SELL SUPPRESSED": "SELL_SUPPRESSED", - "EXHAUSTION": "EXHAUSTION_PENALTY", - "TREND EXHAUSTION": "TREND_EXHAUSTION", - "NO PULLBACK": "NO_PULLBACK_PENALTY", - "Weak pullback": "WEAK_PULLBACK_PENALTY", - "Extended entry": "EXTENDED_ENTRY_PENALTY", - "Counter-trend": "COUNTER_TREND_PENALTY", - "EXTREME volatility":"EXTREME_VOL_PENALTY", - "NEAR EMA200": "NEAR_EMA200_PENALTY", - "FAR FROM EMA200": "FAR_EMA200_PENALTY", - "EVENT RISK": "EVENT_RISK", - "REGIME FILTER": "REGIME_FILTER", - "Timeframes CONFLICT": "MTF_CONFLICT", - } + # Known filter tags, matched against the exact warning strings the + # aggregator emits (refreshed 2026-07-30 for gates 6d-6i). ORDER + # MATTERS: most-specific prefix first, and each warning maps to at + # most ONE tag (break on first hit) — substring families like + # "M5 FILTER: SELL quality" vs "M5 FILTER" and "TREND EXHAUSTION" + # vs "EXHAUSTION" would otherwise double-count. + filter_tags = [ + # M5 noise gates (6i, _apply_m5_filters) + ("M5 FILTER: |score|", "M5_SCORE"), + ("M5 FILTER: no MTF agreement", "M5_MTF"), + ("M5 FILTER: no confirmation TF", "M5_CONF"), + ("M5 FILTER: SELL quality", "M5_SELL_QUALITY"), + ("M5 FILTER: SELL blocked — RSI", "M5_SELL_RSI"), + ("M5 FILTER: '", "M5_VOL"), + ("M5 FILTER", "M5_OTHER"), + # XAU SELL gates (6e/6g/6g2/6g3) + ("SELL FILTER: quality", "SELL_QUALITY"), + ("SELL FILTER: volatility", "SELL_VOL"), + ("SELL FILTER: regime", "SELL_REGIME"), + ("UTC entry blocked for shorts", "SELL_HOUR"), + # Hard gates + ("MTF OPPOSITION", "MTF_OPPOSITION"), + ("EXHAUSTED TREND FILTER", "EXHAUSTED_TREND"), + ("BIG CANDLE FILTER", "BIG_CANDLE"), + ("WEEKEND FILTER", "WEEKEND_FILTER"), + ("SESSION FILTER", "SESSION_FILTER"), + ("DAY FILTER", "DAY_FILTER"), + ("LOW ATR FILTER", "LOW_ATR_FILTER"), + ("REGIME FILTER", "REGIME_FILTER"), + # Soft penalties / context warnings + ("TREND EXHAUSTION", "TREND_EXHAUSTION"), + ("EXHAUSTION", "EXHAUSTION_PENALTY"), + ("NO PULLBACK", "NO_PULLBACK_PENALTY"), + ("Weak pullback", "WEAK_PULLBACK_PENALTY"), + ("Extended — RSI", "RSI_EXTENDED_PENALTY"), + ("Counter-trend", "COUNTER_TREND_PENALTY"), + ("EXTREME volatility", "EXTREME_VOL_PENALTY"), + ("NEAR EMA200", "NEAR_EMA200_PENALTY"), + ("FAR FROM EMA200", "FAR_EMA200_PENALTY"), + ("EVENT RISK", "EVENT_RISK"), + ("Timeframes CONFLICT", "MTF_CONFLICT"), + ] for w in report.warnings: - for tag, name in filter_tags.items(): + for tag, name in filter_tags: if tag in w: filters_hit.append(name) + break # Determine outcome non_neutral = report.final_signal not in (SignalStrength.NEUTRAL,) diff --git a/backtest/gpu_backtest.py b/backtest/gpu_backtest.py index 7d5d6a7..cc8a6ed 100644 --- a/backtest/gpu_backtest.py +++ b/backtest/gpu_backtest.py @@ -30,6 +30,7 @@ Usage: from __future__ import annotations import argparse +import json import logging import multiprocessing as mp import os @@ -2117,6 +2118,19 @@ def _parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace: p.add_argument("--cp-patterns", default=None, help="Comma-separated pattern subset for --cp (e.g. " "double_bottom,bull_flag). Default: all detectors.") + p.add_argument("--big-candle", type=float, default=None, + help="Force xau_strategy.big_candle_atr_multiple=K for this " + "run (no entries while the lookback window contains a " + "bar with range >= K x prior ATR) — for validating the " + "candidate.") + p.add_argument("--big-candle-hours", type=float, default=None, + help="Override big_candle_lookback_hours for --big-candle " + "runs (default 24).") + p.add_argument("--set", action="append", default=[], metavar="PATH=JSON", + help="Override any PlatformConfig leaf by dotted path; the " + "value is JSON-parsed for typing (research knob; " + "repeatable). e.g. --set xau_strategy.m5_min_abs_score=25 " + "--set 'xau_strategy.sell_min_quality=[\"A+\",\"A\",\"B\",\"C\"]'") p.add_argument("--no-cache", action="store_true", help="Skip parquet cache (force API fetch)") p.add_argument("--spread", type=float, default=None, help="Round-trip cost in price units per trade (default: pair's " @@ -2148,6 +2162,39 @@ def _resolve_tf(s: str) -> Timeframe: raise SystemExit(f"Unknown timeframe '{s}'. Valid: {[t.value for t in Timeframe]}") +def _apply_config_overrides(cfg: PlatformConfig, overrides: List[str]) -> None: + """Apply repeatable --set PATH=JSON research overrides onto the config. + + Unknown paths are a hard error: dataclass instances silently accept + setattr of any name, so a typo'd path would otherwise produce a + no-op run that looks like 'the relaxation had no effect'. Every + applied override is logged — the run log is the config audit trail + required by instructions_for_backtest.txt. + """ + for item in overrides: + path, sep, raw = item.partition("=") + if not sep: + raise SystemExit(f"--set expects PATH=VALUE, got '{item}'") + try: + value = json.loads(raw) + except json.JSONDecodeError: + value = raw # bare-string convenience ("oanda" etc.) + obj = cfg + parts = path.strip().split(".") + for attr in parts[:-1]: + if not hasattr(obj, attr): + raise SystemExit(f"--set: unknown config path '{path}'") + obj = getattr(obj, attr) + leaf = parts[-1] + if not hasattr(obj, leaf): + raise SystemExit(f"--set: unknown config path '{path}'") + old = getattr(obj, leaf) + if hasattr(old, "__dataclass_fields__"): + raise SystemExit(f"--set: '{path}' is a config section, not a leaf") + setattr(obj, leaf, value) + logger.info(f"--set override: {path} = {value!r} (was {old!r})") + + def main(argv: Optional[List[str]] = None) -> int: args = _parse_args(argv) @@ -2197,6 +2244,12 @@ def main(argv: Optional[List[str]] = None) -> int: platform_cfg.xau_strategy.cp_patterns = [ s.strip() for s in args.cp_patterns.split(",") if s.strip() ] + if args.big_candle is not None: + platform_cfg.xau_strategy.big_candle_atr_multiple = args.big_candle + if args.big_candle_hours is not None: + platform_cfg.xau_strategy.big_candle_lookback_hours = args.big_candle_hours + if args.set: + _apply_config_overrides(platform_cfg, args.set) confirm_tfs = list(platform_cfg.confirmation_timeframes) end = ( diff --git a/backtest/research/book_diff.py b/backtest/research/book_diff.py new file mode 100644 index 0000000..c143131 --- /dev/null +++ b/backtest/research/book_diff.py @@ -0,0 +1,86 @@ +"""Trade-book diff: relaxed/gated run vs its control. + +Usage: + python backtest/research/book_diff.py CONTROL.csv CANDIDATE.csv BAR_MINUTES + +Decision numbers are RUN-LEVEL (ΔtotalR, ΔPF, ΔmaxDD, Δtrades) — never +the added-trade bucket alone: the 5-bar entry cooldown means new +entries displace existing ones. The added/removed decomposition and the +per-decision-hour histogram of added trades are diagnostics. + +Shifted trades = removed+added pairs in the same direction within +±5 bars (cooldown displacement, not a genuinely new/lost signal). +""" +import sys + +import pandas as pd + + +def _metrics(df): + r = df["r_multiple"] + gains, losses = r[r > 0].sum(), -r[r <= 0].sum() + cum = r.cumsum() + return dict(trades=len(df), total_r=r.sum(), + pf=gains / max(losses, 1e-9), + wr=(r > 0).mean() * 100 if len(df) else 0.0, + max_dd=(cum.cummax() - cum).max() if len(cum) else 0.0) + + +def diff_books(control_csv, candidate_csv, bar_minutes, split="2026-01-29"): + ctrl, cand = pd.read_csv(control_csv), pd.read_csv(candidate_csv) + for df in (ctrl, cand): + df["_t"] = pd.to_datetime(df["entry_time"], utc=True, format="mixed") + df["_key"] = df["entry_time"].astype(str) + "|" + df["direction"] + + added = cand[~cand["_key"].isin(ctrl["_key"])].copy() + removed = ctrl[~ctrl["_key"].isin(cand["_key"])].copy() + + # cooldown displacement: same-direction removed/added within ±5 bars + tol = pd.Timedelta(minutes=5 * bar_minutes) + shifted = 0 + rem_pool = removed.copy() + for _, a in added.iterrows(): + m = rem_pool[(rem_pool["direction"] == a["direction"]) + & ((rem_pool["_t"] - a["_t"]).abs() <= tol)] + if len(m): + shifted += 1 + rem_pool = rem_pool.drop(m.index[0]) + + mc, mk = _metrics(ctrl), _metrics(cand) + split_ts = pd.Timestamp(split, tz="UTC") + halves = (cand[cand["_t"] < split_ts]["r_multiple"].sum(), + cand[cand["_t"] >= split_ts]["r_multiple"].sum()) + hour_hist = None + if len(added): + added["dec_hour"] = (added["_t"] + - pd.Timedelta(minutes=bar_minutes)).dt.hour + hour_hist = added.groupby("dec_hour")["r_multiple"] \ + .agg(["count", "sum"]).round(2) + return dict( + control=mc, candidate=mk, + delta_r=mk["total_r"] - mc["total_r"], + delta_pf=mk["pf"] - mc["pf"], + delta_dd=mk["max_dd"] - mc["max_dd"], + delta_trades=mk["trades"] - mc["trades"], + added_n=len(added), added_r=added["r_multiple"].sum(), + removed_n=len(removed), removed_r=removed["r_multiple"].sum(), + shifted=shifted, halves=halves, added_by_hour=hour_hist, + ) + + +if __name__ == "__main__": + d = diff_books(sys.argv[1], sys.argv[2], int(sys.argv[3])) + c, k = d["control"], d["candidate"] + print(f"control : {c['trades']:4d} trades {c['total_r']:+7.1f}R " + f"PF {c['pf']:.2f} WR {c['wr']:.1f}% DD {c['max_dd']:.1f}") + print(f"candidate: {k['trades']:4d} trades {k['total_r']:+7.1f}R " + f"PF {k['pf']:.2f} WR {k['wr']:.1f}% DD {k['max_dd']:.1f}") + print(f"ΔR {d['delta_r']:+.1f} ΔPF {d['delta_pf']:+.2f} " + f"ΔDD {d['delta_dd']:+.1f} Δtrades {d['delta_trades']:+d}") + print(f"added {d['added_n']} ({d['added_r']:+.1f}R) " + f"removed {d['removed_n']} ({d['removed_r']:+.1f}R) " + f"shifted≈{d['shifted']}") + print(f"candidate halves: H1 {d['halves'][0]:+.1f}R H2 {d['halves'][1]:+.1f}R") + if d["added_by_hour"] is not None: + print("\nadded trades by decision hour:") + print(d["added_by_hour"].to_string()) diff --git a/backtest/research/burst_profile.py b/backtest/research/burst_profile.py new file mode 100644 index 0000000..31def0c --- /dev/null +++ b/backtest/research/burst_profile.py @@ -0,0 +1,42 @@ +"""Burst-candle frequency by UTC hour — the empirical "hedge fund hours". + +Usage: + python backtest/research/burst_profile.py PARQUET_GLOB + +Per UTC bar-close hour of the given bars: % of bars whose range is +>= K x the PRIOR bar's Wilder ATR(14), for K in {3,4,5}; mean bar +range; and mean signed drift over the 6 bars following a >=4x burst +(does price continue or snap back?). Analysis tooling only. +""" +import glob +import sys + +import pandas as pd + + +def burst_table(parquet_glob: str, bar_minutes: int = 5): + files = sorted(glob.glob(parquet_glob)) + if not files: + raise SystemExit(f"no parquet matches {parquet_glob}") + df = pd.concat([pd.read_parquet(f) for f in files]).sort_index() + df = df[~df.index.duplicated(keep="last")] + h, l, c = df["High"], df["Low"], df["Close"] + tr = pd.concat([h - l, (h - c.shift()).abs(), (l - c.shift()).abs()], + axis=1).max(axis=1) + atr = tr.ewm(alpha=1 / 14, min_periods=14, adjust=False).mean() + ratio = (h - l) / atr.shift(1) + close_hour = (df.index + pd.Timedelta(minutes=bar_minutes)).hour + fwd6 = c.shift(-6) - c # signed move over the 6 bars after this one + out = pd.DataFrame({ + "bars": ratio.groupby(close_hour).size(), + **{f"pct_ge_{k}x": (ratio >= k).groupby(close_hour).mean() * 100 + for k in (3, 4, 5)}, + "mean_range": (h - l).groupby(close_hour).mean(), + "post4x_drift6": fwd6[ratio >= 4].groupby(close_hour[ratio >= 4]).mean(), + }) + return out.round(3) + + +if __name__ == "__main__": + print(burst_table(sys.argv[1], + int(sys.argv[2]) if len(sys.argv) > 2 else 5).to_string()) diff --git a/backtest/research/hour_analysis.py b/backtest/research/hour_analysis.py new file mode 100644 index 0000000..41a4ee4 --- /dev/null +++ b/backtest/research/hour_analysis.py @@ -0,0 +1,40 @@ +"""Per-hour P&L breakdown of a backtest trade book. + +Usage: + python backtest/research/hour_analysis.py BOOK.csv BAR_MINUTES + +Reports, per UTC hour (both entry hour and DECISION hour = entry_time +minus one bar, the clock blocked_hours_utc evaluates): trade count, +total R, mean R, win rate, profit factor. Also a per-(hour, direction) +table. Analysis tooling only — never imported by the live code. +""" +import sys + +import pandas as pd + + +def hour_tables(book_csv: str, bar_minutes: int): + df = pd.read_csv(book_csv) + et = pd.to_datetime(df["entry_time"], utc=True, format="mixed") + df["entry_hour"] = et.dt.hour + df["decision_hour"] = (et - pd.Timedelta(minutes=bar_minutes)).dt.hour + + def table(group_cols): + g = df.groupby(group_cols)["r_multiple"] + out = g.agg(count="count", total_r="sum", mean_r="mean", + wr=lambda r: (r > 0).mean() * 100) + out["pf"] = g.apply( + lambda r: r[r > 0].sum() / max(-r[r <= 0].sum(), 1e-9)) + return out.round(2) + + return (table("decision_hour"), table(["decision_hour", "direction"]), + table("entry_hour")) + + +if __name__ == "__main__": + book, mins = sys.argv[1], int(sys.argv[2]) + by_dec, by_dec_dir, by_ent = hour_tables(book, mins) + print(f"\n── {book} · by DECISION hour (bar-close clock) ──") + print(by_dec.to_string()) + print(f"\n── by (decision hour, direction) ──") + print(by_dec_dir.to_string()) diff --git a/config.py b/config.py index 2d810be..a83f377 100755 --- a/config.py +++ b/config.py @@ -203,9 +203,16 @@ class XauStrategyConfig: # M5 signals are noisy; require stronger confirmation before # generating trade setups. Without these, M5 is breakeven # (+0.01R/trade) with 21% drawdown. + # Relaxed 2026-07-30 (30→25, 10→5) in the relaxation campaign: + # the combined config (with the M5 hour-21 unblock) beat control + # in ALL 6 calendar years of the 5yr Oanda replay (+230.9R vs + # +168.0R, PF 1.26 vs 1.21, max DD 21.3 vs 27.0), 12mo +107.4R + # vs +84.6R PF 1.36 both halves positive, OOS 2024-06→2025-03 + # +26.1R vs +17.2R, and at $0.70 spread +86.9R vs +66.4R. + # Score floors below 25 are vacuous (grade C requires |score|≥25). m5_require_mtf: bool = True - m5_min_abs_score: float = 30.0 - m5_min_confirmation_score: float = 10.0 + m5_min_abs_score: float = 25.0 + m5_min_confirmation_score: float = 5.0 # A-grade M5 SELLs enter at the extreme of temporary dips and # lose (-0.017R/trade on 163 trades) — maximum conviction at # the worst entry. Cap SELL quality so A-grade can't trigger @@ -271,6 +278,15 @@ class XauStrategyConfig: cp_flag_max_bars: int = 12 cp_flag_retrace: tuple = (0.1, 0.5) # flag depth as fraction of pole + # ── Big-candle entry gate ────────────────────────────────────── + # When the recent window on the scan's own primary TF contains an + # abnormally large candle (bar range >= K x the PRIOR bar's ATR — + # institutional burst; price may reverse violently), no XAU entry + # of ANY kind may fire (core, XAU_PB, XAU_CP). 0.0 = disabled. + # Per-TF factories opt in only after the standard validation rule. + big_candle_atr_multiple: float = 0.0 + big_candle_lookback_hours: float = 24.0 + @dataclass class FilterConfig: diff --git a/pair_signals/xauusd.py b/pair_signals/xauusd.py index b575426..2a3902b 100644 --- a/pair_signals/xauusd.py +++ b/pair_signals/xauusd.py @@ -99,6 +99,24 @@ in 5/7 calendar years including winning years (2022 -9.2R). Blocking: 12mo book is unchanged (+139.0R vs +139.3R). M15-and-up keep hour 9: it is POSITIVE there (M15 6yr +1.7R, M15 OOS +8.6R), so the block is M5-only, same per-timeframe pattern as the M15 overrides. + +M5 override 2 — relaxation campaign (validated 2026-07-30, Oanda +bars, $0.30 spread, trailing exits): UNBLOCK hour 21 UTC on M5 only, +combined with the config.py M5 gate relaxations (m5_min_abs_score +30->25, m5_min_confirmation_score 10->5). The combined config beat +the pre-relaxation control in ALL 6 calendar years of the 5yr replay +2021-07-30 -> 2026-07-30 (+230.9R/2319 PF 1.26 DD 21.3 vs ++168.0R/2048 PF 1.21 DD 27.0), on 12mo (+107.4R PF 1.36, halves ++68.6/+38.8 vs +84.6R PF 1.33), on OOS 2024-06-01 -> 2025-03-30 +(+26.1R vs +17.2R), and at $0.70 spread (+86.9R vs +66.4R). Hour-21 +trades alone: +28.0R/22, positive every year — overturning the old +block per the full-evidence rule. Hour 21 stays BLOCKED on M15+ +(12mo M15 unblock tested -0.3R). Declined-but-open from the same +campaign: an M5 hour-22 block (+4.8R on 12mo — the burst-heavy +"institutional" hour — but -3.2R on 5yr and -1.1R on OOS: regime- +specific); M15 hour-16 unblock (+0.8R/+57 trades 12mo but -18.3R +DD +34% on 5yr); every SELL-gate/vol/TP-size relaxation retested +negative (see memory notes for the full grid). """ from typing import Optional, List @@ -126,10 +144,11 @@ def make_config( confirmation_tfs or _CONFIRMATIONS.get(primary_tf, [Timeframe.M15, Timeframe.H4]) ) if primary_tf == Timeframe.M5: - # Block hour 9 UTC on M5 only — see module docstring. - config.xau_strategy.blocked_hours_utc = ( - config.xau_strategy.blocked_hours_utc + [9] - ) + # Block hour 9 UTC on M5 only; UNBLOCK hour 21 on M5 only + # (positive every year of the 5yr replay) — see docstring. + config.xau_strategy.blocked_hours_utc = [ + h for h in config.xau_strategy.blocked_hours_utc if h != 21 + ] + [9] # BUY-only chart patterns on M5 — see module docstring. config.xau_strategy.cp_enabled = True config.xau_strategy.cp_patterns = ["inv_hs", "double_bottom", "bull_flag"] diff --git a/signal_aggregator.py b/signal_aggregator.py index 1a646f0..b72974b 100755 --- a/signal_aggregator.py +++ b/signal_aggregator.py @@ -306,6 +306,44 @@ class SignalAggregator: return None return float(adx.iloc[-1]) + def _detect_big_candle(self, primary_df: pd.DataFrame) -> Optional[tuple]: + """Largest abnormal candle in the recent primary-TF window. + + Returns (timestamp, ratio) of the biggest bar whose range is + >= big_candle_atr_multiple x the PRIOR bar's ATR within the last + big_candle_lookback_hours of primary-TF bars, else None. The + prior-bar ATR (shift 1) matters: a huge candle inflates the + Wilder ATR immediately, so comparing against the concurrent ATR + under-detects the very bars this gate exists to catch. + + Pure pandas on the caller-sliced df, so the live path and the + backtest replay (which calls analyze_from_dataframes on prefix + slices) compute bit-identical answers. The tail slice keeps the + per-call cost bounded (~n + 42 rows); min_periods NaNs and short + histories degrade to "no trip". + """ + cfg = self.config.xau_strategy + bar_secs = BaseDataProvider.CANDLE_SECONDS[self.config.primary_timeframe] + n = max(1, int(cfg.big_candle_lookback_hours * 3600 // bar_secs)) + period = self.config.technical.atr_period + tail = primary_df.iloc[-(n + 3 * period):] + high, low, close = tail["High"], tail["Low"], tail["Close"] + tr = pd.concat( + [high - low, + (high - close.shift()).abs(), + (low - close.shift()).abs()], + axis=1, + ).max(axis=1) + atr = tr.ewm(alpha=1 / period, min_periods=period, adjust=False).mean() + ratio = (high - low) / atr.shift(1) + window = ratio.iloc[-n:] + if window.empty: + return None + max_ratio = window.max() + if pd.isna(max_ratio) or max_ratio < cfg.big_candle_atr_multiple: + return None + return window.idxmax(), float(max_ratio) + def _apply_m5_filters(self, report: SignalReport) -> None: """Extra selectivity for M5 — noisy TF needs stronger confirmation.""" if report.signal_quality not in ("A+", "A", "B", "C"): @@ -957,6 +995,30 @@ class SignalAggregator: report.signal_quality = "D" report.trade_setup = None + # ── 6h. Big-candle entry gate (opt-in via per-TF factories) ── + # An abnormally large candle in the recent window = institutional + # burst; price may reverse violently, so NO entry of any kind may + # fire. Detection runs for ANY grade: the warning is the + # suppression channel for the XAU_PB path (which accepts D) and + # the XAU_CP path (which accepts even F). The D-downgrade itself + # touches actionable grades only — F stays F (same convention as + # 6e). + if (self.config.xau_strategy.enabled + and report.display_name == "XAU/USD" + and self.config.xau_strategy.big_candle_atr_multiple > 0): + big_candle = self._detect_big_candle(primary_df) + if big_candle is not None: + bc_ts, bc_ratio = big_candle + report.warnings.append( + f"BIG CANDLE FILTER: {bc_ratio:.1f}x ATR bar at {bc_ts} " + f"within " + f"{self.config.xau_strategy.big_candle_lookback_hours:g}h " + f"— no entries" + ) + if report.signal_quality in ("A+", "A", "B", "C"): + report.signal_quality = "D" + report.trade_setup = None + # ── 6i. M5 noise gate (gold-tuned thresholds — XAU only) ── if (self.config.primary_timeframe == Timeframe.M5 and report.display_name == "XAU/USD"): @@ -985,7 +1047,9 @@ class SignalAggregator: and report.display_name == "XAU/USD" and self.config.xau_strategy.m5_pullback_enabled and report.trade_setup is None - and report.signal_quality != "F"): + and report.signal_quality != "F" + and not any(w.startswith("BIG CANDLE FILTER") + for w in report.warnings)): self._try_m5_pullback(report, primary_df, pip_value) # ── 10. Chart-pattern entry (XAU_CP) ──────────────────────── @@ -1002,7 +1066,8 @@ class SignalAggregator: and self.config.xau_strategy.cp_enabled and report.trade_setup is None and not any(w.startswith(("SESSION FILTER", "DAY FILTER", - "WEEKEND FILTER")) + "WEEKEND FILTER", + "BIG CANDLE FILTER")) for w in report.warnings)): self._try_chart_pattern(report, primary_df, pip_value) diff --git a/test_all.py b/test_all.py index 87078d8..72d8ac2 100644 --- a/test_all.py +++ b/test_all.py @@ -2000,10 +2000,11 @@ def _(): inline_cfg.signals.trail_dist_r = 0.75 if tf == Timeframe.M5: # Documented per-TF overrides (see xauusd.py docstring): - # hour 9 UTC blocked + BUY-only chart patterns on M5 only. - inline_cfg.xau_strategy.blocked_hours_utc = ( - inline_cfg.xau_strategy.blocked_hours_utc + [9] - ) + # hour 9 UTC blocked, hour 21 UNBLOCKED (2026-07-30 + # relaxation campaign) + BUY-only chart patterns on M5 only. + inline_cfg.xau_strategy.blocked_hours_utc = [ + h for h in inline_cfg.xau_strategy.blocked_hours_utc if h != 21 + ] + [9] inline_cfg.xau_strategy.cp_enabled = True inline_cfg.xau_strategy.cp_patterns = ["inv_hs", "double_bottom", "bull_flag"] inline_cfg.signals.trail_arm_r = 0.60 @@ -4309,6 +4310,249 @@ def _(): assert td != oa, "providers must not share cache files" +# ══════════════════════════════════════════════════════════════════════ +# MODULE 17: Big-candle entry gate (gate 6h) +# ══════════════════════════════════════════════════════════════════════ + +print("\n═══ Module 17: big-candle entry gate ═══") + + +def _inject_big_bar(df, pos=-250, mult=8.0): + """Copy df and widen one bar to ~mult x the median range (a burst + candle) at position `pos` — far enough back not to disturb the + near-window indicators (RSI/BB/ATR 14-20 bars, swing S/R 120).""" + out = df.copy() + atr_est = (out["High"] - out["Low"]).median() + ts = out.index[pos] + out.loc[ts, "High"] = out.loc[ts, "Open"] + mult * atr_est + out.loc[ts, "Low"] = out.loc[ts, "Open"] - 0.5 * atr_est + out.loc[ts, "Close"] = out.loc[ts, "Open"] + mult * 0.9 * atr_est + return out + + +def _bc_agg(k=3.0, tf=Timeframe.M5, confirms=(Timeframe.M15, Timeframe.H1)): + from pair_signals.xauusd import make_config as xau_make_config + cfg = xau_make_config(tf, list(confirms)) + cfg.xau_strategy.big_candle_atr_multiple = k + return SignalAggregator(cfg) + + +# Deterministic ACTIONABLE fixture: grade A BUY with a GENERIC setup +# through the factory M5 config at an unblocked hour (Wed 14:55 UTC), +# huge bar pre-injected 250 bars back (verified by seed search). +def _bc_actionable_dfs(): + end = "2026-07-08 14:55" + primary = _inject_big_bar(make_xau_bars(end=end, drift=0.8, seed=0)) + m15 = make_xau_bars(end="2026-07-08 14:45", freq="15min", drift=0.8, seed=100) + h1 = make_xau_bars(end="2026-07-08 14:00", freq="60min", drift=0.8, seed=200) + return {Timeframe.M5: primary, Timeframe.M15: m15, Timeframe.H1: h1} + + +@test("BC detect: quiet fixture → None; injected 8x bar → (its ts, ratio ≥ 3)") +def _(): + agg = _bc_agg(k=3.0) + quiet = make_xau_bars(end="2026-07-08 14:55") + assert agg._detect_big_candle(quiet) is None, \ + "quiet bars must not trip the detector" + spiked = _inject_big_bar(quiet, pos=-250) + hit = agg._detect_big_candle(spiked) + assert hit is not None, "injected burst bar not detected" + ts, ratio = hit + assert ts == spiked.index[-250], f"wrong offender bar: {ts}" + assert ratio >= 3.0, f"ratio {ratio} below threshold" + + +@test("BC detect: burst bar as LAST row still trips (prior-ATR shift)") +def _(): + # The huge bar inflates the concurrent Wilder ATR; the shift(1) + # denominator keeps the very newest burst detectable. + agg = _bc_agg(k=3.0) + spiked = _inject_big_bar(make_xau_bars(end="2026-07-08 14:55"), pos=-1) + hit = agg._detect_big_candle(spiked) + assert hit is not None and hit[0] == spiked.index[-1], f"last-row burst missed: {hit}" + + +@test("BC detect: bar older than the lookback window ignored; short df safe") +def _(): + agg = _bc_agg(k=3.0) + # M5 window = 288 bars (+42 ewm warm-up tail). A burst at -350 is + # outside even the warm-up slice of a 400-bar df. + old_spike = _inject_big_bar(make_xau_bars(n=400, end="2026-07-08 14:55"), pos=-350) + assert agg._detect_big_candle(old_spike) is None, \ + "bar outside the lookback window must not block" + # Short history: no raise, graceful no-trip on quiet bars. + assert agg._detect_big_candle(make_xau_bars(n=60, end="2026-07-08 14:55")) is None + + +@test("BC gate 6h: actionable A-grade → warning + D + setup killed") +def _(): + dfs = _bc_actionable_dfs() + control = _bc_agg(k=0.0).analyze_from_dataframes(XAUUSD, dfs, use_news=False) + assert control.signal_quality == "A" and control.trade_setup is not None, \ + f"fixture drifted: expected actionable A, got {control.signal_quality}" + assert not any("BIG CANDLE" in w for w in control.warnings), \ + "disabled gate must stay silent" + gated = _bc_agg(k=3.0).analyze_from_dataframes(XAUUSD, dfs, use_news=False) + assert any(w.startswith("BIG CANDLE FILTER") for w in gated.warnings), \ + f"gate did not fire: {gated.warnings}" + assert gated.signal_quality == "D", f"expected D, got {gated.signal_quality}" + assert gated.trade_setup is None, "setup must be killed by the gate" + + +@test("BC gate 6h: session-blocked F stays F (warning only, no F→D)") +def _(): + end = "2026-07-08 16:55" # blocked hour → gate 6b sets F + dfs = { + Timeframe.M5: _inject_big_bar(make_xau_bars(end=end)), + Timeframe.M15: make_xau_bars(end="2026-07-08 16:45", freq="15min"), + Timeframe.H1: make_xau_bars(end="2026-07-08 16:00", freq="60min"), + } + report = _bc_agg(k=3.0).analyze_from_dataframes(XAUUSD, dfs, use_news=False) + assert any("SESSION FILTER" in w for w in report.warnings) + assert any(w.startswith("BIG CANDLE FILTER") for w in report.warnings), \ + "detection must run regardless of grade (PB/CP suppression channel)" + assert report.signal_quality == "F", \ + f"blocked bar must stay F, got {report.signal_quality}" + + +@test("BC gate 6h: XAU_PB and XAU_CP entry paths suppressed") +def _(): + # The D-downgrade alone would NOT stop them: PB accepts D (check is + # != 'F') and CP even F — the warning is the suppression channel. + dfs = _bc_actionable_dfs() + + def run(k): + agg = _bc_agg(k=k) + agg.config.xau_strategy.cp_enabled = True + calls = [] + agg._try_m5_pullback = lambda *a, **kw: calls.append("pb") + agg._try_chart_pattern = lambda *a, **kw: calls.append("cp") + # Keep the setup slot empty so steps 9/10 are actually reached. + agg._generate_trade_setup = lambda *a, **kw: None + agg._generate_xau_atr_trade_setup = lambda *a, **kw: None + agg.analyze_from_dataframes(XAUUSD, dfs, use_news=False) + return calls + + assert run(k=0.0) == ["pb", "cp"], "control must reach both entry paths" + assert run(k=3.0) == [], "tripped gate must suppress both entry paths" + + +@test("BC gate: disabled by default in every factory TF config") +def _(): + from pair_signals.xauusd import make_config as xau_make_config + for tf in (Timeframe.M5, Timeframe.M15, Timeframe.H1, Timeframe.H4): + cfg = xau_make_config(tf) + assert cfg.xau_strategy.big_candle_atr_multiple == 0.0, \ + f"{tf}: gate must ship default-OFF (validate-first rollout)" + assert cfg.xau_strategy.big_candle_lookback_hours == 24.0 + + +@test("BC backtester: --big-candle / --big-candle-hours parse, default None") +def _(): + from gpu_backtest import _parse_args + args = _parse_args(["--pair", "XAUUSD", + "--big-candle", "3.0", "--big-candle-hours", "12"]) + assert args.big_candle == 3.0 and args.big_candle_hours == 12.0 + args2 = _parse_args(["--pair", "XAUUSD"]) + assert args2.big_candle is None and args2.big_candle_hours is None + + +# ══════════════════════════════════════════════════════════════════════ +# MODULE 18: --set research override + diagnose_filters tag freshness +# ══════════════════════════════════════════════════════════════════════ + +print("\n═══ Module 18: --set override + diagnose tags ═══") + + +@test("--set: nested float/bool/JSON-list overrides apply with typing") +def _(): + from gpu_backtest import _apply_config_overrides + cfg = PlatformConfig() + _apply_config_overrides(cfg, [ + "xau_strategy.m5_min_abs_score=25", + "xau_strategy.m5_require_mtf=false", + 'xau_strategy.sell_blocked_hours_utc=[15,17]', + "signals.min_tp_cost_multiple=20.5", + ]) + assert cfg.xau_strategy.m5_min_abs_score == 25 + assert cfg.xau_strategy.m5_require_mtf is False + assert cfg.xau_strategy.sell_blocked_hours_utc == [15, 17] + assert cfg.signals.min_tp_cost_multiple == 20.5 + + +@test("--set: unknown path / section leaf / missing '=' all fail loudly") +def _(): + # A typo'd path MUST NOT become a silent no-op run: dataclass + # instances happily accept setattr of any name. + from gpu_backtest import _apply_config_overrides + cfg = PlatformConfig() + for bad in ("xau_strategy.m5_min_abs_scroe=25", # typo leaf + "xau_stragety.m5_min_abs_score=25", # typo section + "xau_strategy=5", # section, not leaf + "no_equals_sign"): + try: + _apply_config_overrides(cfg, [bad]) + except SystemExit: + pass + else: + raise AssertionError(f"'{bad}' must raise SystemExit") + + +@test("--set: bare-string fallback for non-JSON values") +def _(): + from gpu_backtest import _apply_config_overrides + cfg = PlatformConfig() + _apply_config_overrides(cfg, ["technical.rsi_mode=classic"]) + assert cfg.technical.rsi_mode == "classic" + + +@test("--set: CLI flag parses as repeatable list, default empty") +def _(): + from gpu_backtest import _parse_args + args = _parse_args(["--pair", "XAUUSD", + "--set", "xau_strategy.m5_min_abs_score=25", + "--set", "signals.tp_floor_rr=1.0"]) + assert args.set == ["xau_strategy.m5_min_abs_score=25", + "signals.tp_floor_rr=1.0"] + assert _parse_args(["--pair", "XAUUSD"]).set == [] + + +@test("diagnose_filters: every filter tag has a live emitter (staleness guard)") +def _(): + import inspect + import diagnose_filters + import signal_aggregator as sa + src = inspect.getsource(sa) + tag_src = inspect.getsource(diagnose_filters.classify_bar) + # Extract the tag strings from the ordered list in classify_bar. + import re + tags = re.findall(r'\("([^"]+)",\s+"[A-Z0-9_]+"\)', tag_src) + assert len(tags) >= 25, f"tag extraction broke: found {len(tags)}" + for tag in tags: + assert tag in src, f"filter tag '{tag}' has no emitter in signal_aggregator.py" + + +@test("diagnose_filters: one tag per warning, specific prefix wins") +def _(): + import diagnose_filters as d + from signal_aggregator import SignalReport + + class _R: + pass + r = _R() + r.warnings = [ + "M5 FILTER: SELL quality A capped — high-conviction M5 shorts enter at extremes", + "SELL FILTER: 15:00 UTC entry blocked for shorts", + "TREND EXHAUSTION — price extended >3.5 ATR past swing high", + ] + r.final_signal = SignalStrength.SELL + r.signal_quality = "D" + r.trade_setup = None + outcome, hits = d.classify_bar(r, True) + assert outcome == "LOW_QUALITY" + assert hits == ["M5_SELL_QUALITY", "SELL_HOUR", "TREND_EXHAUSTION"], hits + + # ══════════════════════════════════════════════════════════════════════ # SUMMARY # ══════════════════════════════════════════════════════════════════════