Big-candle gate (default-OFF) + M5 relaxation campaign + --set tooling
Gate 6h (big-candle entry gate): blocks all XAU entries (core, XAU_PB, XAU_CP) when the primary TF's last 24h contains a bar with range >= K x prior-bar ATR. Ships default-OFF: validation rejected every threshold (K 3-6 on M5/M15 12mo) — removed trades were net-POSITIVE at every cell; M5 quiet-window trades are the only losing bucket. Config knobs big_candle_atr_multiple/lookback_hours, --big-candle backtest flag, Module 17 tests. M5 relaxation SHIPPED (validated 2026-07-30, Oanda bars, $0.30 spread): m5_min_abs_score 30->25, m5_min_confirmation_score 10->5, hour-21 UTC unblocked M5-only. Beats control in ALL 6 calendar years of the 5yr replay (+230.9R/2319 PF 1.26 DD 21.3 vs +168.0R/2048 PF 1.21 DD 27.0), 12mo +107.4R PF 1.36 vs +84.6R PF 1.33, OOS +26.1R vs +17.2R, $0.70 spread +86.9R vs +66.4R. M15: every relaxation retested negative — config unchanged. Declined-but-open: M5 hour-22 block, M15 hour-16 unblock (both fail 5yr/OOS). Evidence in the factory docstring. Tooling: generic --set PATH=JSON research override in gpu_backtest (typo-proof, logged); diagnose_filters filter_tags refreshed for gates 6d-6i with a staleness-guard test (Module 18); research scripts hour_analysis/burst_profile/book_diff. Suite: 268 tests. Claude-Session: https://claude.ai/code/session_014jqHpmNYmwYMowF6bPZyeD
This commit is contained in:
Harwinder Sekhon
@@ -79,34 +79,54 @@ def classify_bar(
|
||||
"""
|
||||
filters_hit: List[str] = []
|
||||
|
||||
# Scan warnings for known filter tags
|
||||
filter_tags = {
|
||||
"H4 RSI GATE": "H4_RSI_GATE",
|
||||
"H1 MACD GATE": "H1_MACD_GATE",
|
||||
"RSI ZONE GATE": "RSI_ZONE_GATE",
|
||||
"XAU BUY FILTER": "XAU_BUY_FILTER",
|
||||
"SESSION FILTER": "SESSION_FILTER",
|
||||
"DAY FILTER": "DAY_FILTER",
|
||||
"LOW ATR FILTER": "LOW_ATR_FILTER",
|
||||
"SELL SUPPRESSED": "SELL_SUPPRESSED",
|
||||
"EXHAUSTION": "EXHAUSTION_PENALTY",
|
||||
"TREND EXHAUSTION": "TREND_EXHAUSTION",
|
||||
"NO PULLBACK": "NO_PULLBACK_PENALTY",
|
||||
"Weak pullback": "WEAK_PULLBACK_PENALTY",
|
||||
"Extended entry": "EXTENDED_ENTRY_PENALTY",
|
||||
"Counter-trend": "COUNTER_TREND_PENALTY",
|
||||
"EXTREME volatility":"EXTREME_VOL_PENALTY",
|
||||
"NEAR EMA200": "NEAR_EMA200_PENALTY",
|
||||
"FAR FROM EMA200": "FAR_EMA200_PENALTY",
|
||||
"EVENT RISK": "EVENT_RISK",
|
||||
"REGIME FILTER": "REGIME_FILTER",
|
||||
"Timeframes CONFLICT": "MTF_CONFLICT",
|
||||
}
|
||||
# Known filter tags, matched against the exact warning strings the
|
||||
# aggregator emits (refreshed 2026-07-30 for gates 6d-6i). ORDER
|
||||
# MATTERS: most-specific prefix first, and each warning maps to at
|
||||
# most ONE tag (break on first hit) — substring families like
|
||||
# "M5 FILTER: SELL quality" vs "M5 FILTER" and "TREND EXHAUSTION"
|
||||
# vs "EXHAUSTION" would otherwise double-count.
|
||||
filter_tags = [
|
||||
# M5 noise gates (6i, _apply_m5_filters)
|
||||
("M5 FILTER: |score|", "M5_SCORE"),
|
||||
("M5 FILTER: no MTF agreement", "M5_MTF"),
|
||||
("M5 FILTER: no confirmation TF", "M5_CONF"),
|
||||
("M5 FILTER: SELL quality", "M5_SELL_QUALITY"),
|
||||
("M5 FILTER: SELL blocked — RSI", "M5_SELL_RSI"),
|
||||
("M5 FILTER: '", "M5_VOL"),
|
||||
("M5 FILTER", "M5_OTHER"),
|
||||
# XAU SELL gates (6e/6g/6g2/6g3)
|
||||
("SELL FILTER: quality", "SELL_QUALITY"),
|
||||
("SELL FILTER: volatility", "SELL_VOL"),
|
||||
("SELL FILTER: regime", "SELL_REGIME"),
|
||||
("UTC entry blocked for shorts", "SELL_HOUR"),
|
||||
# Hard gates
|
||||
("MTF OPPOSITION", "MTF_OPPOSITION"),
|
||||
("EXHAUSTED TREND FILTER", "EXHAUSTED_TREND"),
|
||||
("BIG CANDLE FILTER", "BIG_CANDLE"),
|
||||
("WEEKEND FILTER", "WEEKEND_FILTER"),
|
||||
("SESSION FILTER", "SESSION_FILTER"),
|
||||
("DAY FILTER", "DAY_FILTER"),
|
||||
("LOW ATR FILTER", "LOW_ATR_FILTER"),
|
||||
("REGIME FILTER", "REGIME_FILTER"),
|
||||
# Soft penalties / context warnings
|
||||
("TREND EXHAUSTION", "TREND_EXHAUSTION"),
|
||||
("EXHAUSTION", "EXHAUSTION_PENALTY"),
|
||||
("NO PULLBACK", "NO_PULLBACK_PENALTY"),
|
||||
("Weak pullback", "WEAK_PULLBACK_PENALTY"),
|
||||
("Extended — RSI", "RSI_EXTENDED_PENALTY"),
|
||||
("Counter-trend", "COUNTER_TREND_PENALTY"),
|
||||
("EXTREME volatility", "EXTREME_VOL_PENALTY"),
|
||||
("NEAR EMA200", "NEAR_EMA200_PENALTY"),
|
||||
("FAR FROM EMA200", "FAR_EMA200_PENALTY"),
|
||||
("EVENT RISK", "EVENT_RISK"),
|
||||
("Timeframes CONFLICT", "MTF_CONFLICT"),
|
||||
]
|
||||
|
||||
for w in report.warnings:
|
||||
for tag, name in filter_tags.items():
|
||||
for tag, name in filter_tags:
|
||||
if tag in w:
|
||||
filters_hit.append(name)
|
||||
break
|
||||
|
||||
# Determine outcome
|
||||
non_neutral = report.final_signal not in (SignalStrength.NEUTRAL,)
|
||||
|
||||
@@ -30,6 +30,7 @@ Usage:
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import multiprocessing as mp
|
||||
import os
|
||||
@@ -2117,6 +2118,19 @@ def _parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace:
|
||||
p.add_argument("--cp-patterns", default=None,
|
||||
help="Comma-separated pattern subset for --cp (e.g. "
|
||||
"double_bottom,bull_flag). Default: all detectors.")
|
||||
p.add_argument("--big-candle", type=float, default=None,
|
||||
help="Force xau_strategy.big_candle_atr_multiple=K for this "
|
||||
"run (no entries while the lookback window contains a "
|
||||
"bar with range >= K x prior ATR) — for validating the "
|
||||
"candidate.")
|
||||
p.add_argument("--big-candle-hours", type=float, default=None,
|
||||
help="Override big_candle_lookback_hours for --big-candle "
|
||||
"runs (default 24).")
|
||||
p.add_argument("--set", action="append", default=[], metavar="PATH=JSON",
|
||||
help="Override any PlatformConfig leaf by dotted path; the "
|
||||
"value is JSON-parsed for typing (research knob; "
|
||||
"repeatable). e.g. --set xau_strategy.m5_min_abs_score=25 "
|
||||
"--set 'xau_strategy.sell_min_quality=[\"A+\",\"A\",\"B\",\"C\"]'")
|
||||
p.add_argument("--no-cache", action="store_true", help="Skip parquet cache (force API fetch)")
|
||||
p.add_argument("--spread", type=float, default=None,
|
||||
help="Round-trip cost in price units per trade (default: pair's "
|
||||
@@ -2148,6 +2162,39 @@ def _resolve_tf(s: str) -> Timeframe:
|
||||
raise SystemExit(f"Unknown timeframe '{s}'. Valid: {[t.value for t in Timeframe]}")
|
||||
|
||||
|
||||
def _apply_config_overrides(cfg: PlatformConfig, overrides: List[str]) -> None:
|
||||
"""Apply repeatable --set PATH=JSON research overrides onto the config.
|
||||
|
||||
Unknown paths are a hard error: dataclass instances silently accept
|
||||
setattr of any name, so a typo'd path would otherwise produce a
|
||||
no-op run that looks like 'the relaxation had no effect'. Every
|
||||
applied override is logged — the run log is the config audit trail
|
||||
required by instructions_for_backtest.txt.
|
||||
"""
|
||||
for item in overrides:
|
||||
path, sep, raw = item.partition("=")
|
||||
if not sep:
|
||||
raise SystemExit(f"--set expects PATH=VALUE, got '{item}'")
|
||||
try:
|
||||
value = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
value = raw # bare-string convenience ("oanda" etc.)
|
||||
obj = cfg
|
||||
parts = path.strip().split(".")
|
||||
for attr in parts[:-1]:
|
||||
if not hasattr(obj, attr):
|
||||
raise SystemExit(f"--set: unknown config path '{path}'")
|
||||
obj = getattr(obj, attr)
|
||||
leaf = parts[-1]
|
||||
if not hasattr(obj, leaf):
|
||||
raise SystemExit(f"--set: unknown config path '{path}'")
|
||||
old = getattr(obj, leaf)
|
||||
if hasattr(old, "__dataclass_fields__"):
|
||||
raise SystemExit(f"--set: '{path}' is a config section, not a leaf")
|
||||
setattr(obj, leaf, value)
|
||||
logger.info(f"--set override: {path} = {value!r} (was {old!r})")
|
||||
|
||||
|
||||
def main(argv: Optional[List[str]] = None) -> int:
|
||||
args = _parse_args(argv)
|
||||
|
||||
@@ -2197,6 +2244,12 @@ def main(argv: Optional[List[str]] = None) -> int:
|
||||
platform_cfg.xau_strategy.cp_patterns = [
|
||||
s.strip() for s in args.cp_patterns.split(",") if s.strip()
|
||||
]
|
||||
if args.big_candle is not None:
|
||||
platform_cfg.xau_strategy.big_candle_atr_multiple = args.big_candle
|
||||
if args.big_candle_hours is not None:
|
||||
platform_cfg.xau_strategy.big_candle_lookback_hours = args.big_candle_hours
|
||||
if args.set:
|
||||
_apply_config_overrides(platform_cfg, args.set)
|
||||
confirm_tfs = list(platform_cfg.confirmation_timeframes)
|
||||
|
||||
end = (
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Trade-book diff: relaxed/gated run vs its control.
|
||||
|
||||
Usage:
|
||||
python backtest/research/book_diff.py CONTROL.csv CANDIDATE.csv BAR_MINUTES
|
||||
|
||||
Decision numbers are RUN-LEVEL (ΔtotalR, ΔPF, ΔmaxDD, Δtrades) — never
|
||||
the added-trade bucket alone: the 5-bar entry cooldown means new
|
||||
entries displace existing ones. The added/removed decomposition and the
|
||||
per-decision-hour histogram of added trades are diagnostics.
|
||||
|
||||
Shifted trades = removed+added pairs in the same direction within
|
||||
±5 bars (cooldown displacement, not a genuinely new/lost signal).
|
||||
"""
|
||||
import sys
|
||||
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def _metrics(df):
|
||||
r = df["r_multiple"]
|
||||
gains, losses = r[r > 0].sum(), -r[r <= 0].sum()
|
||||
cum = r.cumsum()
|
||||
return dict(trades=len(df), total_r=r.sum(),
|
||||
pf=gains / max(losses, 1e-9),
|
||||
wr=(r > 0).mean() * 100 if len(df) else 0.0,
|
||||
max_dd=(cum.cummax() - cum).max() if len(cum) else 0.0)
|
||||
|
||||
|
||||
def diff_books(control_csv, candidate_csv, bar_minutes, split="2026-01-29"):
|
||||
ctrl, cand = pd.read_csv(control_csv), pd.read_csv(candidate_csv)
|
||||
for df in (ctrl, cand):
|
||||
df["_t"] = pd.to_datetime(df["entry_time"], utc=True, format="mixed")
|
||||
df["_key"] = df["entry_time"].astype(str) + "|" + df["direction"]
|
||||
|
||||
added = cand[~cand["_key"].isin(ctrl["_key"])].copy()
|
||||
removed = ctrl[~ctrl["_key"].isin(cand["_key"])].copy()
|
||||
|
||||
# cooldown displacement: same-direction removed/added within ±5 bars
|
||||
tol = pd.Timedelta(minutes=5 * bar_minutes)
|
||||
shifted = 0
|
||||
rem_pool = removed.copy()
|
||||
for _, a in added.iterrows():
|
||||
m = rem_pool[(rem_pool["direction"] == a["direction"])
|
||||
& ((rem_pool["_t"] - a["_t"]).abs() <= tol)]
|
||||
if len(m):
|
||||
shifted += 1
|
||||
rem_pool = rem_pool.drop(m.index[0])
|
||||
|
||||
mc, mk = _metrics(ctrl), _metrics(cand)
|
||||
split_ts = pd.Timestamp(split, tz="UTC")
|
||||
halves = (cand[cand["_t"] < split_ts]["r_multiple"].sum(),
|
||||
cand[cand["_t"] >= split_ts]["r_multiple"].sum())
|
||||
hour_hist = None
|
||||
if len(added):
|
||||
added["dec_hour"] = (added["_t"]
|
||||
- pd.Timedelta(minutes=bar_minutes)).dt.hour
|
||||
hour_hist = added.groupby("dec_hour")["r_multiple"] \
|
||||
.agg(["count", "sum"]).round(2)
|
||||
return dict(
|
||||
control=mc, candidate=mk,
|
||||
delta_r=mk["total_r"] - mc["total_r"],
|
||||
delta_pf=mk["pf"] - mc["pf"],
|
||||
delta_dd=mk["max_dd"] - mc["max_dd"],
|
||||
delta_trades=mk["trades"] - mc["trades"],
|
||||
added_n=len(added), added_r=added["r_multiple"].sum(),
|
||||
removed_n=len(removed), removed_r=removed["r_multiple"].sum(),
|
||||
shifted=shifted, halves=halves, added_by_hour=hour_hist,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
d = diff_books(sys.argv[1], sys.argv[2], int(sys.argv[3]))
|
||||
c, k = d["control"], d["candidate"]
|
||||
print(f"control : {c['trades']:4d} trades {c['total_r']:+7.1f}R "
|
||||
f"PF {c['pf']:.2f} WR {c['wr']:.1f}% DD {c['max_dd']:.1f}")
|
||||
print(f"candidate: {k['trades']:4d} trades {k['total_r']:+7.1f}R "
|
||||
f"PF {k['pf']:.2f} WR {k['wr']:.1f}% DD {k['max_dd']:.1f}")
|
||||
print(f"ΔR {d['delta_r']:+.1f} ΔPF {d['delta_pf']:+.2f} "
|
||||
f"ΔDD {d['delta_dd']:+.1f} Δtrades {d['delta_trades']:+d}")
|
||||
print(f"added {d['added_n']} ({d['added_r']:+.1f}R) "
|
||||
f"removed {d['removed_n']} ({d['removed_r']:+.1f}R) "
|
||||
f"shifted≈{d['shifted']}")
|
||||
print(f"candidate halves: H1 {d['halves'][0]:+.1f}R H2 {d['halves'][1]:+.1f}R")
|
||||
if d["added_by_hour"] is not None:
|
||||
print("\nadded trades by decision hour:")
|
||||
print(d["added_by_hour"].to_string())
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Burst-candle frequency by UTC hour — the empirical "hedge fund hours".
|
||||
|
||||
Usage:
|
||||
python backtest/research/burst_profile.py PARQUET_GLOB
|
||||
|
||||
Per UTC bar-close hour of the given bars: % of bars whose range is
|
||||
>= K x the PRIOR bar's Wilder ATR(14), for K in {3,4,5}; mean bar
|
||||
range; and mean signed drift over the 6 bars following a >=4x burst
|
||||
(does price continue or snap back?). Analysis tooling only.
|
||||
"""
|
||||
import glob
|
||||
import sys
|
||||
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def burst_table(parquet_glob: str, bar_minutes: int = 5):
|
||||
files = sorted(glob.glob(parquet_glob))
|
||||
if not files:
|
||||
raise SystemExit(f"no parquet matches {parquet_glob}")
|
||||
df = pd.concat([pd.read_parquet(f) for f in files]).sort_index()
|
||||
df = df[~df.index.duplicated(keep="last")]
|
||||
h, l, c = df["High"], df["Low"], df["Close"]
|
||||
tr = pd.concat([h - l, (h - c.shift()).abs(), (l - c.shift()).abs()],
|
||||
axis=1).max(axis=1)
|
||||
atr = tr.ewm(alpha=1 / 14, min_periods=14, adjust=False).mean()
|
||||
ratio = (h - l) / atr.shift(1)
|
||||
close_hour = (df.index + pd.Timedelta(minutes=bar_minutes)).hour
|
||||
fwd6 = c.shift(-6) - c # signed move over the 6 bars after this one
|
||||
out = pd.DataFrame({
|
||||
"bars": ratio.groupby(close_hour).size(),
|
||||
**{f"pct_ge_{k}x": (ratio >= k).groupby(close_hour).mean() * 100
|
||||
for k in (3, 4, 5)},
|
||||
"mean_range": (h - l).groupby(close_hour).mean(),
|
||||
"post4x_drift6": fwd6[ratio >= 4].groupby(close_hour[ratio >= 4]).mean(),
|
||||
})
|
||||
return out.round(3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(burst_table(sys.argv[1],
|
||||
int(sys.argv[2]) if len(sys.argv) > 2 else 5).to_string())
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Per-hour P&L breakdown of a backtest trade book.
|
||||
|
||||
Usage:
|
||||
python backtest/research/hour_analysis.py BOOK.csv BAR_MINUTES
|
||||
|
||||
Reports, per UTC hour (both entry hour and DECISION hour = entry_time
|
||||
minus one bar, the clock blocked_hours_utc evaluates): trade count,
|
||||
total R, mean R, win rate, profit factor. Also a per-(hour, direction)
|
||||
table. Analysis tooling only — never imported by the live code.
|
||||
"""
|
||||
import sys
|
||||
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def hour_tables(book_csv: str, bar_minutes: int):
|
||||
df = pd.read_csv(book_csv)
|
||||
et = pd.to_datetime(df["entry_time"], utc=True, format="mixed")
|
||||
df["entry_hour"] = et.dt.hour
|
||||
df["decision_hour"] = (et - pd.Timedelta(minutes=bar_minutes)).dt.hour
|
||||
|
||||
def table(group_cols):
|
||||
g = df.groupby(group_cols)["r_multiple"]
|
||||
out = g.agg(count="count", total_r="sum", mean_r="mean",
|
||||
wr=lambda r: (r > 0).mean() * 100)
|
||||
out["pf"] = g.apply(
|
||||
lambda r: r[r > 0].sum() / max(-r[r <= 0].sum(), 1e-9))
|
||||
return out.round(2)
|
||||
|
||||
return (table("decision_hour"), table(["decision_hour", "direction"]),
|
||||
table("entry_hour"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
book, mins = sys.argv[1], int(sys.argv[2])
|
||||
by_dec, by_dec_dir, by_ent = hour_tables(book, mins)
|
||||
print(f"\n── {book} · by DECISION hour (bar-close clock) ──")
|
||||
print(by_dec.to_string())
|
||||
print(f"\n── by (decision hour, direction) ──")
|
||||
print(by_dec_dir.to_string())
|
||||
@@ -203,9 +203,16 @@ class XauStrategyConfig:
|
||||
# M5 signals are noisy; require stronger confirmation before
|
||||
# generating trade setups. Without these, M5 is breakeven
|
||||
# (+0.01R/trade) with 21% drawdown.
|
||||
# Relaxed 2026-07-30 (30→25, 10→5) in the relaxation campaign:
|
||||
# the combined config (with the M5 hour-21 unblock) beat control
|
||||
# in ALL 6 calendar years of the 5yr Oanda replay (+230.9R vs
|
||||
# +168.0R, PF 1.26 vs 1.21, max DD 21.3 vs 27.0), 12mo +107.4R
|
||||
# vs +84.6R PF 1.36 both halves positive, OOS 2024-06→2025-03
|
||||
# +26.1R vs +17.2R, and at $0.70 spread +86.9R vs +66.4R.
|
||||
# Score floors below 25 are vacuous (grade C requires |score|≥25).
|
||||
m5_require_mtf: bool = True
|
||||
m5_min_abs_score: float = 30.0
|
||||
m5_min_confirmation_score: float = 10.0
|
||||
m5_min_abs_score: float = 25.0
|
||||
m5_min_confirmation_score: float = 5.0
|
||||
# A-grade M5 SELLs enter at the extreme of temporary dips and
|
||||
# lose (-0.017R/trade on 163 trades) — maximum conviction at
|
||||
# the worst entry. Cap SELL quality so A-grade can't trigger
|
||||
@@ -271,6 +278,15 @@ class XauStrategyConfig:
|
||||
cp_flag_max_bars: int = 12
|
||||
cp_flag_retrace: tuple = (0.1, 0.5) # flag depth as fraction of pole
|
||||
|
||||
# ── Big-candle entry gate ──────────────────────────────────────
|
||||
# When the recent window on the scan's own primary TF contains an
|
||||
# abnormally large candle (bar range >= K x the PRIOR bar's ATR —
|
||||
# institutional burst; price may reverse violently), no XAU entry
|
||||
# of ANY kind may fire (core, XAU_PB, XAU_CP). 0.0 = disabled.
|
||||
# Per-TF factories opt in only after the standard validation rule.
|
||||
big_candle_atr_multiple: float = 0.0
|
||||
big_candle_lookback_hours: float = 24.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class FilterConfig:
|
||||
|
||||
+23
-4
@@ -99,6 +99,24 @@ in 5/7 calendar years including winning years (2022 -9.2R). Blocking:
|
||||
12mo book is unchanged (+139.0R vs +139.3R). M15-and-up keep hour 9:
|
||||
it is POSITIVE there (M15 6yr +1.7R, M15 OOS +8.6R), so the block is
|
||||
M5-only, same per-timeframe pattern as the M15 overrides.
|
||||
|
||||
M5 override 2 — relaxation campaign (validated 2026-07-30, Oanda
|
||||
bars, $0.30 spread, trailing exits): UNBLOCK hour 21 UTC on M5 only,
|
||||
combined with the config.py M5 gate relaxations (m5_min_abs_score
|
||||
30->25, m5_min_confirmation_score 10->5). The combined config beat
|
||||
the pre-relaxation control in ALL 6 calendar years of the 5yr replay
|
||||
2021-07-30 -> 2026-07-30 (+230.9R/2319 PF 1.26 DD 21.3 vs
|
||||
+168.0R/2048 PF 1.21 DD 27.0), on 12mo (+107.4R PF 1.36, halves
|
||||
+68.6/+38.8 vs +84.6R PF 1.33), on OOS 2024-06-01 -> 2025-03-30
|
||||
(+26.1R vs +17.2R), and at $0.70 spread (+86.9R vs +66.4R). Hour-21
|
||||
trades alone: +28.0R/22, positive every year — overturning the old
|
||||
block per the full-evidence rule. Hour 21 stays BLOCKED on M15+
|
||||
(12mo M15 unblock tested -0.3R). Declined-but-open from the same
|
||||
campaign: an M5 hour-22 block (+4.8R on 12mo — the burst-heavy
|
||||
"institutional" hour — but -3.2R on 5yr and -1.1R on OOS: regime-
|
||||
specific); M15 hour-16 unblock (+0.8R/+57 trades 12mo but -18.3R
|
||||
DD +34% on 5yr); every SELL-gate/vol/TP-size relaxation retested
|
||||
negative (see memory notes for the full grid).
|
||||
"""
|
||||
|
||||
from typing import Optional, List
|
||||
@@ -126,10 +144,11 @@ def make_config(
|
||||
confirmation_tfs or _CONFIRMATIONS.get(primary_tf, [Timeframe.M15, Timeframe.H4])
|
||||
)
|
||||
if primary_tf == Timeframe.M5:
|
||||
# Block hour 9 UTC on M5 only — see module docstring.
|
||||
config.xau_strategy.blocked_hours_utc = (
|
||||
config.xau_strategy.blocked_hours_utc + [9]
|
||||
)
|
||||
# Block hour 9 UTC on M5 only; UNBLOCK hour 21 on M5 only
|
||||
# (positive every year of the 5yr replay) — see docstring.
|
||||
config.xau_strategy.blocked_hours_utc = [
|
||||
h for h in config.xau_strategy.blocked_hours_utc if h != 21
|
||||
] + [9]
|
||||
# BUY-only chart patterns on M5 — see module docstring.
|
||||
config.xau_strategy.cp_enabled = True
|
||||
config.xau_strategy.cp_patterns = ["inv_hs", "double_bottom", "bull_flag"]
|
||||
|
||||
+67
-2
@@ -306,6 +306,44 @@ class SignalAggregator:
|
||||
return None
|
||||
return float(adx.iloc[-1])
|
||||
|
||||
def _detect_big_candle(self, primary_df: pd.DataFrame) -> Optional[tuple]:
|
||||
"""Largest abnormal candle in the recent primary-TF window.
|
||||
|
||||
Returns (timestamp, ratio) of the biggest bar whose range is
|
||||
>= big_candle_atr_multiple x the PRIOR bar's ATR within the last
|
||||
big_candle_lookback_hours of primary-TF bars, else None. The
|
||||
prior-bar ATR (shift 1) matters: a huge candle inflates the
|
||||
Wilder ATR immediately, so comparing against the concurrent ATR
|
||||
under-detects the very bars this gate exists to catch.
|
||||
|
||||
Pure pandas on the caller-sliced df, so the live path and the
|
||||
backtest replay (which calls analyze_from_dataframes on prefix
|
||||
slices) compute bit-identical answers. The tail slice keeps the
|
||||
per-call cost bounded (~n + 42 rows); min_periods NaNs and short
|
||||
histories degrade to "no trip".
|
||||
"""
|
||||
cfg = self.config.xau_strategy
|
||||
bar_secs = BaseDataProvider.CANDLE_SECONDS[self.config.primary_timeframe]
|
||||
n = max(1, int(cfg.big_candle_lookback_hours * 3600 // bar_secs))
|
||||
period = self.config.technical.atr_period
|
||||
tail = primary_df.iloc[-(n + 3 * period):]
|
||||
high, low, close = tail["High"], tail["Low"], tail["Close"]
|
||||
tr = pd.concat(
|
||||
[high - low,
|
||||
(high - close.shift()).abs(),
|
||||
(low - close.shift()).abs()],
|
||||
axis=1,
|
||||
).max(axis=1)
|
||||
atr = tr.ewm(alpha=1 / period, min_periods=period, adjust=False).mean()
|
||||
ratio = (high - low) / atr.shift(1)
|
||||
window = ratio.iloc[-n:]
|
||||
if window.empty:
|
||||
return None
|
||||
max_ratio = window.max()
|
||||
if pd.isna(max_ratio) or max_ratio < cfg.big_candle_atr_multiple:
|
||||
return None
|
||||
return window.idxmax(), float(max_ratio)
|
||||
|
||||
def _apply_m5_filters(self, report: SignalReport) -> None:
|
||||
"""Extra selectivity for M5 — noisy TF needs stronger confirmation."""
|
||||
if report.signal_quality not in ("A+", "A", "B", "C"):
|
||||
@@ -957,6 +995,30 @@ class SignalAggregator:
|
||||
report.signal_quality = "D"
|
||||
report.trade_setup = None
|
||||
|
||||
# ── 6h. Big-candle entry gate (opt-in via per-TF factories) ──
|
||||
# An abnormally large candle in the recent window = institutional
|
||||
# burst; price may reverse violently, so NO entry of any kind may
|
||||
# fire. Detection runs for ANY grade: the warning is the
|
||||
# suppression channel for the XAU_PB path (which accepts D) and
|
||||
# the XAU_CP path (which accepts even F). The D-downgrade itself
|
||||
# touches actionable grades only — F stays F (same convention as
|
||||
# 6e).
|
||||
if (self.config.xau_strategy.enabled
|
||||
and report.display_name == "XAU/USD"
|
||||
and self.config.xau_strategy.big_candle_atr_multiple > 0):
|
||||
big_candle = self._detect_big_candle(primary_df)
|
||||
if big_candle is not None:
|
||||
bc_ts, bc_ratio = big_candle
|
||||
report.warnings.append(
|
||||
f"BIG CANDLE FILTER: {bc_ratio:.1f}x ATR bar at {bc_ts} "
|
||||
f"within "
|
||||
f"{self.config.xau_strategy.big_candle_lookback_hours:g}h "
|
||||
f"— no entries"
|
||||
)
|
||||
if report.signal_quality in ("A+", "A", "B", "C"):
|
||||
report.signal_quality = "D"
|
||||
report.trade_setup = None
|
||||
|
||||
# ── 6i. M5 noise gate (gold-tuned thresholds — XAU only) ──
|
||||
if (self.config.primary_timeframe == Timeframe.M5
|
||||
and report.display_name == "XAU/USD"):
|
||||
@@ -985,7 +1047,9 @@ class SignalAggregator:
|
||||
and report.display_name == "XAU/USD"
|
||||
and self.config.xau_strategy.m5_pullback_enabled
|
||||
and report.trade_setup is None
|
||||
and report.signal_quality != "F"):
|
||||
and report.signal_quality != "F"
|
||||
and not any(w.startswith("BIG CANDLE FILTER")
|
||||
for w in report.warnings)):
|
||||
self._try_m5_pullback(report, primary_df, pip_value)
|
||||
|
||||
# ── 10. Chart-pattern entry (XAU_CP) ────────────────────────
|
||||
@@ -1002,7 +1066,8 @@ class SignalAggregator:
|
||||
and self.config.xau_strategy.cp_enabled
|
||||
and report.trade_setup is None
|
||||
and not any(w.startswith(("SESSION FILTER", "DAY FILTER",
|
||||
"WEEKEND FILTER"))
|
||||
"WEEKEND FILTER",
|
||||
"BIG CANDLE FILTER"))
|
||||
for w in report.warnings)):
|
||||
self._try_chart_pattern(report, primary_df, pip_value)
|
||||
|
||||
|
||||
+248
-4
@@ -2000,10 +2000,11 @@ def _():
|
||||
inline_cfg.signals.trail_dist_r = 0.75
|
||||
if tf == Timeframe.M5:
|
||||
# Documented per-TF overrides (see xauusd.py docstring):
|
||||
# hour 9 UTC blocked + BUY-only chart patterns on M5 only.
|
||||
inline_cfg.xau_strategy.blocked_hours_utc = (
|
||||
inline_cfg.xau_strategy.blocked_hours_utc + [9]
|
||||
)
|
||||
# hour 9 UTC blocked, hour 21 UNBLOCKED (2026-07-30
|
||||
# relaxation campaign) + BUY-only chart patterns on M5 only.
|
||||
inline_cfg.xau_strategy.blocked_hours_utc = [
|
||||
h for h in inline_cfg.xau_strategy.blocked_hours_utc if h != 21
|
||||
] + [9]
|
||||
inline_cfg.xau_strategy.cp_enabled = True
|
||||
inline_cfg.xau_strategy.cp_patterns = ["inv_hs", "double_bottom", "bull_flag"]
|
||||
inline_cfg.signals.trail_arm_r = 0.60
|
||||
@@ -4309,6 +4310,249 @@ def _():
|
||||
assert td != oa, "providers must not share cache files"
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# MODULE 17: Big-candle entry gate (gate 6h)
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
print("\n═══ Module 17: big-candle entry gate ═══")
|
||||
|
||||
|
||||
def _inject_big_bar(df, pos=-250, mult=8.0):
|
||||
"""Copy df and widen one bar to ~mult x the median range (a burst
|
||||
candle) at position `pos` — far enough back not to disturb the
|
||||
near-window indicators (RSI/BB/ATR 14-20 bars, swing S/R 120)."""
|
||||
out = df.copy()
|
||||
atr_est = (out["High"] - out["Low"]).median()
|
||||
ts = out.index[pos]
|
||||
out.loc[ts, "High"] = out.loc[ts, "Open"] + mult * atr_est
|
||||
out.loc[ts, "Low"] = out.loc[ts, "Open"] - 0.5 * atr_est
|
||||
out.loc[ts, "Close"] = out.loc[ts, "Open"] + mult * 0.9 * atr_est
|
||||
return out
|
||||
|
||||
|
||||
def _bc_agg(k=3.0, tf=Timeframe.M5, confirms=(Timeframe.M15, Timeframe.H1)):
|
||||
from pair_signals.xauusd import make_config as xau_make_config
|
||||
cfg = xau_make_config(tf, list(confirms))
|
||||
cfg.xau_strategy.big_candle_atr_multiple = k
|
||||
return SignalAggregator(cfg)
|
||||
|
||||
|
||||
# Deterministic ACTIONABLE fixture: grade A BUY with a GENERIC setup
|
||||
# through the factory M5 config at an unblocked hour (Wed 14:55 UTC),
|
||||
# huge bar pre-injected 250 bars back (verified by seed search).
|
||||
def _bc_actionable_dfs():
|
||||
end = "2026-07-08 14:55"
|
||||
primary = _inject_big_bar(make_xau_bars(end=end, drift=0.8, seed=0))
|
||||
m15 = make_xau_bars(end="2026-07-08 14:45", freq="15min", drift=0.8, seed=100)
|
||||
h1 = make_xau_bars(end="2026-07-08 14:00", freq="60min", drift=0.8, seed=200)
|
||||
return {Timeframe.M5: primary, Timeframe.M15: m15, Timeframe.H1: h1}
|
||||
|
||||
|
||||
@test("BC detect: quiet fixture → None; injected 8x bar → (its ts, ratio ≥ 3)")
|
||||
def _():
|
||||
agg = _bc_agg(k=3.0)
|
||||
quiet = make_xau_bars(end="2026-07-08 14:55")
|
||||
assert agg._detect_big_candle(quiet) is None, \
|
||||
"quiet bars must not trip the detector"
|
||||
spiked = _inject_big_bar(quiet, pos=-250)
|
||||
hit = agg._detect_big_candle(spiked)
|
||||
assert hit is not None, "injected burst bar not detected"
|
||||
ts, ratio = hit
|
||||
assert ts == spiked.index[-250], f"wrong offender bar: {ts}"
|
||||
assert ratio >= 3.0, f"ratio {ratio} below threshold"
|
||||
|
||||
|
||||
@test("BC detect: burst bar as LAST row still trips (prior-ATR shift)")
|
||||
def _():
|
||||
# The huge bar inflates the concurrent Wilder ATR; the shift(1)
|
||||
# denominator keeps the very newest burst detectable.
|
||||
agg = _bc_agg(k=3.0)
|
||||
spiked = _inject_big_bar(make_xau_bars(end="2026-07-08 14:55"), pos=-1)
|
||||
hit = agg._detect_big_candle(spiked)
|
||||
assert hit is not None and hit[0] == spiked.index[-1], f"last-row burst missed: {hit}"
|
||||
|
||||
|
||||
@test("BC detect: bar older than the lookback window ignored; short df safe")
|
||||
def _():
|
||||
agg = _bc_agg(k=3.0)
|
||||
# M5 window = 288 bars (+42 ewm warm-up tail). A burst at -350 is
|
||||
# outside even the warm-up slice of a 400-bar df.
|
||||
old_spike = _inject_big_bar(make_xau_bars(n=400, end="2026-07-08 14:55"), pos=-350)
|
||||
assert agg._detect_big_candle(old_spike) is None, \
|
||||
"bar outside the lookback window must not block"
|
||||
# Short history: no raise, graceful no-trip on quiet bars.
|
||||
assert agg._detect_big_candle(make_xau_bars(n=60, end="2026-07-08 14:55")) is None
|
||||
|
||||
|
||||
@test("BC gate 6h: actionable A-grade → warning + D + setup killed")
|
||||
def _():
|
||||
dfs = _bc_actionable_dfs()
|
||||
control = _bc_agg(k=0.0).analyze_from_dataframes(XAUUSD, dfs, use_news=False)
|
||||
assert control.signal_quality == "A" and control.trade_setup is not None, \
|
||||
f"fixture drifted: expected actionable A, got {control.signal_quality}"
|
||||
assert not any("BIG CANDLE" in w for w in control.warnings), \
|
||||
"disabled gate must stay silent"
|
||||
gated = _bc_agg(k=3.0).analyze_from_dataframes(XAUUSD, dfs, use_news=False)
|
||||
assert any(w.startswith("BIG CANDLE FILTER") for w in gated.warnings), \
|
||||
f"gate did not fire: {gated.warnings}"
|
||||
assert gated.signal_quality == "D", f"expected D, got {gated.signal_quality}"
|
||||
assert gated.trade_setup is None, "setup must be killed by the gate"
|
||||
|
||||
|
||||
@test("BC gate 6h: session-blocked F stays F (warning only, no F→D)")
|
||||
def _():
|
||||
end = "2026-07-08 16:55" # blocked hour → gate 6b sets F
|
||||
dfs = {
|
||||
Timeframe.M5: _inject_big_bar(make_xau_bars(end=end)),
|
||||
Timeframe.M15: make_xau_bars(end="2026-07-08 16:45", freq="15min"),
|
||||
Timeframe.H1: make_xau_bars(end="2026-07-08 16:00", freq="60min"),
|
||||
}
|
||||
report = _bc_agg(k=3.0).analyze_from_dataframes(XAUUSD, dfs, use_news=False)
|
||||
assert any("SESSION FILTER" in w for w in report.warnings)
|
||||
assert any(w.startswith("BIG CANDLE FILTER") for w in report.warnings), \
|
||||
"detection must run regardless of grade (PB/CP suppression channel)"
|
||||
assert report.signal_quality == "F", \
|
||||
f"blocked bar must stay F, got {report.signal_quality}"
|
||||
|
||||
|
||||
@test("BC gate 6h: XAU_PB and XAU_CP entry paths suppressed")
|
||||
def _():
|
||||
# The D-downgrade alone would NOT stop them: PB accepts D (check is
|
||||
# != 'F') and CP even F — the warning is the suppression channel.
|
||||
dfs = _bc_actionable_dfs()
|
||||
|
||||
def run(k):
|
||||
agg = _bc_agg(k=k)
|
||||
agg.config.xau_strategy.cp_enabled = True
|
||||
calls = []
|
||||
agg._try_m5_pullback = lambda *a, **kw: calls.append("pb")
|
||||
agg._try_chart_pattern = lambda *a, **kw: calls.append("cp")
|
||||
# Keep the setup slot empty so steps 9/10 are actually reached.
|
||||
agg._generate_trade_setup = lambda *a, **kw: None
|
||||
agg._generate_xau_atr_trade_setup = lambda *a, **kw: None
|
||||
agg.analyze_from_dataframes(XAUUSD, dfs, use_news=False)
|
||||
return calls
|
||||
|
||||
assert run(k=0.0) == ["pb", "cp"], "control must reach both entry paths"
|
||||
assert run(k=3.0) == [], "tripped gate must suppress both entry paths"
|
||||
|
||||
|
||||
@test("BC gate: disabled by default in every factory TF config")
|
||||
def _():
|
||||
from pair_signals.xauusd import make_config as xau_make_config
|
||||
for tf in (Timeframe.M5, Timeframe.M15, Timeframe.H1, Timeframe.H4):
|
||||
cfg = xau_make_config(tf)
|
||||
assert cfg.xau_strategy.big_candle_atr_multiple == 0.0, \
|
||||
f"{tf}: gate must ship default-OFF (validate-first rollout)"
|
||||
assert cfg.xau_strategy.big_candle_lookback_hours == 24.0
|
||||
|
||||
|
||||
@test("BC backtester: --big-candle / --big-candle-hours parse, default None")
|
||||
def _():
|
||||
from gpu_backtest import _parse_args
|
||||
args = _parse_args(["--pair", "XAUUSD",
|
||||
"--big-candle", "3.0", "--big-candle-hours", "12"])
|
||||
assert args.big_candle == 3.0 and args.big_candle_hours == 12.0
|
||||
args2 = _parse_args(["--pair", "XAUUSD"])
|
||||
assert args2.big_candle is None and args2.big_candle_hours is None
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# MODULE 18: --set research override + diagnose_filters tag freshness
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
print("\n═══ Module 18: --set override + diagnose tags ═══")
|
||||
|
||||
|
||||
@test("--set: nested float/bool/JSON-list overrides apply with typing")
|
||||
def _():
|
||||
from gpu_backtest import _apply_config_overrides
|
||||
cfg = PlatformConfig()
|
||||
_apply_config_overrides(cfg, [
|
||||
"xau_strategy.m5_min_abs_score=25",
|
||||
"xau_strategy.m5_require_mtf=false",
|
||||
'xau_strategy.sell_blocked_hours_utc=[15,17]',
|
||||
"signals.min_tp_cost_multiple=20.5",
|
||||
])
|
||||
assert cfg.xau_strategy.m5_min_abs_score == 25
|
||||
assert cfg.xau_strategy.m5_require_mtf is False
|
||||
assert cfg.xau_strategy.sell_blocked_hours_utc == [15, 17]
|
||||
assert cfg.signals.min_tp_cost_multiple == 20.5
|
||||
|
||||
|
||||
@test("--set: unknown path / section leaf / missing '=' all fail loudly")
|
||||
def _():
|
||||
# A typo'd path MUST NOT become a silent no-op run: dataclass
|
||||
# instances happily accept setattr of any name.
|
||||
from gpu_backtest import _apply_config_overrides
|
||||
cfg = PlatformConfig()
|
||||
for bad in ("xau_strategy.m5_min_abs_scroe=25", # typo leaf
|
||||
"xau_stragety.m5_min_abs_score=25", # typo section
|
||||
"xau_strategy=5", # section, not leaf
|
||||
"no_equals_sign"):
|
||||
try:
|
||||
_apply_config_overrides(cfg, [bad])
|
||||
except SystemExit:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError(f"'{bad}' must raise SystemExit")
|
||||
|
||||
|
||||
@test("--set: bare-string fallback for non-JSON values")
|
||||
def _():
|
||||
from gpu_backtest import _apply_config_overrides
|
||||
cfg = PlatformConfig()
|
||||
_apply_config_overrides(cfg, ["technical.rsi_mode=classic"])
|
||||
assert cfg.technical.rsi_mode == "classic"
|
||||
|
||||
|
||||
@test("--set: CLI flag parses as repeatable list, default empty")
|
||||
def _():
|
||||
from gpu_backtest import _parse_args
|
||||
args = _parse_args(["--pair", "XAUUSD",
|
||||
"--set", "xau_strategy.m5_min_abs_score=25",
|
||||
"--set", "signals.tp_floor_rr=1.0"])
|
||||
assert args.set == ["xau_strategy.m5_min_abs_score=25",
|
||||
"signals.tp_floor_rr=1.0"]
|
||||
assert _parse_args(["--pair", "XAUUSD"]).set == []
|
||||
|
||||
|
||||
@test("diagnose_filters: every filter tag has a live emitter (staleness guard)")
|
||||
def _():
|
||||
import inspect
|
||||
import diagnose_filters
|
||||
import signal_aggregator as sa
|
||||
src = inspect.getsource(sa)
|
||||
tag_src = inspect.getsource(diagnose_filters.classify_bar)
|
||||
# Extract the tag strings from the ordered list in classify_bar.
|
||||
import re
|
||||
tags = re.findall(r'\("([^"]+)",\s+"[A-Z0-9_]+"\)', tag_src)
|
||||
assert len(tags) >= 25, f"tag extraction broke: found {len(tags)}"
|
||||
for tag in tags:
|
||||
assert tag in src, f"filter tag '{tag}' has no emitter in signal_aggregator.py"
|
||||
|
||||
|
||||
@test("diagnose_filters: one tag per warning, specific prefix wins")
|
||||
def _():
|
||||
import diagnose_filters as d
|
||||
from signal_aggregator import SignalReport
|
||||
|
||||
class _R:
|
||||
pass
|
||||
r = _R()
|
||||
r.warnings = [
|
||||
"M5 FILTER: SELL quality A capped — high-conviction M5 shorts enter at extremes",
|
||||
"SELL FILTER: 15:00 UTC entry blocked for shorts",
|
||||
"TREND EXHAUSTION — price extended >3.5 ATR past swing high",
|
||||
]
|
||||
r.final_signal = SignalStrength.SELL
|
||||
r.signal_quality = "D"
|
||||
r.trade_setup = None
|
||||
outcome, hits = d.classify_bar(r, True)
|
||||
assert outcome == "LOW_QUALITY"
|
||||
assert hits == ["M5_SELL_QUALITY", "SELL_HOUR", "TREND_EXHAUSTION"], hits
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# SUMMARY
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
Reference in New Issue
Block a user