Big-candle gate (default-OFF) + M5 relaxation campaign + --set tooling
Gate 6h (big-candle entry gate): blocks all XAU entries (core, XAU_PB, XAU_CP) when the primary TF's last 24h contains a bar with range >= K x prior-bar ATR. Ships default-OFF: validation rejected every threshold (K 3-6 on M5/M15 12mo) — removed trades were net-POSITIVE at every cell; M5 quiet-window trades are the only losing bucket. Config knobs big_candle_atr_multiple/lookback_hours, --big-candle backtest flag, Module 17 tests. M5 relaxation SHIPPED (validated 2026-07-30, Oanda bars, $0.30 spread): m5_min_abs_score 30->25, m5_min_confirmation_score 10->5, hour-21 UTC unblocked M5-only. Beats control in ALL 6 calendar years of the 5yr replay (+230.9R/2319 PF 1.26 DD 21.3 vs +168.0R/2048 PF 1.21 DD 27.0), 12mo +107.4R PF 1.36 vs +84.6R PF 1.33, OOS +26.1R vs +17.2R, $0.70 spread +86.9R vs +66.4R. M15: every relaxation retested negative — config unchanged. Declined-but-open: M5 hour-22 block, M15 hour-16 unblock (both fail 5yr/OOS). Evidence in the factory docstring. Tooling: generic --set PATH=JSON research override in gpu_backtest (typo-proof, logged); diagnose_filters filter_tags refreshed for gates 6d-6i with a staleness-guard test (Module 18); research scripts hour_analysis/burst_profile/book_diff. Suite: 268 tests. Claude-Session: https://claude.ai/code/session_014jqHpmNYmwYMowF6bPZyeD
This commit is contained in:
Harwinder Sekhon
@@ -79,34 +79,54 @@ def classify_bar(
|
||||
"""
|
||||
filters_hit: List[str] = []
|
||||
|
||||
# Scan warnings for known filter tags
|
||||
filter_tags = {
|
||||
"H4 RSI GATE": "H4_RSI_GATE",
|
||||
"H1 MACD GATE": "H1_MACD_GATE",
|
||||
"RSI ZONE GATE": "RSI_ZONE_GATE",
|
||||
"XAU BUY FILTER": "XAU_BUY_FILTER",
|
||||
"SESSION FILTER": "SESSION_FILTER",
|
||||
"DAY FILTER": "DAY_FILTER",
|
||||
"LOW ATR FILTER": "LOW_ATR_FILTER",
|
||||
"SELL SUPPRESSED": "SELL_SUPPRESSED",
|
||||
"EXHAUSTION": "EXHAUSTION_PENALTY",
|
||||
"TREND EXHAUSTION": "TREND_EXHAUSTION",
|
||||
"NO PULLBACK": "NO_PULLBACK_PENALTY",
|
||||
"Weak pullback": "WEAK_PULLBACK_PENALTY",
|
||||
"Extended entry": "EXTENDED_ENTRY_PENALTY",
|
||||
"Counter-trend": "COUNTER_TREND_PENALTY",
|
||||
"EXTREME volatility":"EXTREME_VOL_PENALTY",
|
||||
"NEAR EMA200": "NEAR_EMA200_PENALTY",
|
||||
"FAR FROM EMA200": "FAR_EMA200_PENALTY",
|
||||
"EVENT RISK": "EVENT_RISK",
|
||||
"REGIME FILTER": "REGIME_FILTER",
|
||||
"Timeframes CONFLICT": "MTF_CONFLICT",
|
||||
}
|
||||
# Known filter tags, matched against the exact warning strings the
|
||||
# aggregator emits (refreshed 2026-07-30 for gates 6d-6i). ORDER
|
||||
# MATTERS: most-specific prefix first, and each warning maps to at
|
||||
# most ONE tag (break on first hit) — substring families like
|
||||
# "M5 FILTER: SELL quality" vs "M5 FILTER" and "TREND EXHAUSTION"
|
||||
# vs "EXHAUSTION" would otherwise double-count.
|
||||
filter_tags = [
|
||||
# M5 noise gates (6i, _apply_m5_filters)
|
||||
("M5 FILTER: |score|", "M5_SCORE"),
|
||||
("M5 FILTER: no MTF agreement", "M5_MTF"),
|
||||
("M5 FILTER: no confirmation TF", "M5_CONF"),
|
||||
("M5 FILTER: SELL quality", "M5_SELL_QUALITY"),
|
||||
("M5 FILTER: SELL blocked — RSI", "M5_SELL_RSI"),
|
||||
("M5 FILTER: '", "M5_VOL"),
|
||||
("M5 FILTER", "M5_OTHER"),
|
||||
# XAU SELL gates (6e/6g/6g2/6g3)
|
||||
("SELL FILTER: quality", "SELL_QUALITY"),
|
||||
("SELL FILTER: volatility", "SELL_VOL"),
|
||||
("SELL FILTER: regime", "SELL_REGIME"),
|
||||
("UTC entry blocked for shorts", "SELL_HOUR"),
|
||||
# Hard gates
|
||||
("MTF OPPOSITION", "MTF_OPPOSITION"),
|
||||
("EXHAUSTED TREND FILTER", "EXHAUSTED_TREND"),
|
||||
("BIG CANDLE FILTER", "BIG_CANDLE"),
|
||||
("WEEKEND FILTER", "WEEKEND_FILTER"),
|
||||
("SESSION FILTER", "SESSION_FILTER"),
|
||||
("DAY FILTER", "DAY_FILTER"),
|
||||
("LOW ATR FILTER", "LOW_ATR_FILTER"),
|
||||
("REGIME FILTER", "REGIME_FILTER"),
|
||||
# Soft penalties / context warnings
|
||||
("TREND EXHAUSTION", "TREND_EXHAUSTION"),
|
||||
("EXHAUSTION", "EXHAUSTION_PENALTY"),
|
||||
("NO PULLBACK", "NO_PULLBACK_PENALTY"),
|
||||
("Weak pullback", "WEAK_PULLBACK_PENALTY"),
|
||||
("Extended — RSI", "RSI_EXTENDED_PENALTY"),
|
||||
("Counter-trend", "COUNTER_TREND_PENALTY"),
|
||||
("EXTREME volatility", "EXTREME_VOL_PENALTY"),
|
||||
("NEAR EMA200", "NEAR_EMA200_PENALTY"),
|
||||
("FAR FROM EMA200", "FAR_EMA200_PENALTY"),
|
||||
("EVENT RISK", "EVENT_RISK"),
|
||||
("Timeframes CONFLICT", "MTF_CONFLICT"),
|
||||
]
|
||||
|
||||
for w in report.warnings:
|
||||
for tag, name in filter_tags.items():
|
||||
for tag, name in filter_tags:
|
||||
if tag in w:
|
||||
filters_hit.append(name)
|
||||
break
|
||||
|
||||
# Determine outcome
|
||||
non_neutral = report.final_signal not in (SignalStrength.NEUTRAL,)
|
||||
|
||||
@@ -30,6 +30,7 @@ Usage:
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import multiprocessing as mp
|
||||
import os
|
||||
@@ -2117,6 +2118,19 @@ def _parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace:
|
||||
p.add_argument("--cp-patterns", default=None,
|
||||
help="Comma-separated pattern subset for --cp (e.g. "
|
||||
"double_bottom,bull_flag). Default: all detectors.")
|
||||
p.add_argument("--big-candle", type=float, default=None,
|
||||
help="Force xau_strategy.big_candle_atr_multiple=K for this "
|
||||
"run (no entries while the lookback window contains a "
|
||||
"bar with range >= K x prior ATR) — for validating the "
|
||||
"candidate.")
|
||||
p.add_argument("--big-candle-hours", type=float, default=None,
|
||||
help="Override big_candle_lookback_hours for --big-candle "
|
||||
"runs (default 24).")
|
||||
p.add_argument("--set", action="append", default=[], metavar="PATH=JSON",
|
||||
help="Override any PlatformConfig leaf by dotted path; the "
|
||||
"value is JSON-parsed for typing (research knob; "
|
||||
"repeatable). e.g. --set xau_strategy.m5_min_abs_score=25 "
|
||||
"--set 'xau_strategy.sell_min_quality=[\"A+\",\"A\",\"B\",\"C\"]'")
|
||||
p.add_argument("--no-cache", action="store_true", help="Skip parquet cache (force API fetch)")
|
||||
p.add_argument("--spread", type=float, default=None,
|
||||
help="Round-trip cost in price units per trade (default: pair's "
|
||||
@@ -2148,6 +2162,39 @@ def _resolve_tf(s: str) -> Timeframe:
|
||||
raise SystemExit(f"Unknown timeframe '{s}'. Valid: {[t.value for t in Timeframe]}")
|
||||
|
||||
|
||||
def _apply_config_overrides(cfg: PlatformConfig, overrides: List[str]) -> None:
|
||||
"""Apply repeatable --set PATH=JSON research overrides onto the config.
|
||||
|
||||
Unknown paths are a hard error: dataclass instances silently accept
|
||||
setattr of any name, so a typo'd path would otherwise produce a
|
||||
no-op run that looks like 'the relaxation had no effect'. Every
|
||||
applied override is logged — the run log is the config audit trail
|
||||
required by instructions_for_backtest.txt.
|
||||
"""
|
||||
for item in overrides:
|
||||
path, sep, raw = item.partition("=")
|
||||
if not sep:
|
||||
raise SystemExit(f"--set expects PATH=VALUE, got '{item}'")
|
||||
try:
|
||||
value = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
value = raw # bare-string convenience ("oanda" etc.)
|
||||
obj = cfg
|
||||
parts = path.strip().split(".")
|
||||
for attr in parts[:-1]:
|
||||
if not hasattr(obj, attr):
|
||||
raise SystemExit(f"--set: unknown config path '{path}'")
|
||||
obj = getattr(obj, attr)
|
||||
leaf = parts[-1]
|
||||
if not hasattr(obj, leaf):
|
||||
raise SystemExit(f"--set: unknown config path '{path}'")
|
||||
old = getattr(obj, leaf)
|
||||
if hasattr(old, "__dataclass_fields__"):
|
||||
raise SystemExit(f"--set: '{path}' is a config section, not a leaf")
|
||||
setattr(obj, leaf, value)
|
||||
logger.info(f"--set override: {path} = {value!r} (was {old!r})")
|
||||
|
||||
|
||||
def main(argv: Optional[List[str]] = None) -> int:
|
||||
args = _parse_args(argv)
|
||||
|
||||
@@ -2197,6 +2244,12 @@ def main(argv: Optional[List[str]] = None) -> int:
|
||||
platform_cfg.xau_strategy.cp_patterns = [
|
||||
s.strip() for s in args.cp_patterns.split(",") if s.strip()
|
||||
]
|
||||
if args.big_candle is not None:
|
||||
platform_cfg.xau_strategy.big_candle_atr_multiple = args.big_candle
|
||||
if args.big_candle_hours is not None:
|
||||
platform_cfg.xau_strategy.big_candle_lookback_hours = args.big_candle_hours
|
||||
if args.set:
|
||||
_apply_config_overrides(platform_cfg, args.set)
|
||||
confirm_tfs = list(platform_cfg.confirmation_timeframes)
|
||||
|
||||
end = (
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Trade-book diff: relaxed/gated run vs its control.
|
||||
|
||||
Usage:
|
||||
python backtest/research/book_diff.py CONTROL.csv CANDIDATE.csv BAR_MINUTES
|
||||
|
||||
Decision numbers are RUN-LEVEL (ΔtotalR, ΔPF, ΔmaxDD, Δtrades) — never
|
||||
the added-trade bucket alone: the 5-bar entry cooldown means new
|
||||
entries displace existing ones. The added/removed decomposition and the
|
||||
per-decision-hour histogram of added trades are diagnostics.
|
||||
|
||||
Shifted trades = removed+added pairs in the same direction within
|
||||
±5 bars (cooldown displacement, not a genuinely new/lost signal).
|
||||
"""
|
||||
import sys
|
||||
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def _metrics(df):
|
||||
r = df["r_multiple"]
|
||||
gains, losses = r[r > 0].sum(), -r[r <= 0].sum()
|
||||
cum = r.cumsum()
|
||||
return dict(trades=len(df), total_r=r.sum(),
|
||||
pf=gains / max(losses, 1e-9),
|
||||
wr=(r > 0).mean() * 100 if len(df) else 0.0,
|
||||
max_dd=(cum.cummax() - cum).max() if len(cum) else 0.0)
|
||||
|
||||
|
||||
def diff_books(control_csv, candidate_csv, bar_minutes, split="2026-01-29"):
|
||||
ctrl, cand = pd.read_csv(control_csv), pd.read_csv(candidate_csv)
|
||||
for df in (ctrl, cand):
|
||||
df["_t"] = pd.to_datetime(df["entry_time"], utc=True, format="mixed")
|
||||
df["_key"] = df["entry_time"].astype(str) + "|" + df["direction"]
|
||||
|
||||
added = cand[~cand["_key"].isin(ctrl["_key"])].copy()
|
||||
removed = ctrl[~ctrl["_key"].isin(cand["_key"])].copy()
|
||||
|
||||
# cooldown displacement: same-direction removed/added within ±5 bars
|
||||
tol = pd.Timedelta(minutes=5 * bar_minutes)
|
||||
shifted = 0
|
||||
rem_pool = removed.copy()
|
||||
for _, a in added.iterrows():
|
||||
m = rem_pool[(rem_pool["direction"] == a["direction"])
|
||||
& ((rem_pool["_t"] - a["_t"]).abs() <= tol)]
|
||||
if len(m):
|
||||
shifted += 1
|
||||
rem_pool = rem_pool.drop(m.index[0])
|
||||
|
||||
mc, mk = _metrics(ctrl), _metrics(cand)
|
||||
split_ts = pd.Timestamp(split, tz="UTC")
|
||||
halves = (cand[cand["_t"] < split_ts]["r_multiple"].sum(),
|
||||
cand[cand["_t"] >= split_ts]["r_multiple"].sum())
|
||||
hour_hist = None
|
||||
if len(added):
|
||||
added["dec_hour"] = (added["_t"]
|
||||
- pd.Timedelta(minutes=bar_minutes)).dt.hour
|
||||
hour_hist = added.groupby("dec_hour")["r_multiple"] \
|
||||
.agg(["count", "sum"]).round(2)
|
||||
return dict(
|
||||
control=mc, candidate=mk,
|
||||
delta_r=mk["total_r"] - mc["total_r"],
|
||||
delta_pf=mk["pf"] - mc["pf"],
|
||||
delta_dd=mk["max_dd"] - mc["max_dd"],
|
||||
delta_trades=mk["trades"] - mc["trades"],
|
||||
added_n=len(added), added_r=added["r_multiple"].sum(),
|
||||
removed_n=len(removed), removed_r=removed["r_multiple"].sum(),
|
||||
shifted=shifted, halves=halves, added_by_hour=hour_hist,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
d = diff_books(sys.argv[1], sys.argv[2], int(sys.argv[3]))
|
||||
c, k = d["control"], d["candidate"]
|
||||
print(f"control : {c['trades']:4d} trades {c['total_r']:+7.1f}R "
|
||||
f"PF {c['pf']:.2f} WR {c['wr']:.1f}% DD {c['max_dd']:.1f}")
|
||||
print(f"candidate: {k['trades']:4d} trades {k['total_r']:+7.1f}R "
|
||||
f"PF {k['pf']:.2f} WR {k['wr']:.1f}% DD {k['max_dd']:.1f}")
|
||||
print(f"ΔR {d['delta_r']:+.1f} ΔPF {d['delta_pf']:+.2f} "
|
||||
f"ΔDD {d['delta_dd']:+.1f} Δtrades {d['delta_trades']:+d}")
|
||||
print(f"added {d['added_n']} ({d['added_r']:+.1f}R) "
|
||||
f"removed {d['removed_n']} ({d['removed_r']:+.1f}R) "
|
||||
f"shifted≈{d['shifted']}")
|
||||
print(f"candidate halves: H1 {d['halves'][0]:+.1f}R H2 {d['halves'][1]:+.1f}R")
|
||||
if d["added_by_hour"] is not None:
|
||||
print("\nadded trades by decision hour:")
|
||||
print(d["added_by_hour"].to_string())
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Burst-candle frequency by UTC hour — the empirical "hedge fund hours".
|
||||
|
||||
Usage:
|
||||
python backtest/research/burst_profile.py PARQUET_GLOB
|
||||
|
||||
Per UTC bar-close hour of the given bars: % of bars whose range is
|
||||
>= K x the PRIOR bar's Wilder ATR(14), for K in {3,4,5}; mean bar
|
||||
range; and mean signed drift over the 6 bars following a >=4x burst
|
||||
(does price continue or snap back?). Analysis tooling only.
|
||||
"""
|
||||
import glob
|
||||
import sys
|
||||
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def burst_table(parquet_glob: str, bar_minutes: int = 5):
|
||||
files = sorted(glob.glob(parquet_glob))
|
||||
if not files:
|
||||
raise SystemExit(f"no parquet matches {parquet_glob}")
|
||||
df = pd.concat([pd.read_parquet(f) for f in files]).sort_index()
|
||||
df = df[~df.index.duplicated(keep="last")]
|
||||
h, l, c = df["High"], df["Low"], df["Close"]
|
||||
tr = pd.concat([h - l, (h - c.shift()).abs(), (l - c.shift()).abs()],
|
||||
axis=1).max(axis=1)
|
||||
atr = tr.ewm(alpha=1 / 14, min_periods=14, adjust=False).mean()
|
||||
ratio = (h - l) / atr.shift(1)
|
||||
close_hour = (df.index + pd.Timedelta(minutes=bar_minutes)).hour
|
||||
fwd6 = c.shift(-6) - c # signed move over the 6 bars after this one
|
||||
out = pd.DataFrame({
|
||||
"bars": ratio.groupby(close_hour).size(),
|
||||
**{f"pct_ge_{k}x": (ratio >= k).groupby(close_hour).mean() * 100
|
||||
for k in (3, 4, 5)},
|
||||
"mean_range": (h - l).groupby(close_hour).mean(),
|
||||
"post4x_drift6": fwd6[ratio >= 4].groupby(close_hour[ratio >= 4]).mean(),
|
||||
})
|
||||
return out.round(3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(burst_table(sys.argv[1],
|
||||
int(sys.argv[2]) if len(sys.argv) > 2 else 5).to_string())
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Per-hour P&L breakdown of a backtest trade book.
|
||||
|
||||
Usage:
|
||||
python backtest/research/hour_analysis.py BOOK.csv BAR_MINUTES
|
||||
|
||||
Reports, per UTC hour (both entry hour and DECISION hour = entry_time
|
||||
minus one bar, the clock blocked_hours_utc evaluates): trade count,
|
||||
total R, mean R, win rate, profit factor. Also a per-(hour, direction)
|
||||
table. Analysis tooling only — never imported by the live code.
|
||||
"""
|
||||
import sys
|
||||
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def hour_tables(book_csv: str, bar_minutes: int):
|
||||
df = pd.read_csv(book_csv)
|
||||
et = pd.to_datetime(df["entry_time"], utc=True, format="mixed")
|
||||
df["entry_hour"] = et.dt.hour
|
||||
df["decision_hour"] = (et - pd.Timedelta(minutes=bar_minutes)).dt.hour
|
||||
|
||||
def table(group_cols):
|
||||
g = df.groupby(group_cols)["r_multiple"]
|
||||
out = g.agg(count="count", total_r="sum", mean_r="mean",
|
||||
wr=lambda r: (r > 0).mean() * 100)
|
||||
out["pf"] = g.apply(
|
||||
lambda r: r[r > 0].sum() / max(-r[r <= 0].sum(), 1e-9))
|
||||
return out.round(2)
|
||||
|
||||
return (table("decision_hour"), table(["decision_hour", "direction"]),
|
||||
table("entry_hour"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
book, mins = sys.argv[1], int(sys.argv[2])
|
||||
by_dec, by_dec_dir, by_ent = hour_tables(book, mins)
|
||||
print(f"\n── {book} · by DECISION hour (bar-close clock) ──")
|
||||
print(by_dec.to_string())
|
||||
print(f"\n── by (decision hour, direction) ──")
|
||||
print(by_dec_dir.to_string())
|
||||
Reference in New Issue
Block a user