From 6bf58fd9d44fa4384df03d0fa1038751b4da56ef Mon Sep 17 00:00:00 2001 From: Harwinder Sekhon Date: Sat, 5 Sep 2026 21:19:24 -0400 Subject: [PATCH] research: 3-year stake-sizing campaign + violent-move studies (nothing changes signals) Walk-forward "second opinion" stake multiplier (meta-labelling on 65 decision-time features, 3yr Oanda books @ $0.65) FAILED the pre-registered rule on M5 (clean null) and M15 (lift inside the label-shuffle null, explained by BUY-vs-SELL). Nothing that changes numbers ships; the 12-month M5 book re-run is byte-identical to the pinned one. Behaviour-neutral plumbing: - position_sizing.build_features (65 features; multiplier path returns 1.0, no artifact) - SignalReport.penalty_tags / penalty_total / momentum_fading (record-keeping) - TradeSetup.stake_multiplier = None (reserved) - gpu_backtest --export-features (default export stays the pinned 63 columns; extras forced to the tail even when the first trade lacks an indicator) Research tools (backtest/research): book_report, burst_map, burst_precursors, open_trade_bursts, calendar_impact, burst_indicator_map, sizing_model, watch_rule_eval. Findings recorded in backtest/research/sizing_campaign_2026-09.md: M5 stretch measures are positive in both halves; every open-trade protection against violent bars loses on both TFs; high-impact USD releases cause 20-31% of violent bars but trades held through them do better; violent bars come when price is already at an extreme, not from calm. Module 29 tests (13). Suite: 435 passed. Claude-Session: https://claude.ai/code/session_01RWV2yLt3YXHHGVYHSQU2EE --- .gitignore | 2 + CLAUDE.md | 1 + backtest/gpu_backtest.py | 66 +- backtest/instructions_for_backtest.txt | 10 + backtest/research/book_report.py | 581 +++++++++++ backtest/research/burst_indicator_map.py | 121 +++ backtest/research/burst_map.py | 445 +++++++++ backtest/research/burst_precursors.py | 127 +++ backtest/research/calendar_impact.py | 111 +++ backtest/research/open_trade_bursts.py | 150 +++ backtest/research/sizing_campaign_2026-09.md | 348 +++++++ backtest/research/sizing_model.py | 961 +++++++++++++++++++ backtest/research/watch_rule_eval.py | 78 ++ position_sizing.py | 622 ++++++++++++ signal_aggregator.py | 17 + test_all.py | 439 +++++++++ 16 files changed, 4076 insertions(+), 3 deletions(-) create mode 100644 backtest/research/book_report.py create mode 100644 backtest/research/burst_indicator_map.py create mode 100644 backtest/research/burst_map.py create mode 100644 backtest/research/burst_precursors.py create mode 100644 backtest/research/calendar_impact.py create mode 100644 backtest/research/open_trade_bursts.py create mode 100644 backtest/research/sizing_campaign_2026-09.md create mode 100644 backtest/research/sizing_model.py create mode 100644 backtest/research/watch_rule_eval.py create mode 100644 position_sizing.py diff --git a/.gitignore b/.gitignore index 625688e..cf2f34d 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,5 @@ backtest/__pycache__/ logs/ backtest/research/commit_ladder_2026-08/ backtest/research/llm_news_eval_2026-08/ + +backtest/research/sizing_2026-09/ diff --git a/CLAUDE.md b/CLAUDE.md index 126208f..9e78309 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -93,4 +93,5 @@ Signal currency everywhere is `SignalStrength` (−2..+2). DataFrames are UTC tz - Tiingo is streaming-only (`data_providers.streaming: "tiingo"`), not a designated candle fallback. Oanda-Tiingo signal parity is documented in `backtest/research/signal_parity_2026-08-14.md`; Phase 4's two-week live shadow scan remains a manual operational step. - Human-facing docs in `docs/` (`user_guide.html`, `beginner_guide.html` + PDF builds) must be kept in sync when signal behavior changes. - 2026-09-03 signal-log campaign shipped two new per-TF gates (`backtest/research/signal_log_campaign_2026-09.md`): M5 `sell_blocked_regimes=["transitioning"]` and H1 `sell_min_quality=[]` (all H1 shorts blocked). Current-HEAD M5 canonical book is now `backtest/oanda_m5_12mo_065_nopb_20260903.csv` (557 / +112.21R / PF 1.56, halves +55.9/+56.4); H1 reference `backtest/oanda_h1_12mo_065_20260903.csv` (91 / +22.2R / PF 1.79; 5yr +49.1R PF 1.46; OOS +16.7R PF 1.74). +- 2026-09-06 stake-sizing campaign (`backtest/research/sizing_campaign_2026-09.md`): a walk-forward "second opinion" stake multiplier (meta-labelling on 65 decision-time features, 3-year Oanda books at $0.65) FAILED the pre-registered rule on M5 (clean null) and M15 (lift inside the label-shuffle null, explained by BUY-vs-SELL). Nothing that changes numbers shipped. Behaviour-neutral tooling remains: `position_sizing.build_features` + `--export-features` (default export stays 63 columns), `SignalReport.penalty_tags/penalty_total/momentum_fading`, `TradeSetup.stake_multiplier` (None; reserved), `backtest/research/{book_report,burst_map,sizing_model}.py`. Stage-B facts: M5 stretch measures are POSITIVE in both halves (do not re-propose exhaustion sizing on M5); M15 exhaustion tag is weakly negative (≈ +3R/yr, noise); bursts cluster at 12–15 UTC and 01 UTC bar-close and do NOT hurt entries. - Auto-execution (oanda_daemon.py / oanda_executor.py) was removed 2026-09-03 — the owner is not auto-trading; last commit containing it is 31002cb. Book-parity rules that lived in its docstring (no broker TP, 5-bar cooldown, once-per-bar trail) remain the backtester's model. diff --git a/backtest/gpu_backtest.py b/backtest/gpu_backtest.py index 213144d..532f743 100644 --- a/backtest/gpu_backtest.py +++ b/backtest/gpu_backtest.py @@ -62,6 +62,7 @@ from config import ( # noqa: E402 from providers import BaseDataProvider, make_data_provider # noqa: E402 from engine import IndicatorSignal, TechnicalAnalysis, TechnicalEngine # noqa: E402 from signal_aggregator import SignalAggregator, SignalReport, TradeSetup # noqa: E402 +import position_sizing # noqa: E402 # ───────────────────────────────────────────────────────────────────── # GPU shim — CuPy if available AND CUDA runtime works, else NumPy. @@ -1074,6 +1075,13 @@ class Trade: indicators_neutral: int = 0 indicators_total: int = 0 # total scored indicators + # RESEARCH (--export-features only): decision-time feature vector + # (position_sizing.FEATURE_ORDER) and the stake multiplier the setup + # carried. Empty / 1.0 on every normal run, so the default book is + # untouched. + features: Dict[str, float] = field(default_factory=dict) + stake_multiplier: float = 1.0 + # Filled on exit: exit_time: Optional[pd.Timestamp] = None exit_price: Optional[float] = None @@ -1454,6 +1462,11 @@ class BacktestConfig: # afterwards in a fast ordered pass. Results are identical to the # sequential path by construction. workers: int = 0 + # RESEARCH: also compute position_sizing.build_features() at each + # entry and append them to --export-csv. Off by default — the extra + # columns would break comparison against the pinned 63-column books, + # and the feature build costs time per candidate. + export_features: bool = False def _is_xau_blocked_by_calendar(ts: pd.Timestamp, cfg: PlatformConfig) -> Optional[str]: @@ -1610,6 +1623,17 @@ def _replay_span(span: Tuple[int, int]): adj_tp1 = entry_price - tp1_distance adj_tp2 = entry_price - tp2_distance + # RESEARCH: decision-time features (same frames the aggregator + # just saw, so live == backtest by construction). Deliberately + # NOT wrapped in try/except — build_features already swallows its + # own failures, so anything escaping here is a real bug. + feats: Dict[str, float] = {} + if bt_cfg.export_features: + feats = position_sizing.build_features( + report, ts_setup, slice_dfs, bt_cfg.primary_tf, + list(bt_cfg.confirmation_tfs), + ) + candidates.append((pos, Trade( pair=bt_cfg.pair, entry_time=entry_time, @@ -1635,6 +1659,10 @@ def _replay_span(span: Tuple[int, int]): indicators_opposing=oppose, indicators_neutral=neutral, indicators_total=total, + features=feats, + stake_multiplier=float(ts_setup.stake_multiplier + if ts_setup.stake_multiplier is not None + else 1.0), trail_arm_r=(platform_cfg.signals.pb_trail_arm_r if tag == "XAU_PB" else 0.0), trail_dist_r=(platform_cfg.signals.pb_trail_dist_r @@ -2157,11 +2185,17 @@ def print_trade_reasoning(trades: List[Trade], limit: int = 5) -> None: print(f" MFE / MAE: {t.mfe_r:.2f}R / {t.mae_r:.2f}R") -def trades_to_dataframe(trades: List[Trade]) -> pd.DataFrame: +def trades_to_dataframe(trades: List[Trade], + include_features: bool = False) -> pd.DataFrame: """Export trades for downstream analysis (e.g. parquet/CSV). Includes per-instructions: entry reasoning, indicator-by-indicator signal/value/agreement, and overall agreement count. + + `include_features` (research, --export-features) appends + stake_multiplier, r_sized and the position_sizing.FEATURE_ORDER + columns AFTER the existing ones. Default off so the column set stays + byte-identical to the pinned books. """ rows = [] for t in trades: @@ -2200,8 +2234,22 @@ def trades_to_dataframe(trades: List[Trade]) -> pd.DataFrame: row[f"ind_{name}_value"] = b["value"] row[f"ind_{name}_conf"] = b["confidence"] row[f"ind_{name}_agrees"] = b["agrees"] + if include_features: + row["stake_multiplier"] = t.stake_multiplier + row["r_sized"] = t.r_multiple * t.stake_multiplier + for name in position_sizing.FEATURE_ORDER: + row[name] = t.features.get(name, float("nan")) rows.append(row) - return pd.DataFrame(rows) + df = pd.DataFrame(rows) + if include_features and len(df.columns): + # A later trade can introduce an indicator column the first trade + # lacked (pandas orders columns by first appearance), which would + # land AFTER the extras. Force the extras to the tail so the default + # export is always a strict prefix of the feature export. + extras = ["stake_multiplier", "r_sized", *position_sizing.FEATURE_ORDER] + base_cols = [c for c in df.columns if c not in extras] + df = df[base_cols + [c for c in extras if c in df.columns]] + return df # ═════════════════════════════════════════════════════════════════════ @@ -2263,6 +2311,11 @@ def _parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace: help="Round-trip cost in price units per trade (default: pair's " "default_spread from PAIR_INFO — XAU $0.30; 0 disables)") p.add_argument("--export-csv", help="Optional path to dump per-trade CSV (incl. indicator breakdown)") + p.add_argument("--export-features", action="store_true", + help="Append the 65 decision-time f_* feature columns " + "(+ stake_multiplier, r_sized) to --export-csv; " + "research flag, default off so pinned 63-column " + "books stay comparable") p.add_argument("--detail", action="store_true", help="Print full per-trade reasoning (indicators, signals, agreement, why entered)") p.add_argument("--workers", type=int, default=0, @@ -2416,7 +2469,14 @@ def main(argv: Optional[List[str]] = None) -> int: spread=spread, workers=args.workers, fixed_rr_tp=args.fixed_rr_tp or 0.0, + export_features=args.export_features, ) + if args.export_features: + logger.info( + f"--export-features ON: appending stake_multiplier, r_sized and " + f"{len(position_sizing.FEATURE_ORDER)} f_* feature columns to the " + f"CSV export (research flag; book itself is unchanged)" + ) logger.info( f"Backend: {'CuPy GPU' if _GPU_AVAILABLE else 'NumPy CPU fallback'} | " @@ -2430,7 +2490,7 @@ def main(argv: Optional[List[str]] = None) -> int: print_trade_reasoning(trades, limit=args.reasoning_samples) if args.export_csv: - df = trades_to_dataframe(trades) + df = trades_to_dataframe(trades, include_features=args.export_features) df.to_csv(args.export_csv, index=False) logger.info(f"Exported {len(df)} trades → {args.export_csv}") diff --git a/backtest/instructions_for_backtest.txt b/backtest/instructions_for_backtest.txt index 6736afd..31f09bc 100644 --- a/backtest/instructions_for_backtest.txt +++ b/backtest/instructions_for_backtest.txt @@ -57,3 +57,13 @@ Win rate over rolling 20-trade windows (check for edge decay) Critical extra "Do not trade" log: setups flagged but skipped, and what happened after + +--- 2026-09-06: decision-time feature export --- +- `--export-features` appends stake_multiplier, r_sized and the 65 `f_*` decision-time + features (position_sizing.FEATURE_ORDER) AFTER the 63 pinned columns; default export is + byte-identical to the pinned books. Use backtest/research/book_report.py for weekday / + hour x weekday / session / streak / rolling-WR / feature-quintile (both halves) tables, + backtest/research/burst_map.py for the burst-by-hour map, and + backtest/research/sizing_model.py for the walk-forward + shuffle-control harness. +- Any sizing report must show the FLAT book and the SIZED book side by side and the + label-shuffle null; the 2026-09-06 campaign verdict (nothing ships) is the reference. diff --git a/backtest/research/book_report.py b/backtest/research/book_report.py new file mode 100644 index 0000000..f240c47 --- /dev/null +++ b/backtest/research/book_report.py @@ -0,0 +1,581 @@ +"""Full descriptive report of a backtest trade book (research only). + +Usage: + python backtest/research/book_report.py --book BOOK.csv --bar-minutes N \ + [--split YYYY-MM-DD|median] [--parquet-4h PATH] [--parquet-1d PATH] \ + [--out DIR] + +Prints (and writes to DIR/report.txt) headline metrics for the whole book +and for each half, group tables (weekday / decision hour / session / quality +/ regime / vol_regime / direction / strategy / exit_reason / indicators_agree), +win-loss streaks, a rolling win-rate summary, and a quantile-bucket screen of +every numeric decision-time feature the book carries (`f_*` / `x_*` columns +plus score, confidence, rr_planned, indicators_agree/oppose and `ind_*_value`). + +The screen is deliberately half-split: a feature only "passes" when the +top-bucket-minus-bottom-bucket mean-R gap has the SAME SIGN in both halves of +the book and both extreme buckets are populated in both halves. Bin edges are +fit on the WHOLE book (so the halves share edges and stay comparable) — that +leaks a little cross-half information into the edges themselves, which is why +the sign test, not the size of the gap, is the thing to trust. + +Everything here is measurement, not simulation: it never re-runs the signal +engine, it only describes a book that gpu_backtest.py already produced. +Analysis tooling only — never imported by the live code. +""" +from __future__ import annotations + +import argparse +import math +import os +from typing import Iterable + +import numpy as np +import pandas as pd + +SESSIONS = ( + ("asian", 3, 6), + ("london", 7, 12), + ("overlap", 13, 16), + ("ny", 17, 22), +) + +# Numeric non-`f_`/`x_` columns that are known at DECISION time and are +# therefore legal screen candidates. bars_held / r_multiple / mfe_r / mae_r +# are post-entry outcomes and must never appear here. +BASE_SCREEN_COLS = ("score", "confidence", "rr_planned", + "indicators_agree", "indicators_oppose") + + +def session_of(hour: int) -> str: + for name, lo, hi in SESSIONS: + if lo <= hour <= hi: + return name + return "off" + + +def feature_columns(df: pd.DataFrame) -> list[str]: + """Decision-time feature columns carried by this book, generically.""" + extra = [c for c in df.columns if c.startswith(("f_", "x_"))] + base = [c for c in BASE_SCREEN_COLS if c in df.columns] + ind = [c for c in df.columns + if c.startswith("ind_") and c.endswith("_value")] + out, seen = [], set() + for c in extra + base + ind: + if c not in seen: + seen.add(c) + out.append(c) + return out + + +# ───────────────────────────── loading ────────────────────────────── + +def load_book(csv: str, bar_minutes: int = 5, + split: str = "median") -> pd.DataFrame: + """Read a gpu_backtest trade book and add the decision-time columns. + + `decision_time` = entry_time − one bar: entries fill at the NEXT bar's + open, so the bar the signal actually decided on closed one bar earlier. + That is the clock blocked_hours_utc and every session filter evaluate. + `half` is 1 before the split instant and 2 at/after it. + """ + df = pd.read_csv(csv) + for col in ("entry_time", "exit_time"): + if col in df.columns: + df[col] = pd.to_datetime(df[col], utc=True, format="mixed") + df["decision_time"] = df["entry_time"] - pd.Timedelta(minutes=bar_minutes) + df["decision_hour"] = df["decision_time"].dt.hour + df["weekday"] = df["decision_time"].dt.weekday + df["session"] = df["decision_hour"].map(session_of) + df = df.sort_values("entry_time", kind="mergesort").reset_index(drop=True) + df["half"] = _halves(df["entry_time"], split) + return df + + +def split_instant(entry_time: pd.Series, split: str) -> pd.Timestamp: + if split == "median": + return entry_time.median() + return pd.Timestamp(split, tz="UTC") + + +def _halves(entry_time: pd.Series, split: str) -> pd.Series: + cut = split_instant(entry_time, split) + return np.where(entry_time < cut, 1, 2) + + +# ───────────────────────────── metrics ────────────────────────────── + +def base_metrics(r: pd.Series, df: pd.DataFrame | None = None) -> dict: + """Headline stats for a series of R multiples. + + max_dd uses the gpu_backtest.compute_metrics convention: a 0.0 is + prepended to the cumulative-R curve so an opening losing streak counts + as drawdown from flat, not from an already-negative first trade. + pf/expectancy split on r > 0 / r < 0 (exact zeros are neither). + """ + r = pd.Series(r, dtype="float64").dropna() + n = int(len(r)) + if n == 0: + return {"n": 0, "total_r": 0.0, "mean_r": 0.0, "median_r": 0.0, + "wr": 0.0, "pf": 0.0, "max_dd": 0.0, "sharpe": 0.0, + "expectancy": 0.0, "expectancy_alt": 0.0, "recovery": 0.0, + "avg_bars": float("nan"), "avg_mfe": float("nan"), + "avg_mae": float("nan")} + wins, losses = r[r > 0], r[r < 0] + gross_win = float(wins.sum()) + gross_loss = float(-losses.sum()) + cum = np.concatenate(([0.0], np.cumsum(r.to_numpy()))) + max_dd = float((np.maximum.accumulate(cum) - cum).max()) + std = float(r.std(ddof=0)) + mean_r = float(r.mean()) + wr = len(wins) / n + lr = len(losses) / n + avg_win = float(wins.mean()) if len(wins) else 0.0 + avg_loss = float(-losses.mean()) if len(losses) else 0.0 + expectancy_alt = wr * avg_win - lr * avg_loss + # Identity check: the textbook expectancy form must equal the plain mean + # whenever no trade is exactly flat (and it does even then, since a 0 R + # trade contributes 0 to both sides). + assert math.isclose(expectancy_alt, mean_r, rel_tol=1e-9, abs_tol=1e-9), ( + f"expectancy mismatch: {expectancy_alt} vs mean {mean_r}") + out = { + "n": n, + "total_r": float(r.sum()), + "mean_r": mean_r, + "median_r": float(r.median()), + "wr": wr * 100.0, + "pf": (gross_win / gross_loss) if gross_loss > 0 else float("inf"), + "max_dd": max_dd, + "sharpe": (mean_r / std) if std > 0 else 0.0, + "expectancy": mean_r, + "expectancy_alt": expectancy_alt, + "recovery": (float(r.sum()) / max_dd) if max_dd > 0 else float("inf"), + "avg_bars": float("nan"), + "avg_mfe": float("nan"), + "avg_mae": float("nan"), + } + if df is not None: + for key, col in (("avg_bars", "bars_held"), ("avg_mfe", "mfe_r"), + ("avg_mae", "mae_r")): + if col in df.columns: + out[key] = float(pd.to_numeric(df[col], + errors="coerce").mean()) + return out + + +def _pf(r: pd.Series) -> float: + """Profit factor, hour_analysis-compatible (loss side is r <= 0).""" + pos = r[r > 0].sum() + neg = -r[r <= 0].sum() + return float(pos / max(neg, 1e-9)) + + +def group_table(df: pd.DataFrame, by, min_n: int = 10) -> pd.DataFrame: + """Per-group P&L with a half-1 / half-2 sign-stability flag.""" + keys = [by] if isinstance(by, str) else list(by) + rows = [] + for key, sub in df.groupby(keys, dropna=False, sort=True): + r = sub["r_multiple"] + h1, h2 = sub[sub["half"] == 1]["r_multiple"], sub[sub["half"] == 2]["r_multiple"] + m1 = float(h1.mean()) if len(h1) else float("nan") + m2 = float(h2.mean()) if len(h2) else float("nan") + stable = bool(len(h1) >= min_n and len(h2) >= min_n + and np.sign(m1) != 0 and np.sign(m1) == np.sign(m2)) + rec = {} + # pandas >= 2.2 hands back a 1-tuple even for a single group key. + vals = key if isinstance(key, tuple) else (key,) + for k, v in zip(keys, vals): + rec[k] = v + rec.update({ + "n": int(len(sub)), + "total_r": round(float(r.sum()), 2), + "mean_r": round(float(r.mean()), 3), + "wr": round(float((r > 0).mean() * 100), 2), + "pf": round(_pf(r), 2), + "n_h1": int(len(h1)), + "total_r_h1": round(float(h1.sum()), 2), + "mean_r_h1": round(m1, 3) if len(h1) else float("nan"), + "n_h2": int(len(h2)), + "total_r_h2": round(float(h2.sum()), 2), + "mean_r_h2": round(m2, 3) if len(h2) else float("nan"), + "sign_stable": stable, + }) + rows.append(rec) + out = pd.DataFrame(rows) + if not out.empty: + out = out.set_index(keys) + return out + + +def streaks(df: pd.DataFrame) -> dict: + """Longest win/loss runs and the longest stretch under water (in trades).""" + d = df.sort_values("entry_time", kind="mergesort") + r = d["r_multiple"].to_numpy(dtype="float64") + best_w = best_l = cur_w = cur_l = 0 + for x in r: + if x > 0: + cur_w, cur_l = cur_w + 1, 0 + elif x < 0: + cur_l, cur_w = cur_l + 1, 0 + else: + cur_w = cur_l = 0 + best_w, best_l = max(best_w, cur_w), max(best_l, cur_l) + cum = np.concatenate(([0.0], np.cumsum(r))) + peak = np.maximum.accumulate(cum) + flat, longest = 0, 0 + for i in range(1, len(cum)): + flat = 0 if cum[i] >= peak[i] else flat + 1 + longest = max(longest, flat) + return {"max_win_streak": best_w, "max_loss_streak": best_l, + "longest_flat_trades": longest} + + +def rolling_wr(df: pd.DataFrame, window: int = 20) -> pd.Series: + d = df.sort_values("entry_time", kind="mergesort") + s = (d["r_multiple"] > 0).astype(float).rolling(window).mean() * 100 + s.index = d["entry_time"].to_numpy() + return s + + +def equity_curve(df: pd.DataFrame) -> pd.DataFrame: + d = df.sort_values("entry_time", kind="mergesort") + cum = d["r_multiple"].cumsum() + peak = pd.concat([pd.Series([0.0]), cum]).cummax().iloc[1:] + peak.index = cum.index + return pd.DataFrame({"entry_time": d["entry_time"].to_numpy(), + "cum_r": cum.to_numpy(), + "drawdown": (peak - cum).to_numpy()}) + + +# ─────────────────────── quantile bucket screen ───────────────────── + +def quantile_buckets(df: pd.DataFrame, col: str, q: int = 5, + min_n: int = 10) -> pd.DataFrame: + """Equal-frequency buckets of `col` with per-half mean R. + + Bin edges are fit on the WHOLE book (both halves share edges). + Result carries gap_h1 / gap_h2 / sign_stable in `.attrs` and as + repeated columns so the CSV is self-describing. + """ + v = pd.to_numeric(df.get(col), errors="coerce") + d = df.loc[v.notna()].copy() + d["_v"] = v.loc[v.notna()] + empty = pd.DataFrame(columns=["bin", "lo", "hi", "n", "mean_r", "pf", "wr", + "n_h1", "mean_r_h1", "n_h2", "mean_r_h2", + "gap_h1", "gap_h2", "sign_stable"]) + empty.attrs = {"feature": col, "gap_h1": float("nan"), + "gap_h2": float("nan"), "sign_stable": False, "n": 0, + "mode": "empty"} + if d.empty: + return empty + + nuniq = int(d["_v"].nunique()) + if nuniq < 3: + mode = "values" + d["_bin"] = d["_v"] + order = sorted(d["_v"].unique()) + edges = {b: (float(b), float(b)) for b in order} + else: + mode = "qcut" + binned = pd.qcut(d["_v"], q, duplicates="drop") + d["_bin"] = binned + order = list(binned.cat.categories) + edges = {b: (float(b.left), float(b.right)) for b in order} + + rows = [] + for b in order: + sub = d[d["_bin"] == b] + if sub.empty: + continue + r = sub["r_multiple"] + h1 = sub[sub["half"] == 1]["r_multiple"] + h2 = sub[sub["half"] == 2]["r_multiple"] + lo, hi = edges[b] + rows.append({ + "bin": str(b), "lo": round(lo, 4), "hi": round(hi, 4), + "n": int(len(sub)), "mean_r": round(float(r.mean()), 3), + "pf": round(_pf(r), 2), + "wr": round(float((r > 0).mean() * 100), 2), + "n_h1": int(len(h1)), + "mean_r_h1": round(float(h1.mean()), 3) if len(h1) else float("nan"), + "n_h2": int(len(h2)), + "mean_r_h2": round(float(h2.mean()), 3) if len(h2) else float("nan"), + }) + out = pd.DataFrame(rows) + if out.empty: + return empty + + bot, top = out.iloc[0], out.iloc[-1] + gap_h1 = float(top["mean_r_h1"] - bot["mean_r_h1"]) + gap_h2 = float(top["mean_r_h2"] - bot["mean_r_h2"]) + populated = min(bot["n_h1"], bot["n_h2"], top["n_h1"], top["n_h2"]) >= min_n + signs_ok = (not math.isnan(gap_h1) and not math.isnan(gap_h2) + and np.sign(gap_h1) != 0 + and np.sign(gap_h1) == np.sign(gap_h2)) + sign_stable = bool(populated and signs_ok and len(out) >= 2) + out["gap_h1"] = round(gap_h1, 3) if not math.isnan(gap_h1) else float("nan") + out["gap_h2"] = round(gap_h2, 3) if not math.isnan(gap_h2) else float("nan") + out["sign_stable"] = sign_stable + out.attrs = {"feature": col, "gap_h1": gap_h1, "gap_h2": gap_h2, + "sign_stable": sign_stable, "n": int(len(d)), "mode": mode} + return out + + +def feature_screen(df: pd.DataFrame, q: int = 5, min_n: int = 10, + columns: Iterable[str] | None = None) -> pd.DataFrame: + """Bucket screen over every decision-time numeric feature in the book.""" + cols = list(columns) if columns is not None else feature_columns(df) + rows = [] + for c in cols: + bk = quantile_buckets(df, c, q=q, min_n=min_n) + a = bk.attrs + stable = bool(a.get("sign_stable")) + g1, g2 = a.get("gap_h1", float("nan")), a.get("gap_h2", float("nan")) + min_abs = (min(abs(g1), abs(g2)) if stable else 0.0) + rows.append({"feature": c, + "gap_h1": round(g1, 3) if g1 == g1 else float("nan"), + "gap_h2": round(g2, 3) if g2 == g2 else float("nan"), + "min_abs_gap": round(float(min_abs), 3), + "sign_stable": stable, + "n": int(a.get("n", 0)), + "bins": int(len(bk))}) + out = pd.DataFrame(rows) + if out.empty: + return out + return out.sort_values(["min_abs_gap", "n"], + ascending=[False, False]).reset_index(drop=True) + + +# ────────────────────── higher-timeframe extension ────────────────── + +def wilder_atr(bars: pd.DataFrame, period: int = 14) -> pd.Series: + h, l, c = bars["High"], bars["Low"], bars["Close"] + tr = pd.concat([h - l, (h - c.shift()).abs(), (l - c.shift()).abs()], + axis=1).max(axis=1) + return tr.ewm(alpha=1 / period, min_periods=period, adjust=False).mean() + + +def _ext_series(parquet: str, minutes: int) -> pd.DataFrame: + bars = pd.read_parquet(parquet).sort_index() + bars = bars[~bars.index.duplicated(keep="last")] + ext = ((bars["Close"] - bars["Close"].rolling(20).mean()) + / wilder_atr(bars, 14)) + out = pd.DataFrame({ + "close_time": bars.index + pd.Timedelta(minutes=minutes), + "ext": ext.to_numpy(), + }).dropna().reset_index(drop=True) + # Books parse to datetime64[us, UTC] (format="mixed"), parquet indexes are + # [ns, UTC]; merge_asof refuses mismatched resolutions. + out["close_time"] = out["close_time"].astype("datetime64[ns, UTC]") + return out + + +def add_htf_extension(df: pd.DataFrame, parquet_4h: str | None, + parquet_1d: str | None) -> pd.DataFrame: + """Add direction-signed H4 / D1 stretch-from-mean in ATR units. + + ext = (close − 20-bar SMA of close) / Wilder ATR(14) on that timeframe, + taken from the LAST higher-TF bar whose CLOSE time (bar label + one bar) + is <= decision_time — i.e. only information the live scan could have had. + Signed +1 for BUY, −1 for SELL, so positive always means "extended in the + direction of the trade". + """ + out = df.copy() + sign = np.where(out["direction"].astype(str).str.upper().str.startswith("B"), + 1.0, -1.0) + for col, parquet, minutes in (("x_h4_ext_atr", parquet_4h, 240), + ("x_d1_ext_atr", parquet_1d, 1440)): + if not parquet: + continue + right = _ext_series(parquet, minutes) + left = out[["decision_time"]].reset_index() + left["decision_time"] = left["decision_time"].astype( + "datetime64[ns, UTC]") + left = left.sort_values("decision_time", kind="mergesort") + merged = pd.merge_asof(left, right, left_on="decision_time", + right_on="close_time", direction="backward") + merged["_stale_h"] = ((merged["decision_time"] - merged["close_time"]) + .dt.total_seconds() / 3600.0) + merged = merged.set_index("index").reindex(out.index) + out[col] = merged["ext"].to_numpy() * sign + out.attrs[f"{col}_max_stale_h"] = float( + merged["_stale_h"].max()) if len(merged) else float("nan") + return out + + +# ──────────────────────────── reporting ───────────────────────────── + +def _fmt(v, nd=3): + if isinstance(v, float): + if math.isinf(v): + return "inf" + if math.isnan(v): + return "n/a" + return f"{v:.{nd}f}" + return str(v) + + +def _metrics_block(name: str, m: dict) -> list[str]: + return [ + f" {name:<10} n={m['n']:<5} totalR={_fmt(m['total_r'], 2):>8} " + f"meanR={_fmt(m['mean_r'])} medR={_fmt(m['median_r'])} " + f"WR={_fmt(m['wr'], 2)}% PF={_fmt(m['pf'], 2)}", + f" {'':<10} maxDD={_fmt(m['max_dd'], 2)} recovery={_fmt(m['recovery'], 2)}" + f" sharpe={_fmt(m['sharpe'])} expectancy={_fmt(m['expectancy'])}" + f" (wr·avgW − lr·avgL = {_fmt(m['expectancy_alt'])})", + f" {'':<10} avg bars={_fmt(m['avg_bars'], 1)} " + f"avg MFE={_fmt(m['avg_mfe'])}R avg MAE={_fmt(m['avg_mae'])}R", + ] + + +def build_report(df: pd.DataFrame, book: str, bar_minutes: int, split: str, + out_dir: str, q: int = 5, min_n: int = 10) -> str: + lines: list[str] = [] + add = lines.append + cut = split_instant(df["entry_time"], split) + add("=" * 78) + add(f"BOOK REPORT {os.path.basename(book)}") + add("=" * 78) + add(f"trades : {len(df)}") + add(f"entry window : {df['entry_time'].min()} -> {df['entry_time'].max()}") + add(f"bar minutes : {bar_minutes} (decision_time = entry_time − 1 bar)") + add(f"split rule : {split} -> cut at {cut} " + f"(half 1 = before, half 2 = at/after)") + add(f"half sizes : h1={int((df['half'] == 1).sum())} " + f"h2={int((df['half'] == 2).sum())}") + add("session map : asian 3-6, london 7-12, overlap 13-16, ny 17-22, " + "else off (on DECISION hour, UTC)") + add("bucket edges : quantile bins are fit on the WHOLE book; the two " + "halves share the same edges.") + feats = feature_columns(df) + add(f"screen features : {len(feats)} " + f"({sum(1 for c in feats if c.startswith(('f_', 'x_')))} f_/x_ columns)") + for k, v in df.attrs.items(): + add(f"htf staleness : {k} = {_fmt(v, 1)} h (max age of the matched bar)") + + add("") + add("── HEADLINE METRICS ──") + lines.extend(_metrics_block("full", base_metrics(df["r_multiple"], df))) + for h in (1, 2): + sub = df[df["half"] == h] + lines.extend(_metrics_block(f"half {h}", + base_metrics(sub["r_multiple"], sub))) + + group_specs = [ + ("weekday", "weekday"), ("decision_hour", "decision_hour"), + ("session", "session"), ("quality", "quality"), ("regime", "regime"), + ("vol_regime", "vol_regime"), ("direction", "direction"), + ("strategy", "strategy"), ("exit_reason", "exit_reason"), + ("indicators_agree", "indicators_agree"), + ] + tables = {} + for name, col in group_specs: + if col not in df.columns: + continue + t = group_table(df, col, min_n=min_n) + tables[name] = t + add("") + add(f"── BY {name.upper()} ──") + add(t.to_string() if not t.empty else " (empty)") + + # (weekday, decision_hour) pivots + add("") + add("── weekday x decision_hour : TOTAL R ──") + wh = group_table(df, ["weekday", "decision_hour"], min_n=min_n) + if not wh.empty: + piv_r = wh["total_r"].unstack("decision_hour") + piv_n = wh["n"].unstack("decision_hour") + add(piv_r.fillna(0).round(2).to_string()) + add("") + add("── weekday x decision_hour : N ──") + add(piv_n.fillna(0).astype(int).to_string()) + tables["weekday_hour"] = wh + piv_r.to_csv(os.path.join(out_dir, "groups_weekday_hour_total_r.csv")) + piv_n.to_csv(os.path.join(out_dir, "groups_weekday_hour_n.csv")) + + st = streaks(df) + add("") + add("── STREAKS ──") + add(f" max consecutive wins : {st['max_win_streak']}") + add(f" max consecutive losses : {st['max_loss_streak']}") + add(f" longest flat stretch : {st['longest_flat_trades']} trades " + f"between equity highs") + + rw = rolling_wr(df, 20).dropna() + add("") + add("── ROLLING WIN RATE (20 trades) ──") + if len(rw): + add(f" min={_fmt(rw.min(), 2)}% median={_fmt(rw.median(), 2)}% " + f"max={_fmt(rw.max(), 2)}% n={len(rw)}") + add(" last 5: " + ", ".join( + f"{ts:%Y-%m-%d %H:%M}={v:.2f}%" for ts, v in rw.tail(5).items())) + else: + add(" (fewer than 20 trades)") + + scr = feature_screen(df, q=q, min_n=min_n) + add("") + add("── FEATURE SCREEN (top 25 by min |gap| across halves) ──") + add(" gap = mean R(top bucket) − mean R(bottom bucket), per half; " + "min_abs_gap is 0 unless the sign is stable.") + add(scr.head(25).to_string(index=False) if not scr.empty else " (none)") + + stable = scr[scr["sign_stable"]].head(10) if not scr.empty else scr + add("") + add(f"── BUCKET TABLES FOR THE TOP {len(stable)} SIGN-STABLE FEATURES ──") + if stable.empty: + add(" (no feature has a sign-stable top-vs-bottom gap)") + for feat in stable["feature"]: + bk = quantile_buckets(df, feat, q=q, min_n=min_n) + add("") + add(f" · {feat} (mode={bk.attrs['mode']}, n={bk.attrs['n']}, " + f"gap_h1={_fmt(bk.attrs['gap_h1'])}, " + f"gap_h2={_fmt(bk.attrs['gap_h2'])})") + add(bk.to_string(index=False)) + bk.to_csv(os.path.join(out_dir, f"buckets_{feat}.csv"), index=False) + + # CSV artefacts + for name, t in tables.items(): + t.to_csv(os.path.join(out_dir, f"groups_{name}.csv")) + scr.to_csv(os.path.join(out_dir, "feature_screen.csv"), index=False) + equity_curve(df).to_csv(os.path.join(out_dir, "equity.csv"), index=False) + all_buckets = [] + for c in feats: + bk = quantile_buckets(df, c, q=q, min_n=min_n) + if bk.empty: + continue + bk = bk.copy() + bk.insert(0, "feature", c) + all_buckets.append(bk) + if all_buckets: + pd.concat(all_buckets, ignore_index=True).to_csv( + os.path.join(out_dir, "buckets.csv"), index=False) + return "\n".join(lines) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--book", required=True) + ap.add_argument("--bar-minutes", type=int, required=True) + ap.add_argument("--split", default="median", + help="'median' (default) or YYYY-MM-DD") + ap.add_argument("--parquet-4h", default=None) + ap.add_argument("--parquet-1d", default=None) + ap.add_argument("--q", type=int, default=5) + ap.add_argument("--min-n", type=int, default=10) + ap.add_argument("--out", default="backtest/research/book_report_out") + args = ap.parse_args() + + os.makedirs(args.out, exist_ok=True) + df = load_book(args.book, args.bar_minutes, args.split) + if args.parquet_4h or args.parquet_1d: + df = add_htf_extension(df, args.parquet_4h, args.parquet_1d) + text = build_report(df, args.book, args.bar_minutes, args.split, + args.out, q=args.q, min_n=args.min_n) + print(text) + with open(os.path.join(args.out, "report.txt"), "w") as fh: + fh.write(text + "\n") + print(f"\n[written] {os.path.join(args.out, 'report.txt')}") + + +if __name__ == "__main__": + main() diff --git a/backtest/research/burst_indicator_map.py b/backtest/research/burst_indicator_map.py new file mode 100644 index 0000000..0450ee7 --- /dev/null +++ b/backtest/research/burst_indicator_map.py @@ -0,0 +1,121 @@ +"""What do the system's OWN indicators read right before a violent bar? Analysis tooling only. + +Uses backtest.gpu_backtest.precompute_indicators (the same series the backtester feeds the +engine) on 3 years of bars; every state is taken at t-1 (known before the bar). Reports burst +rate (range >= 3x prior ATR14) for the NEXT bar and within the NEXT HOUR by indicator state, +by regime, by volatility regime, by weekday x hour, day of month, calendar proximity — each in +both halves — and whether the state predicts the burst's DIRECTION. + + python backtest/research/burst_indicator_map.py PARQUET BAR_MINUTES [--start --end] +""" +import argparse, sys +from pathlib import Path +import numpy as np, pandas as pd +ROOT = Path(__file__).resolve().parents[2]; sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "backtest")) +from gpu_backtest import precompute_indicators +from config import TechnicalConfig + +CAL = ROOT / "backtest" / "cache" / "calendar.parquet" + + +def wilder_atr(df, period=14): + pc = df["Close"].shift(1) + tr = pd.concat([df["High"] - df["Low"], (df["High"] - pc).abs(), (df["Low"] - pc).abs()], axis=1).max(axis=1) + return tr.ewm(alpha=1 / period, min_periods=period, adjust=False).mean() + + +def rate_table(d, col, target, half, order=None): + base1 = d.loc[half == 1, target].mean(); base2 = d.loc[half == 2, target].mean() + g = d.groupby(col, observed=True) + out = pd.DataFrame({"n": g.size(), + "h1_lift": d[half == 1].groupby(col, observed=True)[target].mean() / base1, + "h2_lift": d[half == 2].groupby(col, observed=True)[target].mean() / base2, + "rate%": 100 * g[target].mean()}) + if order is not None: out = out.reindex([o for o in order if o in out.index]) + out["stable"] = np.where((out.h1_lift > 1.15) & (out.h2_lift > 1.15), "HIGH", np.where((out.h1_lift < 0.85) & (out.h2_lift < 0.85), "low", "")) + return out.round(2) + + +def main(): + ap = argparse.ArgumentParser(); ap.add_argument("parquet"); ap.add_argument("bar_minutes", type=int) + ap.add_argument("--start", default="2023-09-05"); ap.add_argument("--end", default="2026-09-05") + a = ap.parse_args(); bm = a.bar_minutes + px = pd.read_parquet(a.parquet) + px = px[(px.index >= pd.Timestamp(a.start, tz="UTC") - pd.Timedelta(days=30)) & (px.index < pd.Timestamp(a.end, tz="UTC"))] + ic = precompute_indicators(px, TechnicalConfig()) + d = pd.DataFrame(index=px.index) + atr_prev = wilder_atr(px).shift(1) + rng = px["High"] - px["Low"] + d["burst"] = (rng >= 3.0 * atr_prev).astype(int) + H = max(1, 60 // bm) + d["next_hour"] = d["burst"][::-1].rolling(H, min_periods=1).max()[::-1].astype(int) + d["next_bar"] = d["burst"] + d["bdir"] = np.sign(px["Close"] - px["Open"]) + # states at t-1 + sh = lambda arr: pd.Series(np.asarray(arr, dtype=float), index=px.index).shift(1) + rsi = sh(ic.rsi); adx = sh(ic.adx); adxs = sh(ic.adx_slope); hist = sh(ic.macd_hist); bbp = sh(ic.bb_position) + bbw = sh(ic.bb_band_width); bbaw = sh(ic.bb_avg_width); kcsq = sh(ic.kc_squeeze); roc = sh(ic.roc) + ef, em, es = sh(ic.ema_fast), sh(ic.ema_medium), sh(ic.ema_slow); et = sh(ic.ema_trend) if ic.ema_trend is not None else None + close1 = px["Close"].shift(1); atr1 = sh(ic.atr) + kcu, kcl = sh(ic.kc_upper), sh(ic.kc_lower) + d["rsi"] = pd.cut(rsi, [0, 30, 40, 50, 60, 70, 100], labels=["<30", "30-40", "40-50", "50-60", "60-70", ">=70"]) + d["adx"] = pd.cut(adx, [0, 20, 25, 35, 50, 200], labels=["<20", "20-25", "25-35", "35-50", ">=50"]) + d["adx_slope"] = pd.cut(adxs, [-1e9, -3, 0, 3, 1e9], labels=["falling<-3", "-3..0", "0..3", "rising>3"]) + hist_prev = hist.shift(1) + d["macd"] = np.select([(hist > 0) & (hist > hist_prev), (hist > 0), (hist < 0) & (hist < hist_prev), (hist < 0)], + ["pos&rising", "pos&fading", "neg&falling", "neg&fading"], default="flat") + d["bb_pos"] = pd.cut(bbp, [-1e9, 0, 0.2, 0.8, 1.0, 1e9], labels=["<0 (below)", "0-0.2", "0.2-0.8", "0.8-1", ">1 (above)"]) + d["bb_squeeze"] = np.where(bbw < 0.75 * bbaw, "squeeze", "normal") + d["kc_squeeze"] = np.where(kcsq > 0, "TTM squeeze", "no") + d["kc_pos"] = np.select([close1 > kcu, close1 < kcl], ["above KC", "below KC"], default="inside") + align = np.sign(close1 - ef) + np.sign(ef - em) + np.sign(em - es) + (np.sign(es - et) if et is not None else 0) + d["ema_align"] = pd.cut(align, [-5, -3.5, -0.5, 0.5, 3.5, 5], labels=["fully bear(-4)", "mostly bear", "mixed", "mostly bull", "fully bull(+4)"]) + ribbon = (pd.concat([ef, em, es], axis=1).max(axis=1) - pd.concat([ef, em, es], axis=1).min(axis=1)) / atr1 + d["ribbon"] = np.where(ribbon < 1.0, "compressed(<1 ATR)", "expanded") + ema_dir = np.where(align > 0, 1, np.where(align < 0, -1, 0)) + d["regime"] = np.select([(adx >= 25) & (ema_dir > 0), (adx >= 25) & (ema_dir < 0), adx >= 25, adxs > 3.0], + ["trending_up", "trending_down", "trending(mixed)", "transitioning"], default="ranging") + atr_pct = atr1.rolling(50).apply(lambda w: (w[-1] > w).mean() * 100, raw=True) + d["vol_regime"] = pd.cut(atr_pct, [-1, 30, 60, 80, 101], labels=["low", "normal", "high", "extreme"]) + d["roc"] = pd.cut(roc, [-1e9, -1, -0.3, 0.3, 1, 1e9], labels=["<-1%", "-1..-0.3", "-0.3..0.3", "0.3..1", ">1%"]) + d["ema200_dist"] = pd.cut((close1 - et) / et * 100 if et is not None else pd.Series(np.nan, index=px.index), + [-1e9, -2.5, -0.5, 0.5, 2.5, 1e9], labels=["<-2.5%", "-2.5..-0.5", "near(±0.5%)", "0.5..2.5", ">2.5%"]) + # time + ct = px.index + pd.Timedelta(minutes=bm) + d["hour"] = ct.hour; d["weekday"] = ct.weekday; d["dom"] = pd.cut(ct.day, [0, 5, 10, 15, 20, 25, 31], labels=["1-5", "6-10", "11-15", "16-20", "21-25", "26-31"]) + d["first_friday"] = np.where((ct.weekday == 4) & (ct.day <= 7), "first Friday", "other") + # calendar proximity (high-impact USD) + cal = pd.read_parquet(CAL); cal["ts"] = pd.to_datetime(cal.dateline, unit="s", utc=True) + hi = np.sort(cal[(cal.currency == "USD") & (cal.impactName.str.lower() == "high")].ts.values.astype("datetime64[ns]")) + t = ct.values.astype("datetime64[ns]") + nxt = np.searchsorted(hi, t) # next event at/after bar close + mins_to = np.where(nxt < len(hi), (hi[np.minimum(nxt, len(hi) - 1)] - t) / np.timedelta64(1, "m"), 1e9) + prv = nxt - 1 + mins_since = np.where(prv >= 0, (t - hi[np.maximum(prv, 0)]) / np.timedelta64(1, "m"), 1e9) + d["calendar"] = np.select([mins_since <= 30, mins_to <= 15, mins_to <= 60, mins_to <= 180], ["0-30m AFTER hi-impact", "next 15m", "15-60m ahead", "1-3h ahead"], default="none within 3h") + d = d[d.index >= pd.Timestamp(a.start, tz="UTC")].dropna(subset=["rsi", "adx", "bb_pos", "vol_regime"]) + half = pd.Series(np.where(np.arange(len(d)) < len(d) // 2, 1, 2), index=d.index) + print(f"bars {len(d)} base next_bar {100*d.next_bar.mean():.3f}% next_hour {100*d.next_hour.mean():.2f}% (lift = rate / base; 'HIGH' = >1.15x in BOTH halves, 'low' = <0.85x in both)") + groups = ["calendar", "regime", "vol_regime", "rsi", "adx", "adx_slope", "macd", "bb_pos", "bb_squeeze", "kc_squeeze", "kc_pos", "ema_align", "ribbon", "roc", "ema200_dist", "dom", "first_friday"] + for target in ("next_bar", "next_hour"): + print(f"\n################ TARGET: {target} ################") + for g in groups: + print(f"-- {g}"); print(rate_table(d, g, target, half).to_string()) + # weekday x hour top cells (next_bar) + piv = d.pivot_table(index="weekday", columns="hour", values="next_bar", aggfunc="mean") * 100 + base = 100 * d.next_bar.mean() + cells = piv.stack().sort_values(ascending=False) + print(f"\n################ TOP 15 weekday x close-hour cells (next-bar burst pct, base {base:.2f}) ################") + names = {0: "Mon", 1: "Tue", 2: "Wed", 3: "Thu", 4: "Fri", 6: "Sun"} + for (wd, h), v in cells.head(15).items(): + n = int(((d.weekday == wd) & (d.hour == h)).sum()) + print(f" {names.get(wd, wd)} {h:02d}:xx {v:.2f}% ({v/base:.1f}x, n={n})") + # direction: does the state predict the burst direction? + b = d[d.burst == 1] + print("\n################ BURST DIRECTION vs prior state (share of bursts going UP) ################") + for g, up_states in (("ema_align", None), ("rsi", None), ("macd", None), ("regime", None)): + print(f"-- {g}: " + ", ".join(f"{k}: {100*(v>0).mean():.0f}% up (n={len(v)})" for k, v in b.groupby(g, observed=True).bdir if len(v) >= 20)) + + +if __name__ == "__main__": + main() diff --git a/backtest/research/burst_map.py b/backtest/research/burst_map.py new file mode 100644 index 0000000..6297910 --- /dev/null +++ b/backtest/research/burst_map.py @@ -0,0 +1,445 @@ +"""Where and when gold moves violently — burst map of the raw bars. + +Usage: + python backtest/research/burst_map.py --parquet PATH --bar-minutes N \ + [--k 3,4,5] [--book BOOK.csv] [--within-bars 12] \ + [--start YYYY-MM-DD --end YYYY-MM-DD] [--out DIR] + +A "burst" is one bar whose range is >= K x the PRIOR bar's Wilder ATR(14) — +the same definition burst_profile.py uses (that module computes it inline +inside burst_table() and exposes no reusable helper, so the formula is +replicated here bar-for-bar: TR = max(H-L, |H-Cprev|, |L-Cprev|), ATR = +TR.ewm(alpha=1/14, min_periods=14, adjust=False).mean(), ratio = +(H-L) / ATR.shift(1)). + +Reports: + * burst frequency by (weekday, UTC bar-CLOSE hour), plus mean |1 h| and + |4 h| absolute moves in $ and in ATR units on the same grid; + * post-burst drift in the burst's own direction over 6/12/24/48 bars, + overall and by session bucket — does a violent bar continue or snap back; + * the largest trailing-60-minute close-to-close moves, de-overlapped; + * if a trade book is given: how trades that were decided just after a burst, + or that were OPEN when one printed, performed against the rest. + +Every ATR-unit number here is normalised by the PRIOR bar's ATR — the same +denominator as the burst test, so a "4x burst that ran another 4 ATR" reads +directly. All hours are UTC and are the bar's CLOSE hour (label + one bar), +matching the clock the live gates evaluate. + +Analysis tooling only — never imported by the live code. +""" +from __future__ import annotations + +import argparse +import math +import os + +import numpy as np +import pandas as pd + +try: # package import (backtest.research.burst_map) + from backtest.research.book_report import base_metrics, load_book, session_of +except ImportError: # run directly as a script from anywhere + from book_report import base_metrics, load_book, session_of + +SLOT_UNKNOWN = "" + + +# ─────────────────────────── bar primitives ───────────────────────── + +def load_bars(parquet: str, start: str | None = None, + end: str | None = None) -> pd.DataFrame: + """Read a cached OHLCV parquet, de-duplicated and oldest-first.""" + df = pd.read_parquet(parquet).sort_index() + df = df[~df.index.duplicated(keep="last")] + if start: + df = df[df.index >= pd.Timestamp(start, tz="UTC")] + if end: + df = df[df.index <= pd.Timestamp(end, tz="UTC")] + return df + + +def wilder_atr(df: pd.DataFrame, period: int = 14) -> pd.Series: + """Wilder ATR — identical to the inline formula in burst_profile.py.""" + h, l, c = df["High"], df["Low"], df["Close"] + tr = pd.concat([h - l, (h - c.shift()).abs(), (l - c.shift()).abs()], + axis=1).max(axis=1) + return tr.ewm(alpha=1 / period, min_periods=period, adjust=False).mean() + + +def burst_ratio(df: pd.DataFrame, period: int = 14) -> pd.Series: + """Bar range expressed in PRIOR-bar ATRs.""" + return (df["High"] - df["Low"]) / wilder_atr(df, period).shift(1) + + +def burst_flags(df: pd.DataFrame, k: float, period: int = 14) -> pd.Series: + """True where the bar's range is >= k x the prior bar's Wilder ATR(14).""" + return (burst_ratio(df, period) >= k).fillna(False) + + +def burst_dir(df: pd.DataFrame) -> pd.Series: + """+1 up bar, −1 down bar, 0 doji (sign of Close − Open).""" + return np.sign(df["Close"] - df["Open"]).astype(float) + + +def close_times(df: pd.DataFrame, bar_minutes: int) -> pd.DatetimeIndex: + """Bar CLOSE timestamps (the parquet index labels bar OPEN time).""" + return df.index + pd.Timedelta(minutes=bar_minutes) + + +# ───────────────────── when does gold move? tables ────────────────── + +def hour_weekday_table(df: pd.DataFrame, k: float, bar_minutes: int = 5, + ks: tuple[float, ...] | None = None) -> dict: + """Burst frequency and absolute-move size on a (weekday, close-hour) grid. + + Returns a dict of DataFrames (several tables are wanted, one frame each): + burst_pct : weekday rows x close-hour cols, % of bars that are >= k x + bursts (one frame; per-k frames live in `burst_pct_by_k`) + absmove : (weekday, close-hour) rows; bars, mean |1 h move| and + |4 h move| in $ and in prior-bar ATR units + by_hour : the same stats collapsed to close-hour + by_weekday : the same stats collapsed to weekday + """ + ks = tuple(ks) if ks else (k,) + ct = close_times(df, bar_minutes) + hour = pd.Series(ct.hour, index=df.index) + wd = pd.Series(ct.weekday, index=df.index) + atr_prev = wilder_atr(df).shift(1) + + per_hour = max(1, int(round(60 / bar_minutes))) + c = df["Close"] + mv1 = (c - c.shift(per_hour)).abs() + mv4 = (c - c.shift(4 * per_hour)).abs() + stats = pd.DataFrame({ + "weekday": wd, "hour": hour, + "abs1h_usd": mv1, "abs4h_usd": mv4, + "abs1h_atr": mv1 / atr_prev, "abs4h_atr": mv4 / atr_prev, + }) + for kk in ks: + stats[f"burst_{kk:g}x"] = burst_flags(df, kk).astype(float) * 100.0 + + agg = {"abs1h_usd": "mean", "abs4h_usd": "mean", + "abs1h_atr": "mean", "abs4h_atr": "mean"} + agg.update({f"burst_{kk:g}x": "mean" for kk in ks}) + + grid = stats.groupby(["weekday", "hour"]).agg(agg) + grid.insert(0, "bars", stats.groupby(["weekday", "hour"]).size()) + by_hour = stats.groupby("hour").agg(agg) + by_hour.insert(0, "bars", stats.groupby("hour").size()) + by_wd = stats.groupby("weekday").agg(agg) + by_wd.insert(0, "bars", stats.groupby("weekday").size()) + + pct_by_k = {kk: grid[f"burst_{kk:g}x"].unstack("hour") for kk in ks} + return { + "burst_pct": pct_by_k[k if k in pct_by_k else ks[0]].round(3), + "burst_pct_by_k": {kk: v.round(3) for kk, v in pct_by_k.items()}, + "absmove": grid.round(3), + "by_hour": by_hour.round(3), + "by_weekday": by_wd.round(3), + } + + +def post_burst_drift(df: pd.DataFrame, k: float, + horizons: tuple[int, ...] = (6, 12, 24, 48), + bar_minutes: int = 5) -> pd.DataFrame: + """Signed drift IN the burst's direction over the next h bars, in ATRs. + + A positive mean means violent bars continue; negative means they snap + back. Split by the session bucket of the burst bar's close hour. + """ + flags = burst_flags(df, k) + d = burst_dir(df) + atr_prev = wilder_atr(df).shift(1) + ct = close_times(df, bar_minutes) + bucket = pd.Series([session_of(h) for h in ct.hour], index=df.index) + c = df["Close"] + + rows = [] + for h in horizons: + fwd = (c.shift(-h) - c) * d / atr_prev + sub = fwd[flags].dropna() + buckets = bucket[flags].reindex(sub.index) + rows.append({"horizon_bars": h, "bucket": "ALL", "n": int(len(sub)), + "mean_atr": round(float(sub.mean()), 4) if len(sub) else float("nan"), + "median_atr": round(float(sub.median()), 4) if len(sub) else float("nan"), + "pct_positive": round(float((sub > 0).mean() * 100), 2) if len(sub) else float("nan")}) + for b, s in sub.groupby(buckets): + rows.append({"horizon_bars": h, "bucket": b, "n": int(len(s)), + "mean_atr": round(float(s.mean()), 4), + "median_atr": round(float(s.median()), 4), + "pct_positive": round(float((s > 0).mean() * 100), 2)}) + return pd.DataFrame(rows) + + +def slot_label(weekday: int, hour: int, day: int) -> str: + """Time-slot label for a big move. + + NOTE: there is no economic-calendar file in this repo, so these are + TIME SLOTS, not confirmed events — "payroll window" means "the clock + slot US payrolls normally print in", nothing more. + """ + if weekday == 4 and day <= 7 and hour in (12, 13): + return "first-Friday 12:30 slot (payroll-report window)" + if weekday == 2 and hour in (18, 19): + return "Wednesday 18:00 slot (central-bank window)" + if hour in (12, 13): + return "12:30 slot (US data window)" + if hour == 14: + return "14:00 slot" + if hour in (21, 22, 23): + return "settlement/after-hours" + if hour in (0, 1, 2): + return "Asia open" + return SLOT_UNKNOWN + + +def largest_moves(df: pd.DataFrame, n: int = 40, window_minutes: int = 60, + bar_minutes: int = 5) -> pd.DataFrame: + """The n biggest trailing-`window_minutes` close-to-close moves. + + Overlapping windows are de-duplicated greedily: the largest move wins and + anything whose close time is within +/- `window_minutes` of an already + kept move is dropped, so one violent hour contributes one row. + """ + bars = max(1, int(round(window_minutes / bar_minutes))) + c = df["Close"] + move = c - c.shift(bars) + atr_prev = wilder_atr(df).shift(1) + ct = close_times(df, bar_minutes) + tbl = pd.DataFrame({ + "close_time": ct, "move_usd": move.to_numpy(), + "move_atr": (move / atr_prev).to_numpy(), + }).dropna() + tbl["abs_usd"] = tbl["move_usd"].abs() + tbl = tbl.sort_values("abs_usd", ascending=False) + + keep, taken = [], [] + gap = pd.Timedelta(minutes=window_minutes) + for _, row in tbl.iterrows(): + t = row["close_time"] + if any(abs(t - u) <= gap for u in taken): + continue + taken.append(t) + keep.append(row) + if len(keep) >= n: + break + out = pd.DataFrame(keep).reset_index(drop=True) + if out.empty: + return out + out["weekday"] = out["close_time"].dt.weekday + out["hour"] = out["close_time"].dt.hour + out["direction"] = np.where(out["move_usd"] > 0, "UP", "DOWN") + out["slot"] = [slot_label(w, h, t.day) for w, h, t + in zip(out["weekday"], out["hour"], out["close_time"])] + out["move_usd"] = out["move_usd"].round(2) + out["move_atr"] = out["move_atr"].round(2) + return out[["close_time", "weekday", "hour", "direction", "move_usd", + "move_atr", "slot"]] + + +# ───────────────────────── book <-> burst join ────────────────────── + +def _stats(r: pd.Series) -> dict: + m = base_metrics(r) + return {"n": m["n"], "total_r": round(m["total_r"], 2), + "mean_r": round(m["mean_r"], 3), "wr": round(m["wr"], 2), + "pf": round(m["pf"], 2) if math.isfinite(m["pf"]) else float("inf")} + + +def join_book_to_bursts(book_df: pd.DataFrame, bars: pd.DataFrame, k: float, + within_bars: int = 12, + bar_minutes: int = 5) -> dict: + """Split a trade book by its relationship to burst bars. + + (a) `after_burst` : decision_time falls within `within_bars` bars after + the most recent burst's close (bar distance is counted POSITIONALLY on + the bar index, so weekend gaps cannot inflate it). + (b) `burst_during` : at least one burst closed inside [entry, exit). + (c) `burst_agrees` / `burst_opposes`: of (b), whether the FIRST burst that + printed during the trade pushed with or against the trade direction. + """ + ct = close_times(bars, bar_minutes) + ct_arr = ct.to_numpy() + flags = burst_flags(bars, k).to_numpy() + bdir = burst_dir(bars).to_numpy() + burst_pos = np.flatnonzero(flags) + + # Last burst position at or before each bar (−1 where none yet). + last_burst = np.where(flags, np.arange(len(flags)), -1) + last_burst = np.maximum.accumulate(last_burst) + + dec = book_df["decision_time"].astype("datetime64[ns, UTC]").to_numpy() + ent = book_df["entry_time"].astype("datetime64[ns, UTC]").to_numpy() + exi = book_df["exit_time"].astype("datetime64[ns, UTC]").to_numpy() + + # Position of the last bar that had CLOSED by decision_time. + idx = np.searchsorted(ct_arr, dec, side="right") - 1 + valid = idx >= 0 + lb = np.where(valid, last_burst[np.clip(idx, 0, len(flags) - 1)], -1) + bars_since = np.where((lb >= 0) & valid, idx - lb, np.iinfo(np.int32).max) + after = pd.Series((bars_since >= 0) & (bars_since <= within_bars), + index=book_df.index) + + # Bursts printing inside the open trade. + bct = ct_arr[burst_pos] if len(burst_pos) else np.array([], dtype=ct_arr.dtype) + lo = np.searchsorted(bct, ent, side="left") + hi = np.searchsorted(bct, exi, side="left") + during = pd.Series(hi > lo, index=book_df.index) + trade_sign = np.where( + book_df["direction"].astype(str).str.upper().str.startswith("B"), + 1.0, -1.0) + has = hi > lo + first_dir = np.zeros(len(book_df), dtype=float) + if len(burst_pos) and has.any(): + # hi > lo guarantees lo[has] indexes a real burst. + first_dir[has] = bdir[burst_pos[lo[has]]] + agrees = pd.Series(has & (first_dir * trade_sign > 0), + index=book_df.index) + opposes = pd.Series(has & (first_dir * trade_sign < 0), + index=book_df.index) + + r = book_df["r_multiple"] + out = { + "k": k, "within_bars": within_bars, + "bursts_in_window": int(len(burst_pos)), + "after_burst": _stats(r[after]), + "not_after_burst": _stats(r[~after]), + "burst_during": _stats(r[during]), + "no_burst_during": _stats(r[~during]), + "burst_agrees": _stats(r[agrees]), + "burst_opposes": _stats(r[opposes]), + "median_bars_since_burst": float(np.median( + bars_since[bars_since < np.iinfo(np.int32).max])) if ( + bars_since < np.iinfo(np.int32).max).any() else float("nan"), + } + return out + + +# ──────────────────────────── reporting ───────────────────────────── + +def _fmt_stats(name: str, s: dict) -> str: + pf = "inf" if math.isinf(s["pf"]) else f"{s['pf']:.2f}" + return (f" {name:<18} n={s['n']:<5} totalR={s['total_r']:>8.2f} " + f"meanR={s['mean_r']:>7.3f} WR={s['wr']:>6.2f}% PF={pf}") + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--parquet", required=True) + ap.add_argument("--bar-minutes", type=int, required=True) + ap.add_argument("--k", default="3,4,5", + help="burst thresholds in prior-bar ATRs (default 3,4,5)") + ap.add_argument("--book", default=None) + ap.add_argument("--book-bar-minutes", type=int, default=None, + help="bar minutes of the BOOK (default: --bar-minutes)") + ap.add_argument("--within-bars", type=int, default=12) + ap.add_argument("--start", default=None) + ap.add_argument("--end", default=None) + ap.add_argument("--n-moves", type=int, default=40) + ap.add_argument("--out", default="backtest/research/burst_map_out") + args = ap.parse_args() + + ks = [float(x) for x in args.k.split(",") if x.strip()] + os.makedirs(args.out, exist_ok=True) + bars = load_bars(args.parquet, args.start, args.end) + bm = args.bar_minutes + primary = ks[0] + + lines: list[str] = [] + add = lines.append + add("=" * 78) + add(f"BURST MAP {os.path.basename(args.parquet)}") + add("=" * 78) + add(f"bars : {len(bars)} ({bars.index[0]} -> {bars.index[-1]}" + f", labels = bar OPEN time)") + add(f"bar minutes : {bm} hours below are bar CLOSE hours (UTC)") + add(f"burst definition : range >= K x PRIOR bar Wilder ATR(14); " + f"K = {', '.join(f'{k:g}' for k in ks)}") + add("ATR units : always normalised by the PRIOR bar's ATR") + for k in ks: + f = burst_flags(bars, k) + add(f" K={k:g}: {int(f.sum())} burst bars " + f"({float(f.mean()) * 100:.3f}% of bars)") + + tabs = hour_weekday_table(bars, primary, bm, ks=tuple(ks)) + add("") + add(f"── BURST % BY (weekday, close-hour), K={primary:g} ──") + add(tabs["burst_pct"].fillna(0).round(2).to_string()) + add("") + add("── MEAN ABSOLUTE MOVE BY CLOSE-HOUR (all weekdays) ──") + add(tabs["by_hour"].to_string()) + add("") + add("── MEAN ABSOLUTE MOVE BY WEEKDAY ──") + add(tabs["by_weekday"].to_string()) + + add("") + add(f"── POST-BURST DRIFT (in the burst's own direction), K={primary:g} ──") + drift = post_burst_drift(bars, primary, bar_minutes=bm) + add(drift.to_string(index=False)) + + add("") + add(f"── LARGEST {args.n_moves} TRAILING-60-MIN MOVES (de-overlapped) ──") + add(" slot labels are CLOCK SLOTS — no economic calendar exists in " + "this repo, so no event is confirmed.") + moves = largest_moves(bars, n=args.n_moves, bar_minutes=bm) + add(moves.to_string(index=False)) + + join_rows = [] + if args.book: + bbm = args.book_bar_minutes or bm + book = load_book(args.book, bbm) + add("") + add(f"── BOOK JOIN {os.path.basename(args.book)} " + f"({len(book)} trades, book bar = {bbm} min) ──") + first, last = close_times(bars, bm)[0], close_times(bars, bm)[-1] + outside = int(((book["decision_time"] < first) + | (book["exit_time"] > last)).sum()) + if outside: + add(f" !! {outside} trade(s) start before / end after the bar " + f"window ({first} .. {last}) — their burst context is " + f"partial. Widen --start/--end or use a longer parquet.") + add(" CAUTION: group (b) is survivorship-shaped — a trade only sees " + "a burst if it survived long enough to still be open, so its high " + "PF is mostly 'winners last longer', not a burst edge.") + for k in ks: + j = join_book_to_bursts(book, bars, k, args.within_bars, bm) + add(f" K={k:g} ({j['bursts_in_window']} bursts in window; " + f"median bars since last burst = " + f"{j['median_bars_since_burst']:.0f})") + add(f" (a) decided within {args.within_bars} bars of a burst") + add(_fmt_stats("after_burst", j["after_burst"])) + add(_fmt_stats("not_after_burst", j["not_after_burst"])) + add(" (b) a burst printed while the trade was open") + add(_fmt_stats("burst_during", j["burst_during"])) + add(_fmt_stats("no_burst_during", j["no_burst_during"])) + add(" (c) of (b), first burst vs the trade's direction") + add(_fmt_stats("burst_agrees", j["burst_agrees"])) + add(_fmt_stats("burst_opposes", j["burst_opposes"])) + for grp in ("after_burst", "not_after_burst", "burst_during", + "no_burst_during", "burst_agrees", "burst_opposes"): + join_rows.append({"k": k, "group": grp, + "within_bars": args.within_bars, + **j[grp]}) + + text = "\n".join(lines) + print(text) + with open(os.path.join(args.out, "burst_map.txt"), "w") as fh: + fh.write(text + "\n") + pd.concat([v.assign(k=kk) for kk, v in tabs["burst_pct_by_k"].items()] + ).to_csv(os.path.join(args.out, "hour_weekday_burst_pct.csv")) + tabs["absmove"].to_csv(os.path.join(args.out, "hour_weekday_absmove.csv")) + tabs["by_hour"].to_csv(os.path.join(args.out, "by_hour.csv")) + tabs["by_weekday"].to_csv(os.path.join(args.out, "by_weekday.csv")) + pd.concat([post_burst_drift(bars, k, bar_minutes=bm).assign(k=k) + for k in ks]).to_csv( + os.path.join(args.out, "post_burst_drift.csv"), index=False) + moves.to_csv(os.path.join(args.out, "largest_moves.csv"), index=False) + if join_rows: + pd.DataFrame(join_rows).to_csv( + os.path.join(args.out, "book_join.csv"), index=False) + print(f"\n[written] {os.path.join(args.out, 'burst_map.txt')}") + + +if __name__ == "__main__": + main() diff --git a/backtest/research/burst_precursors.py b/backtest/research/burst_precursors.py new file mode 100644 index 0000000..5fa541c --- /dev/null +++ b/backtest/research/burst_precursors.py @@ -0,0 +1,127 @@ +"""Are there signs BEFORE a violent bar? Analysis tooling only — never imported by the live code. + +For every bar t, features are computed from bars <= t-1 only. Targets: + next_bar : bar t itself is a burst (range >= K x prior Wilder ATR14) + next_hour : any burst in bars t .. t+H-1 +Reports per-quintile burst rates (lift vs base rate) in both halves, the same +within/outside the scheduled-data hours, and an out-of-sample logistic model +(fit on half 1, scored on half 2): AUC + lift in the top 5% of scores. + + python backtest/research/burst_precursors.py PARQUET BAR_MINUTES [--k 3] [--start ... --end ...] +""" +import argparse, sys +import numpy as np, pandas as pd + + +def wilder_atr(df, period=14): + pc = df["Close"].shift(1) + tr = pd.concat([df["High"] - df["Low"], (df["High"] - pc).abs(), (df["Low"] - pc).abs()], axis=1).max(axis=1) + return tr.ewm(alpha=1 / period, min_periods=period, adjust=False).mean() + + +def build(df, bar_minutes, k): + d = df.copy() + rng = d["High"] - d["Low"] + atr_prev = wilder_atr(d).shift(1) # ATR known at t-1 + d["burst"] = (rng >= k * atr_prev).astype(int) + H = max(1, 60 // bar_minutes) + # burst within next H bars (t..t+H-1) + d["next_hour"] = d["burst"][::-1].rolling(H, min_periods=1).max()[::-1].astype(int) + d["next_bar"] = d["burst"] + close_t = d.index + pd.Timedelta(minutes=bar_minutes) + d["hour"] = close_t.hour + r1 = rng.shift(1) + d["squeeze6"] = r1.rolling(6).mean() / atr_prev + d["squeeze12"] = r1.rolling(12).mean() / atr_prev + d["last_bar_range"] = r1 / atr_prev + sma = d["Close"].shift(1).rolling(20).mean(); sd = d["Close"].shift(1).rolling(20).std(ddof=0) + bbw = (4 * sd / sma) + d["bbw_pct500"] = bbw.rolling(500).rank(pct=True) + v1 = d["Volume"].shift(1) + d["vol_ratio"] = v1.rolling(3).mean() / v1.rolling(50).mean() + d["drift12"] = (d["Close"].shift(1) - d["Close"].shift(13)).abs() / atr_prev + d["atr_pct500"] = atr_prev.rolling(500).rank(pct=True) + b = d["burst"].shift(1).fillna(0).astype(int) + since = np.zeros(len(d)); cnt = 200 + bv = b.values + for i in range(len(d)): + cnt = 0 if bv[i] == 1 else min(200, cnt + 1) + since[i] = cnt + d["bars_since_burst"] = since + return d.dropna(subset=["squeeze6", "squeeze12", "bbw_pct500", "vol_ratio", "drift12", "atr_pct500"]) + + +FEATS = ["squeeze6", "squeeze12", "last_bar_range", "bbw_pct500", "vol_ratio", "drift12", "atr_pct500", "bars_since_burst"] + + +def quintile_table(d, feat, target, half): + q = pd.qcut(d[feat], 5, labels=False, duplicates="drop") + base1 = d.loc[half == 1, target].mean(); base2 = d.loc[half == 2, target].mean() + rows = [] + for b in sorted(q.dropna().unique()): + m = q == b + lo, hi = d.loc[m, feat].min(), d.loc[m, feat].max() + r1 = d.loc[m & (half == 1), target].mean(); r2 = d.loc[m & (half == 2), target].mean() + rows.append((int(b), lo, hi, int(m.sum()), 100 * r1, r1 / base1, 100 * r2, r2 / base2)) + t = pd.DataFrame(rows, columns=["bin", "lo", "hi", "n", "h1_rate%", "h1_lift", "h2_rate%", "h2_lift"]) + return t.round(3) + + +def main(): + ap = argparse.ArgumentParser(); ap.add_argument("parquet"); ap.add_argument("bar_minutes", type=int) + ap.add_argument("--k", type=float, default=3.0); ap.add_argument("--start"); ap.add_argument("--end") + a = ap.parse_args() + df = pd.read_parquet(a.parquet) + if a.start: df = df[df.index >= pd.Timestamp(a.start, tz="UTC")] + if a.end: df = df[df.index < pd.Timestamp(a.end, tz="UTC")] + d = build(df, a.bar_minutes, a.k) + half = pd.Series(np.where(np.arange(len(d)) < len(d) // 2, 1, 2), index=d.index) + print(f"bars {len(d)} K={a.k} base rate next_bar {100*d.next_bar.mean():.3f}% next_hour {100*d.next_hour.mean():.2f}%") + print(f"half 1 ends {d.index[len(d)//2 - 1]}") + # 1. the clock + hr = d.groupby("hour")[["next_bar", "next_hour"]].mean() * 100 + hr["lift_next_bar"] = hr["next_bar"] / (100 * d.next_bar.mean()) + print("\n== THE CLOCK (bar-close hour UTC): burst rate % and lift ==") + print(hr.round(2).T.to_string()) + sched = d["hour"].isin([1, 12, 13]) + print(f"\nscheduled hours (01,12,13) hold {100*d.loc[sched,'next_bar'].sum()/d.next_bar.sum():.1f}% of bursts in {100*sched.mean():.1f}% of bars") + # 2. per-feature quintiles, both halves + for target in ("next_bar", "next_hour"): + print(f"\n== TARGET {target}: quintile burst rate (lift vs base) — ALL hours ==") + for f in FEATS: + t = quintile_table(d, f, target, half) + print(f"-- {f}: lifts h1 {list(t.h1_lift)} h2 {list(t.h2_lift)} (bins lo..hi {t.lo.iloc[0]:.2f}..{t.hi.iloc[-1]:.2f})") + # 3. same, OUTSIDE the scheduled hours (does anything predict the unscheduled ones?) + u = d[~sched]; hu = half[~sched] + print(f"\n== next_bar lifts OUTSIDE scheduled hours (base {100*u.next_bar.mean():.3f}%) ==") + for f in FEATS: + t = quintile_table(u, f, "next_bar", hu) + print(f"-- {f}: h1 {list(t.h1_lift)} h2 {list(t.h2_lift)}") + # 4. OOS logistic model: fit half 1, score half 2 + try: + from sklearn.linear_model import LogisticRegression + from sklearn.metrics import roc_auc_score + X = d[FEATS].copy() + for h in range(24): X[f"h{h}"] = (d["hour"] == h).astype(float) + Xc = d[FEATS].copy() # clock-free variant + for name, XX in (("features+clock", X), ("features only", Xc), ("clock only", X[[f"h{h}" for h in range(24)]])): + mu, sd = XX[half == 1].mean(), XX[half == 1].std().replace(0, 1) + Z = (XX - mu) / sd + for target in ("next_bar", "next_hour"): + m = LogisticRegression(C=0.1, max_iter=2000, class_weight="balanced").fit(Z[half == 1], d.loc[half == 1, target]) + p = m.predict_proba(Z[half == 2])[:, 1]; y = d.loc[half == 2, target] + auc = roc_auc_score(y, p) + top = p >= np.quantile(p, 0.95) + print(f"OOS {name:15s} {target:9s}: AUC {auc:.3f} | top-5% of scores: burst rate {100*y[top].mean():.2f}% vs base {100*y.mean():.2f}% (lift {y[top].mean()/y.mean():.2f}x), captures {100*y[top].sum()/y.sum():.1f}% of bursts") + except ImportError: + print("sklearn missing — OOS model skipped") + # 5. direction: does the burst continue the prior 12-bar drift? + bt = d[d.burst == 1] + prior = np.sign(bt["Close"].shift(1) - bt["Close"].shift(13)) if False else np.sign(d["Close"].shift(1) - d["Close"].shift(13))[d.burst == 1] + bdir = np.sign(bt["Close"] - bt["Open"]) + agree = (prior == bdir)[(prior != 0) & (bdir != 0)] + print(f"\nburst direction == prior-hour drift direction: {100*agree.mean():.1f}% of {len(agree)} bursts (50% = no information)") + + +if __name__ == "__main__": + main() diff --git a/backtest/research/calendar_impact.py b/backtest/research/calendar_impact.py new file mode 100644 index 0000000..6e99669 --- /dev/null +++ b/backtest/research/calendar_impact.py @@ -0,0 +1,111 @@ +"""Do scheduled economic events move gold violently, and do they hurt OPEN trades? Analysis tooling only. + +Needs backtest/cache/calendar.parquet (ff_calendar_scraper.py). Uses USD events by impact. + python backtest/research/calendar_impact.py PARQUET_BARS BAR_MINUTES BOOK.csv ARM DIST [--start --end] +""" +import argparse, sys +from pathlib import Path +import numpy as np, pandas as pd +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from open_trade_bursts import wilder_atr, simulate, r_of, stats, K, COST + +CAL = Path(__file__).resolve().parent.parent / "cache" / "calendar.parquet" + + +def main(): + ap = argparse.ArgumentParser(); ap.add_argument("parquet"); ap.add_argument("bar_minutes", type=int) + ap.add_argument("book"); ap.add_argument("arm", type=float); ap.add_argument("dist", type=float) + ap.add_argument("--start", default="2023-09-05"); ap.add_argument("--end", default="2026-09-05") + a = ap.parse_args(); bm = a.bar_minutes + cal = pd.read_parquet(CAL) + cal["ts"] = pd.to_datetime(cal["dateline"], unit="s", utc=True) + cal = cal[(cal.ts >= pd.Timestamp(a.start, tz="UTC")) & (cal.ts < pd.Timestamp(a.end, tz="UTC"))] + usd = cal[cal.currency == "USD"].copy() + usd["impact"] = usd["impactName"].str.lower() + print(f"calendar: {len(cal)} events in window, USD {len(usd)}: " + ", ".join(f"{k}={v}" for k, v in usd.impact.value_counts().items())) + px = pd.read_parquet(a.parquet); px = px[(px.index >= pd.Timestamp(a.start, tz="UTC")) & (px.index < pd.Timestamp(a.end, tz="UTC"))] + atr_prev = wilder_atr(px).shift(1) + rng = px["High"] - px["Low"]; burst = (rng >= K * atr_prev).fillna(False) + close = px["Close"]; idx = px.index + H1 = 60 // bm + move60 = (close.shift(-H1) - close).abs() / atr_prev # |move| over the next hour from bar OPEN-ish (label) + move15 = (close.shift(-max(1, 15 // bm)) - close).abs() / atr_prev + burst60 = burst[::-1].rolling(H1, min_periods=1).max()[::-1].astype(bool) + # event -> bar containing it (label <= ts < label + bm) + def bar_pos(ts): + p = idx.searchsorted(ts, side="right") - 1 + return p if 0 <= p < len(idx) and (ts - idx[p]) < pd.Timedelta(minutes=bm) else -1 + usd["pos"] = [bar_pos(t) for t in usd.ts] + usd = usd[usd.pos >= 0] + usd["bar_range_atr"] = rng.values[usd.pos] / atr_prev.values[usd.pos] + usd["burst_in_60m"] = burst60.values[usd.pos] + usd["move60_atr"] = move60.values[usd.pos]; usd["move60_usd"] = (close.shift(-H1) - close).abs().values[usd.pos] + usd["move15_atr"] = move15.values[usd.pos] + # baseline: all bars, and bars in the same close-hours without any USD high/medium event within the hour + chour = (idx + pd.Timedelta(minutes=bm)).hour + ev_bars = set() + for p in usd[usd.impact.isin(["high", "medium"])].pos: ev_bars.update(range(max(0, p - H1), min(len(idx), p + H1))) + quiet = ~np.isin(np.arange(len(idx)), list(ev_bars)) + def base_row(mask, label): + return dict(group=label, n=int(mask.sum()), bar_range_atr=float((rng / atr_prev)[mask].mean()), burst_in_60m_pct=100 * float(burst60[mask].mean()), + move15_atr=float(move15[mask].mean()), move60_atr=float(move60[mask].mean()), move60_usd=float((close.shift(-H1) - close).abs()[mask].mean())) + rows = [base_row(np.ones(len(idx), bool), "ALL bars"), base_row(quiet & np.isin(chour, [12, 13, 14]), "12-14 UTC bars with NO USD event")] + for imp in ("high", "medium", "low"): + g = usd[usd.impact == imp] + rows.append(dict(group=f"USD {imp}-impact events", n=len(g), bar_range_atr=g.bar_range_atr.mean(), burst_in_60m_pct=100 * g.burst_in_60m.mean(), + move15_atr=g.move15_atr.mean(), move60_atr=g.move60_atr.mean(), move60_usd=g.move60_usd.mean())) + print("\n== HOW MUCH GOLD MOVES AROUND USD EVENTS (event bar = the bar containing the release) ==") + print(pd.DataFrame(rows).round(2).to_string(index=False)) + hi = usd[usd.impact == "high"] + top = hi.groupby("name").agg(n=("ts", "size"), bar_range_atr=("bar_range_atr", "mean"), burst60_pct=("burst_in_60m", lambda s: 100 * s.mean()), + move60_usd=("move60_usd", "mean"), move60_atr=("move60_atr", "mean")).sort_values("n", ascending=False) + print("\n== HIGH-IMPACT USD EVENTS BY NAME (n >= 6) ==") + print(top[top.n >= 6].round(2).to_string()) + # what share of ALL bursts sit within ±30 min of a high/medium USD event? + near = np.zeros(len(idx), bool) + for p in usd[usd.impact.isin(["high", "medium"])].pos: near[max(0, p - 30 // bm):min(len(idx), p + 30 // bm + 1)] = True + nearh = np.zeros(len(idx), bool) + for p in hi.pos: nearh[max(0, p - 30 // bm):min(len(idx), p + 30 // bm + 1)] = True + print(f"\nshare of ALL violent bars within ±30 min of a USD high-impact event: {100*burst[nearh].sum()/burst.sum():.1f}% (those bars are {100*nearh.mean():.1f}% of time)") + print(f"share within ±30 min of a USD high- or medium-impact event: {100*burst[near].sum()/burst.sum():.1f}% (bars {100*near.mean():.1f}% of time)") + # open trades through high-impact events + book = pd.read_csv(a.book); book["entry_time"] = pd.to_datetime(book.entry_time, utc=True, format="mixed") + bars = dict(O=px.Open.values, H=px.High.values, L=px.Low.values, C=px.Close.values, burst=burst.values, chour=chour.values) + pos = idx.get_indexer(book.entry_time); ok = pos >= 0; book = book[ok].reset_index(drop=True); pos = pos[ok] + ev_pos = np.sort(hi.pos.values) + # flat mask: exit at the open of the bar that CONTAINS a high-impact event... use the bar before (exit ~5-15 min before) + flat_mask = np.zeros(len(idx), bool); tight_mask = np.zeros(len(idx), bool) + for p in ev_pos: + if p - 1 >= 0: flat_mask[p - 1] = True + tight_mask[max(0, p - 1):min(len(idx), p + H1)] = True + res = {} + for v, kw in (("base", {}), ("mask_flat", dict(flat_mask=flat_mask)), ("mask_tight", dict(tight_mask=tight_mask))): + rs, fl, thru = [], [], [] + for i, t in book.iterrows(): + xp, held, reason, flag = simulate(bars, pos[i], t.direction, t.entry, t.sl, a.arm, a.dist, v, **kw) + rs.append(r_of(t.direction, t.entry, t.sl, xp)); fl.append(flag) + if v == "base": + j0, j1 = pos[i], pos[i] + held + k = ev_pos[(ev_pos > j0) & (ev_pos <= j1)] + thru.append(k[0] if len(k) else -1) + res[v] = (np.array(rs), np.array(fl)) + if v == "base": book["ev_pos"] = thru + base = res["base"][0]; half = np.where(np.arange(len(book)) < len(book) // 2, 1, 2) + hit = book.ev_pos.values >= 0 + print(f"\n== OPEN TRADES THAT HELD THROUGH A HIGH-IMPACT USD EVENT ({a.book.split('/')[-1]}) ==") + print(f"trades through an event: {hit.sum()} of {len(book)} ({100*hit.mean():.1f}%) | mean R {base[hit].mean():+.3f} vs {base[~hit].mean():+.3f} for the rest | WR {100*(base[hit]>0).mean():.0f}% vs {100*(base[~hit]>0).mean():.0f}%") + if hit.sum(): + d = np.where(book.direction == "BUY", 1.0, -1.0) + m15 = np.array([ (close.shift(-max(1,15//bm)).values[p] - close.values[p]) if p >= 0 else np.nan for p in book.ev_pos]) + with_ = hit & (d * m15 > 0); against = hit & (d * m15 < 0) + print(f" event's first 15-min move WITH the trade: n={with_.sum()} mean R {base[with_].mean():+.3f} | AGAINST: n={against.sum()} mean R {base[against].mean():+.3f} (WR {100*(base[against]>0).mean():.0f}%)") + print("\n== PROTECTION AROUND HIGH-IMPACT USD EVENTS ==") + b = stats(base) + for v, (r, f) in res.items(): + s = stats(r); aff = f == 1 + d1 = r[half == 1].sum() - base[half == 1].sum(); d2 = r[half == 2].sum() - base[half == 2].sum() + print(f"{v:11s} total {s['total']:+8.2f} Δ {s['total']-b['total']:+7.2f} PF {s['pf']:.2f} DD {s['dd']:.2f} affected {aff.sum():4d} (affected: base {base[aff].sum():+.2f} -> {r[aff].sum():+.2f}) | Δ h1 {d1:+.2f} / h2 {d2:+.2f}") + + +if __name__ == "__main__": + main() diff --git a/backtest/research/open_trade_bursts.py b/backtest/research/open_trade_bursts.py new file mode 100644 index 0000000..7ebef39 --- /dev/null +++ b/backtest/research/open_trade_bursts.py @@ -0,0 +1,150 @@ +"""What do violent bars do to OPEN trades, and can we protect them? Analysis tooling only. + +Replays every trade of a book on the raw bars with the backtester's exit model (SL-first, +gap-through at open, close-only trail ratchet, 300-bar timeout, $0.65 cost) and compares: + base : reproduce the book (validation: must match r_multiple for almost every trade) + flat_data : exit at the OPEN of the first bar whose close-hour enters {12,13} UTC (US data) + flat_asia : same for close-hour {0,1} UTC (Asian open) + tight_data : entering {12,13}, hold a protective stop 0.5R below the best price for the window + tight_asia : same for {0,1} + react_exit : a burst bar (range >= 3x prior ATR14) closes AGAINST the trade -> exit next open + react_tight : after an opposing burst, protective stop 0.5R below the best price for the rest +Plus a damage table: what happens to a trade when the first burst hits it (with vs against). + + python backtest/research/open_trade_bursts.py BOOK.csv PARQUET BAR_MINUTES ARM DIST +""" +import sys +import numpy as np, pandas as pd + +COST = 0.65; MAX_BARS = 300; K = 3.0; PROT = 0.5 + + +def wilder_atr(df, period=14): + pc = df["Close"].shift(1) + tr = pd.concat([df["High"] - df["Low"], (df["High"] - pc).abs(), (df["Low"] - pc).abs()], axis=1).max(axis=1) + return tr.ewm(alpha=1 / period, min_periods=period, adjust=False).mean() + + +def simulate(bars, i0, direction, entry, sl, arm, dist, variant, hours_flat=(), hours_tight=(), flat_mask=None, tight_mask=None): + """bars: dict of numpy arrays; i0 = index of the entry bar. Returns (exit_price, bars_held, reason, flag).""" + O, H, L, C, burst, chour = bars["O"], bars["H"], bars["L"], bars["C"], bars["burst"], bars["chour"] + d = 1.0 if direction == "BUY" else -1.0 + risk = abs(entry - sl); stop = sl; peak = entry; flag = 0 + n = len(O); tight = False; tight_until = -1; exit_next = False + for j in range(i0, min(n, i0 + MAX_BARS)): + if exit_next: + return O[j], j - i0, "REACT", flag + h = chour[j] + in_flat = h in hours_flat; in_tight = h in hours_tight + if variant.startswith("flat") and in_flat: + return O[j], j - i0, "FLAT", 1 + if variant == "mask_flat" and flat_mask is not None and flat_mask[j]: + return O[j], j - i0, "FLAT", 1 + if variant.startswith("tight") and in_tight: + tight = True; tight_until = j; flag = 1 + if variant == "mask_tight" and tight_mask is not None and tight_mask[j]: + tight = True; tight_until = j; flag = 1 + # stop check (SL first, gap-through at open) + if d > 0: + if O[j] <= stop: return O[j], j - i0, "SL", flag + if L[j] <= stop: return stop, j - i0, "SL", flag + else: + if O[j] >= stop: return O[j], j - i0, "SL", flag + if H[j] >= stop: return stop, j - i0, "SL", flag + # update peak / trail at bar close + peak = max(peak, H[j]) if d > 0 else min(peak, L[j]) + fav = d * (peak - entry) + if fav >= arm * risk: + stop = max(stop, peak - dist * risk) if d > 0 else min(stop, peak + dist * risk) + if tight and (variant == "react_tight" or j <= tight_until or in_tight): + stop = max(stop, peak - PROT * risk) if d > 0 else min(stop, peak + PROT * risk) + if variant.startswith("react") and burst[j] and d * (C[j] - O[j]) < 0: + flag = 1 + if variant == "react_exit": exit_next = True + else: tight = True + j = min(n - 1, i0 + MAX_BARS - 1) + return C[j], j - i0, "TIMEOUT", flag + + +def r_of(direction, entry, sl, exit_price): + d = 1.0 if direction == "BUY" else -1.0 + return (d * (exit_price - entry) - COST) / abs(entry - sl) + + +def stats(r): + r = np.asarray(r); eq = np.concatenate([[0.0], np.cumsum(r)]); dd = (np.maximum.accumulate(eq) - eq).max() + pf = r[r > 0].sum() / max(-r[r < 0].sum(), 1e-9) + return dict(n=len(r), total=r.sum(), mean=r.mean(), wr=100 * (r > 0).mean(), pf=pf, dd=dd) + + +def main(): + book_csv, pq, bm, arm, dist = sys.argv[1], sys.argv[2], int(sys.argv[3]), float(sys.argv[4]), float(sys.argv[5]) + book = pd.read_csv(book_csv); book["entry_time"] = pd.to_datetime(book["entry_time"], utc=True, format="mixed") + px = pd.read_parquet(pq) + atr_prev = wilder_atr(px).shift(1) + bars = dict(O=px["Open"].values, H=px["High"].values, L=px["Low"].values, C=px["Close"].values, + burst=((px["High"] - px["Low"]) >= K * atr_prev).fillna(False).values, + chour=(px.index + pd.Timedelta(minutes=bm)).hour.values) + idx = pd.Index(px.index) + pos = idx.get_indexer(book["entry_time"]) + ok = pos >= 0 + print(f"{book_csv}: {len(book)} trades, {ok.sum()} matched to bars; arm {arm} dist {dist}") + book = book[ok].reset_index(drop=True); pos = pos[ok] + half = np.where(np.arange(len(book)) < len(book) // 2, 1, 2) + variants = {"base": {}, "flat_data": dict(hours_flat=(12, 13)), "flat_asia": dict(hours_flat=(0, 1)), + "tight_data": dict(hours_tight=(12, 13)), "tight_asia": dict(hours_tight=(0, 1)), + "react_exit": {}, "react_tight": {}} + results = {} + for v, kw in variants.items(): + rs, flags = [], [] + for i, t in book.iterrows(): + xp, held, reason, flag = simulate(bars, pos[i], t.direction, t.entry, t.sl, arm, dist, v, **kw) + rs.append(r_of(t.direction, t.entry, t.sl, xp)); flags.append(flag) + results[v] = (np.array(rs), np.array(flags)) + base = results["base"][0] + match = np.isclose(base, book["r_multiple"].values, atol=0.02).mean() + print(f"validation: replay reproduces the book's r_multiple (±0.02) on {100*match:.1f}% of trades; " + f"sum replay {base.sum():+.2f} vs book {book.r_multiple.sum():+.2f}") + print("\n== PROTECTION VARIANTS (whole book; 'affected' = trades where the rule fired) ==") + print(f"{'variant':12s} {'n':>5s} {'affected':>8s} {'total R':>9s} {'Δ vs base':>9s} {'PF':>5s} {'maxDD':>6s} {'WR%':>5s} | affected only: base R -> variant R | Δ h1 / Δ h2") + b = stats(base) + for v, (r, f) in results.items(): + s = stats(r); aff = f == 1 + d1 = r[half == 1].sum() - base[half == 1].sum(); d2 = r[half == 2].sum() - base[half == 2].sum() + print(f"{v:12s} {s['n']:5d} {aff.sum():8d} {s['total']:+9.2f} {s['total']-b['total']:+9.2f} {s['pf']:5.2f} {s['dd']:6.2f} {s['wr']:5.1f} | " + f"{base[aff].sum():+8.2f} -> {r[aff].sum():+8.2f} (n={aff.sum()}) | {d1:+.2f} / {d2:+.2f}") + # damage table: first burst while open (base path), with vs against + print("\n== DAMAGE: first violent bar that hits an OPEN trade (base exits) ==") + rows = [] + for i, t in book.iterrows(): + d = 1.0 if t.direction == "BUY" else -1.0; risk = abs(t.entry - t.sl) + xp, held, reason, _ = simulate(bars, pos[i], t.direction, t.entry, t.sl, arm, dist, "base") + j0 = pos[i]; j1 = j0 + held + for j in range(j0 + 1, j1 + 1): # bursts after the entry bar, up to and including the exit bar + if bars["burst"][j]: + against = d * (bars["C"][j] - bars["O"][j]) < 0 + mtm_before = (d * (bars["C"][j - 1] - t.entry) - COST) / risk + mtm_after = (d * (bars["C"][j] - t.entry) - COST) / risk + rows.append(dict(against=against, bars_open=j - j0, mtm_before=mtm_before, mtm_after=mtm_after, + final=r_of(t.direction, t.entry, t.sl, xp), ended_here=(j == j1), hour=bars["chour"][j], + sched=bars["chour"][j] in (0, 1, 12, 13))) + break + dmg = pd.DataFrame(rows) + if len(dmg): + g = dmg.groupby("against").agg(n=("final", "size"), bars_open_med=("bars_open", "median"), + mtm_before=("mtm_before", "mean"), mtm_after=("mtm_after", "mean"), + final=("final", "mean"), ended_on_that_bar_pct=("ended_here", lambda s: 100 * s.mean()), + in_sched_hours_pct=("sched", lambda s: 100 * s.mean())) + print(g.round(3).to_string()) + ag = dmg[dmg.against] + print(f"\nopposing bursts: {len(ag)} trades ({100*len(ag)/len(book):.1f}% of the book); " + f"mean R just before {ag.mtm_before.mean():+.3f} -> just after {ag.mtm_after.mean():+.3f} -> final {ag.final.mean():+.3f}; " + f"{100*(ag.final<0).mean():.0f}% end as losers; {100*ag.ended_here.mean():.0f}% are stopped ON that bar") + print(f" those with an open PROFIT before the burst (mtm>0): n={int((ag.mtm_before>0).sum())}, final mean {ag[ag.mtm_before>0].final.mean():+.3f}, " + f"{100*(ag[ag.mtm_before>0].final<0).mean():.0f}% end as losers") + print(f" by hour of the opposing burst (n, mean final R): " + ", ".join(f"{h:02d}:{n}/{m:+.2f}" for h, n, m in + ag.groupby("hour").final.agg(["size", "mean"]).reset_index().itertuples(index=False))) + + +if __name__ == "__main__": + main() diff --git a/backtest/research/sizing_campaign_2026-09.md b/backtest/research/sizing_campaign_2026-09.md new file mode 100644 index 0000000..f681e92 --- /dev/null +++ b/backtest/research/sizing_campaign_2026-09.md @@ -0,0 +1,348 @@ +# Stake-sizing campaign — 2026-09-05 (3-year Oanda window, $0.65) + +## Purpose and owner decisions +Owner asked (2026-09-05): predict direction and choose order size better; protect against +trend exhaustion at signal time; map hours/days where institutions move gold violently; +backtests ≤ 3 years. Decisions taken in planning: +- Order size = a per-signal RELATIVE stake ("Stake: 0.7× normal"), not account-based units. +- Exhaustion protection = RISK LESS, DON'T SKIP (every hard exhaustion block tested since + 2026-07 removed net-positive trades — the winners are the stretched runs). +- Model only M5 and M15; H1/H4 measured descriptively. +- scikit-learn / lightgbm allowed in ./venv for research only; the shipped artefact is + numpy-only JSON. Neither library enters requirements.txt. +- No tuned threshold, gate or exit changes (rule 1). Sizing is a post-decision overlay computed + by `position_sizing.build_features` / `stake_multiplier` INSIDE `analyze_from_dataframes`, so + live and backtest use the same call on the same objects (rule 4). + +## Harness +`backtest/gpu_backtest.py --pair XAUUSD --start 2023-09-05 --end 2026-09-05 --spread 0.65 +--workers 12`, pair-factory config (no --set/--quality/--strategies/--confirm-tfs), NumPy CPU +backend (CuPy cannot locate CUDA_PATH on this box; the 2026-09-03 pinned books were also CPU). +Cache: `oanda_XAU_USD_{5min,15min,1h,4h}_20230705_20260905.parquet`, `…_1d_20221110_20260905`. +Feature books add `--export-features` (65 `f_*` columns + `stake_multiplier` + `r_sized`; the +default export stays the pinned 63-column shape). Outputs in `backtest/research/sizing_2026-09/` +(gitignored). + +## Baseline (control books, unmodified backtester, 2026-09-05 23:36–23:41 UTC) +| TF | trades | total R | WR | PF | max DD | +|---|---|---|---|---|---| +| M5 | 857 | +161.32 | 53.7% | 1.53 | 9.85 | +| M15 | 1086 | +148.69 | 59.1% | 1.37 | 15.47 | +| H1 | 255 | +70.52 | 64.3% | 1.98 | 4.43 | +| H4 | 113 | +22.68 | 58.4% | 1.63 | 5.39 | + +### Behaviour-neutrality checks (2026-09-05 23:57 UTC) +- 12-month M5 canonical window (2025-08-14 → 2026-08-14, $0.65) re-run on the modified tree: + `book_diff` vs `backtest/oanda_m5_12mo_065_nopb_20260903.csv` = ΔR +0.0, Δtrades +0, and the + two CSVs have the SAME MD5 (`a28fed4becd056466c26d6a947e1a0e5`). Record-keeping fields, + `SizingConfig`, and the `--export-features` hook change nothing when the flag is off. +- Feature books: the 63 shared columns are value-identical to the control books on every TF + (`assert_frame_equal`). A column-ORDER quirk was found and fixed the same day: pandas orders + export columns by first appearance, so when the first trade of a book lacks an indicator + (the 3-year M5 book's first trade had no candlestick pattern) that indicator's columns were + appended after the extras. `trades_to_dataframe` now forces the extras to the tail; test + extended with that exact case (435 tests green). + +## Pre-registered acceptance rule (written 2026-09-05, BEFORE any model fit) +A per-timeframe stake-multiplier model may ship only if, evaluated on OUT-OF-SAMPLE quarters +only (all calendar quarters after a 12-month burn-in, each predicted by a model trained +exclusively on trades whose entry time is ≥ 300 primary bars before that quarter's start, +expanding window, standardisation fit on train only), ALL of the following hold: +1. Sized total R > flat total R in BOTH halves of the OOS period (split at the median OOS + entry time). +2. Sized total R > flat total R over the full OOS period. +3. Sized max drawdown (0-prepended cumulative sized-R curve, as `compute_metrics`) ≤ flat max + drawdown × 1.10. +4. Sized OOS book has PF ≥ 1.3 and positive total R in both halves (the rule-1 floor). +5. Real OOS lift > the 95th percentile of the lift distribution from 50 label-shuffle refits + (R permuted within training windows only; test labels and walk-forward geometry unchanged). +6. Mean multiplier over OOS trades within 1.00 ± 0.05; every multiplier in [0.5, 1.5]. +Plus the mechanical check: the `--sizing` full-window replay reproduces the flat book's +entry/exit/`r_multiple` columns exactly (sizing changes no entries). +Model family and size map (fixed): primary = ridge regression on winsorised R (clip to +[−1.5, +4]); secondary = L2 logistic on R>0 (ships only if the primary fails and it passes); +simplicity baseline = one-feature model on `f_is_buy` (the shipped model must beat its OOS +lift, else the simpler one ships if it passes); upper bound = lightgbm on all `f_*` (never +ships; if its OOS lift is inside the shuffle null, stop); ablation = exhaustion-block-only model. +Map: `m = 0.5 + 1.0 × clip((z − q10)/(q90 − q10), 0, 1)` with q10/q90 from TRAIN predictions, +times a normaliser making the mean train multiplier 1.0, clipped to [0.5, 1.5], rounded to 0.05. +`SHIPPED_FEATURES` (≤ 12 per TF, ≥ 2 from the exhaustion block) are chosen from the Stage-B +sign-stable screen on the 3-year feature books and recorded below BEFORE the first fit; any +later change restarts the shuffle control. If any clause fails on a timeframe, that timeframe +ships nothing. No widening the clip, changing the split, dropping the purge, re-picking +features or adding quarters to rescue a result. + +## Pre-registered feature lists +Derived MECHANICALLY (script in the session log, saved as `sizing_2026-09/preregistered_features.json`): +sign-stable `f_*` features from the Stage-B screen (top-minus-bottom quintile mean-R gap has the +same sign in both halves, both halves ≥ 10 trades per bucket), zero-variance columns removed, +ranked by min(|gap_h1|, |gap_h2|), top 12; the ≥ 2-from-the-exhaustion-block clause was already +satisfied on both TFs. Written 2026-09-06 00:05 UTC, before any Stage-D fit. +- **M5 (29 sign-stable candidates):** f_sl_atr, f_confidence, f_rsi, f_mtf_c1, f_swing_dist_atr, + f_ret_48_atr, f_tag_no_pullback, f_rsi_dir, f_first_friday, f_mtf_agreement, + f_exhaustion_count, f_ret_24_atr. Exhaustion-block members: f_swing_dist_atr, + f_tag_no_pullback, f_exhaustion_count (all three run POSITIVE in both halves). +- **M15 (37 sign-stable candidates):** f_rsi, f_bb_pos, f_day_of_month, f_tag_ema200_near, + f_atr_pct, f_is_buy, f_confidence, f_close_pos_in_bar, f_tag_weak_pullback, f_tag_exhaustion, + f_penalty_total, f_ret_6_atr. Exhaustion-block members: f_tag_exhaustion, f_penalty_total + (both NEGATIVE in both halves; f_tag_weak_pullback and f_tag_ema200_near also negative). +Exhaustion-only ablation list (fixed by the plan): f_swing_dist_atr, f_ema21_dist_atr, +f_ema200_dist_atr, f_exhaustion_count, f_penalty_total, f_tag_bb_overextended, +f_tag_rsi_extended, f_tag_no_pullback, f_tag_exhaustion. + +## Stage B — book breakdown (3-year feature books) +Tool: `backtest/research/book_report.py` (outputs in `sizing_2026-09/report_{5min,15min,1h,4h}/`). +Halves = median TRADE split (M5 cut 2025-12-12 because trade density doubled in the last year; +M15 cut 2025-04-25; H1 2025-04-09). + +**M5 (857 trades, +161.3R, PF 1.53, DD 9.85; halves +51.0R PF 1.33 / +110.3R PF 1.74; max 11 +wins / 8 losses in a row; longest flat stretch 88 trades).** Mix: GENERIC 469, XAU_CP 388, BUY +765 / SELL 92 (SELLs +11.8R, positive both halves). Grade: A +0.26R/trade (227) > B +0.17 (608) +> C −0.01 (21). Session (decision hour): NY 17–22 best (+0.31R/trade, PF 1.88), overlap 13–16 +weakest (+0.08, PF 1.22) — both stable. Weekday: Wed and Fri best and stable; Mon/Tue flip sign. +Feature screen: 29 sign-stable `f_*` features. **Every stretch measure is POSITIVE**: wider +stop in ATR (+0.22/+0.23 gap), higher RSI (+0.20/+0.19), farther from the last swing +(+0.18/+0.27), 48-bar run-up (+0.18/+0.20), the `no_pullback` tag (+0.17/+0.28), more exhaustion +tags (+0.12/+0.24). Only ATR-relative-to-500-bar-mean is negative (−0.11/−0.12: entries during +a volatility spike do worse). "Risk less when stretched" has NO support on M5 — a model that +honours the data will size stretched M5 trades UP, not down. + +**M15 (1086 trades, +148.7R, PF 1.37, DD 15.47; halves +65.8R / +82.9R; max 12 wins / 8 losses; +flat stretch 131 trades).** All GENERIC; BUY 739 (+158.4R, PF 1.62) vs SELL 347 (−9.7R, PF 0.93; +−19.2R in half 1, +9.4R in half 2). Grade NOT monotone: C +0.45R/trade (82, PF 2.41) > A +0.14 +(481) > B +0.08 (516) — stable in both halves (matches the 2026-08-04 quant review). Session: +Asian and NY ≈ +0.18–0.20R/trade, London and overlap ≈ +0.09–0.10 (all stable). Weekday: +Wednesday flat/negative in both halves (229 trades, −0.6R); Thu/Fri flip sign between halves. +Feature screen: 37 sign-stable features. Here the exhaustion block runs NEGATIVE: `exhaustion` +tag (−0.19/−0.10), `weak_pullback` tag (−0.24/−0.11), `ema200_near` tag (−0.17/−0.32), penalty +total (−0.16/−0.10), ATR percentile (−0.20/−0.15); while raw RSI (+0.31/+0.24) and Bollinger +position (+0.36/+0.22) are positive. So on M15 "stretched by the aggregator's own penalty +logic" is bad while "strong momentum reading" is good — the sizing model gets both. + +**H1 (255 trades, +70.5R, PF 1.98, DD 4.43; halves +28.7R PF 1.79 / +41.9R PF 2.17)** and +**H4 (113 trades, +22.7R, PF 1.63, DD 5.39)** are descriptive only. H1 is all-BUY by +construction (shorts blocked 2026-09-03); Mon/Thu strongest, Friday the only losing weekday +(−1.4R/50, flips between halves); 03:00 UTC decision hour is the best single hour (+20.0R/40, +stable) and 04:00–05:00 negative. + +**Labelling quirk (pre-existing, not fixed):** 33 M5 trades carry the backtester `strategy` +tag GENERIC while their setup strategy is XAU_ATR (`f_strat_atr = 1`); the CSV tag and the +setup's own string disagree for the ATR path. Labels only — R is unaffected. + +**Stop-trigger check: PASSED** (both TFs have many sign-stable features) → proceed to Stage D. + +## Stage C — institutional burst / time map +Tool: `backtest/research/burst_map.py` on the 3-year raw bars (`burst_5min/`, `burst_15min/` +under `sizing_2026-09/`). Burst = bar range ≥ K × prior Wilder ATR(14). Base rate at K=3: +0.86 % of M5 bars (1840), 1.19 % of M15 bars (846). + +**Where the violent bars cluster (bar-CLOSE hour, UTC, K=3 burst % of bars):** +- 12:00–13:00 (US data window): M5 2.4 % / 2.3 %, M15 3.7 % / 5.0 % — 3–4× the base rate; + Friday 12:00–13:00 is the densest weekday cell on M5 (3.4 % / 3.1 %). Largest mean 1-hour + dollar moves are at close-hours 13–15 (M5: $7.8 / $9.4 / $8.5 vs ≈$5 elsewhere). +- 01:00 (Asia open): M5 2.9 %, M15 3.3 %. Also 05:00 (Tokyo/London hand-over) ≈1.2 % on M5. +- 22:00–23:00 (post-settlement thin book): ≈1.3–1.4 % on M5; Sunday reopen 22–23 h is the most + burst-prone slot of all (2.4–3.0 % of bars) but the calendar already blocks Sunday entries + before 18:00 NY. +- Quietest hours: 03:00–04:00 and 16:00–17:00 (0.14–0.3 %). + +**Post-burst drift (in the burst's own direction, ATR units):** ≈ 0 at every horizon on both +TFs (M5 6/12/24/48-bar means −0.01/−0.01/+0.09/+0.05; M15 −0.08/+0.14/+0.18/+0.17; % positive +46–51 %). London-session bursts on M5 show a mild short-horizon reversal (42 % positive at 6 +bars). No continuation or reversal edge worth a rule. + +**Do bursts hurt the system's entries? No.** Trades DECIDED within 12 bars after a K=3 burst: +M5 149 trades, mean +0.199R, PF 1.62 vs +0.186R / PF 1.52 for the rest; M15 242 trades, ++0.160R / PF 1.47 vs +0.130R / PF 1.34. At K=4/5 the after-burst group is better still +(M5 +0.276R PF 1.98; M15 +0.259R PF 1.80). This re-confirms the 2026-07-30 big-candle-gate +rejection with 3 years of data: expansion bars are part of the entry signal, not a danger. +(Group (b) "burst while open" is survivorship-shaped — winners live longer — and is not evidence.) + +**The 40 largest 1-hour moves** are almost all one episode (2026-01-28 → 2026-02-06, incl. a +−$394 hour at 15:45 UTC on 2026-01-29, and 2026-03-23), not recurring clock slots; only 3 of 40 +fall in the 12:30 data slot and 1 in the Wednesday 18:00 central-bank slot (slot labels only — +no economic calendar file exists in the repo). Violent moves are episodic regime events; the +recurring, predictable clustering is the 12:00–15:00 UTC US-data window and the 01:00 Asia open. + +**Use in this campaign:** `f_bars_since_burst` and the hour/session features are already in the +feature set, so Stage D can weigh them as a stake input; no entry gate is proposed (rule 1). + +## Stage D — walk-forward results (2026-09-06 00:20 UTC) +Driver: `backtest/research/sizing_model.py` (numpy closed-form ridge + Newton logistic; lightgbm +4.7 + scikit-learn 1.9 installed in ./venv for the upper bound only; NOT in requirements.txt). +Outputs: `sizing_2026-09/sm_{m5,m15}_{ridge,logistic}/` (summary.txt, oos_trades.csv, +quarters.csv, coefs_by_quarter.csv, shuffle_lifts.csv). Validation of the driver: planted +synthetic effect recovered (lift +116.7R, shuffle rank 100 %, all clauses PASS); pure-noise +book correctly FAILS (rank 32 %); ridge matches `lstsq` to 2e-16; logistic gradient < 3e-15. + +**M5 (5 OOS quarters 2025Q3→2026Q3, 619 trades — 2024Q4–2025Q2 skipped by the pre-registered +min-train rule because only 96/119/142 trades precede them; 2026Q1 alone is +92R of the +125R +flat OOS):** +| model | flat R | sized R | lift | PF | DD | mean m | shuffle p95 | rank | verdict | +|---|---|---|---|---|---|---|---|---|---| +| ridge (primary) | +125.41 | +121.98 | −3.43 | 1.63 | 10.69 | 0.891 | +13.69 | 48 % | FAIL (1,2,5,6) | +| logistic | +125.41 | +128.40 | +2.99 | 1.66 | 8.70 | 0.900 | +14.39 | 62 % | FAIL | +| isbuy baseline | +125.41 | +123.22 | −2.20 | 1.55 | 11.92 | 1.000 | +7.05 | 4 % | FAIL | +| exhaustion-only | +125.41 | +118.03 | −7.38 | 1.59 | 10.88 | 0.918 | +14.36 | 28 % | FAIL | +| lightgbm (all 65) | +125.41 | +116.05 | −9.36 | 1.51 | 11.64 | 1.007 | +7.60 | 10 % | FAIL | +Ridge weights (standardised, 5 quarters): `f_tag_no_pullback` +0.018 sign-stable 5/5 — the +"no pullback" flag INCREASES the stake; `f_swing_dist_atr` and `f_exhaustion_count` ≈ 0 and +sign-unstable. Exhaustion-only sizing LOSES 7.4R. **A clean null: nothing in the 65 features +predicts M5 trade outcomes beyond noise, linear or non-linear.** + +**M15 (8 OOS quarters 2024Q4→2026Q3, 762 trades; flat +115.93R PF 1.42 DD 13.73):** +| model | sized R | lift | PF | DD | mean m | shuffle p95 | rank | clauses | +|---|---|---|---|---|---|---|---|---| +| ridge (primary) | +142.04 | +26.11 | 1.53 | 14.46 | 0.980 | +28.50 | 88 % | 1,2,3,4,6 PASS · **5 FAIL** | +| logistic | +144.78 | +28.84 | 1.53 | 13.97 | 1.002 | +31.89 | 92 % | 1,2,3,4,6 PASS · **5 FAIL** | +| isbuy baseline | +139.38 | +23.45 | 1.50 | 16.44 | 1.007 | +24.47 | 58 % | FAIL (3, 5) | +| exhaustion-only | +123.83 | +7.90 | 1.45 | 14.85 | 1.010 | +19.77 | 70 % | FAIL | +| lightgbm (all 65) | +123.87 | +7.94 | 1.44 | 14.74 | 0.996 | +17.66 | 60 % | FAIL | +Ridge weights sign-stable in 8/8 quarters: `f_tag_exhaustion` −0.022, `f_tag_ema200_near` +−0.033 (both SHRINK the stake — the "risk less when stretched" direction), `f_rsi` +0.028, +`f_bb_pos` +0.030, `f_is_buy` +0.029, `f_close_pos_in_bar` +0.022, `f_ret_6_atr` +0.022; +λ = 1000 every quarter (heavy shrinkage — the score is weak and the map is essentially a rank +transform). Sized R beat flat in 6 of 8 quarters and in both halves, BUT the label-shuffle +null has p95 = +28.5R: a random re-weighting of these 12 pre-screened features produces the +same lift 12 % of the time, and the one-feature BUY/SELL baseline already delivers +23.5R of +the +26.1R. **The model's information beyond "buys are better than sells" (≈ +2.7R over 762 +trades) is indistinguishable from noise.** + +## Verdict (pre-registered rule applied, no rescue attempted) +**FAIL on both timeframes → no stake multiplier ships.** No artifact, no `sizing.enabled`, no +alert line, no `--sizing` backtester flag. What remains in the tree is behaviour-neutral +research plumbing: `position_sizing.build_features` (65 decision-time features; the multiplier +path returns 1.0 and no artifact exists), `SignalReport.penalty_tags/penalty_total/ +momentum_fading` (record-keeping), `TradeSetup.stake_multiplier` (None; reserved), the +`--export-features` flag, `book_report.py`, `burst_map.py`, `sizing_model.py`, and Module 29 +tests. `SizingConfig` was REMOVED again before merge (rule 8: nothing reads it). + +**What the owner should take from this (plain language):** +1. On the 5-minute chart there is nothing to size on. Stretched trades are the good ones; the + system's exhaustion flags carry no usable information about the outcome; a flexible model + given all 65 facts does worse than flat sizing out of sample. +2. On the 15-minute chart the only robust sizing signal is BUY vs SELL, which the gates already + act on. The exhaustion flag does point the "risk less" way in every one of 8 unseen + quarters, but the money it would make (≈ +3R per year) is inside noise at 1,086 trades. +3. "Predict direction better" with a learned second opinion is closed at this sample size: + the strongest non-linear model (all features, gradient boosting) fails on both charts. + +**Open avenues (not rescues — each needs a NEW pre-registration):** +- Re-run the M15 exhaustion-tag sizing when the book has ≈ 2× the trades (≈ 2028) or on a + sealed forward sample; the sign is stable, the magnitude is not proven. +- The shuffle null is wide because the size map rank-stretches a weak score to 0.5–1.5×; a + narrower map (e.g. 0.75–1.25×) would shrink both lift and null — pre-register before testing. +- Session and weekday structure (M5: NY best / overlap weakest; M15: Wednesday flat) is stable + across halves but was rejected as a gate repeatedly; as a *size* input it was inside the + 12-feature model and did not survive the null. + +## Addendum 2026-09-06 — violent bars: signs before them, and protecting OPEN trades +Owner follow-up: "we might have an order running for hours before a violent move; we want +protection." Tools: `backtest/research/burst_precursors.py` (signs before a burst) and +`backtest/research/open_trade_bursts.py` (exact replay of every book trade on raw bars with the +backtester's exit model — reproduces `r_multiple` on 100 % of trades — plus protection variants). +Outputs: `sizing_2026-09/burst_precursors_{m5,m15}.txt`, `open_trade_bursts_{m5,m15}.txt`. + +**Signs before a burst (both TFs, both halves stable):** the clock (close-hours 01, 12, 13 UTC +hold ~40 % of bursts in 13 % of bars); tick activity in the last 3 bars vs 50-bar mean (top +quintile ≈ 2× rate); the last bars already wider than normal (top quintile ≈ 2.2×; the +tightest quintile ≈ 0.3–0.5× — NOT calm-before-storm at bar scale); a move already under way +over the past hour (≈ 1.6×); a burst minutes ago (≈ 1.4×). Out of sample (fit half 1, score +half 2) a logistic model reaches AUC 0.75–0.77 for the next bar; the top 5 % of moments carry a +3.4–5.5 % burst chance vs 0.8–1.1 % base, and 24 % vs 9 % (M5) for "within the next hour". +Burst direction continues the prior hour's drift 61 % of the time. Signs say "ripe", not "now". + +**Damage to open trades (base exits):** M5 — 164 trades (19 %) meet an opposing burst while +open: mean open R +0.42 just before → −0.06 just after → final +0.01; 43 % are stopped on that +bar; 54 % end as losers. M15 — 246 trades (23 %): +0.27 → −0.20 → final −0.12; 39 % stopped on +the bar. Trades already in profit before the burst still finish positive on average (M5 +0.53, +M15 +0.27). Bursts WITH the trade (n 176 / 246) finish at +1.11 / +0.83R — they are where the +winners come from. + +**Protection variants — ALL negative on both TFs, both halves:** +| variant | M5 ΔR (h1/h2) | M15 ΔR (h1/h2) | +|---|---|---| +| flatten before US-data hours (12–13 UTC) | −31.9 (−13.6/−18.3) | −103.0 (−67.4/−35.6) | +| flatten before Asian open (00–01 UTC) | −38.7 (−22.0/−16.7) | −73.0 (−42.0/−30.9) | +| protective 0.5R stop through the data hours | −29.6 (−12.3/−17.3) | −82.3 (−55.5/−26.8) | +| protective 0.5R stop through the Asian open | −19.2 (−9.5/−9.7) | −35.1 (−17.3/−17.8) | +| exit next open after an opposing burst | −19.1 (−6.5/−12.5) | −6.9 (−11.9/+5.0) | +| 0.5R protective stop after an opposing burst | −9.0 (−3.0/−6.0) | −8.3 (−15.5/+7.3) | +The existing stop/trail already contains the opposing-burst damage (≈ 0.4–0.5R per hit); every +pre-emptive rule removes far more from the with-trade bursts than it saves. **Verdict: no +open-trade protection rule ships; do not re-propose flatten/tighten-before-window or +react-to-burst exits without a new mechanism.** Display-only "violent-move conditions" note on +alerts is the only harmless use of the precursor signs. + +## Addendum 2026-09-06 — calendar events, indicator states and time slots before violent bars +Tools: `backtest/research/calendar_impact.py` (needs `backtest/cache/calendar.parquet`, scraped +2023-09-01→2026-09-10 with `ff_calendar_scraper.py`: 15,124 events, 1,055 high-impact USD in +window) and `backtest/research/burst_indicator_map.py` (states from the backtester's own +`precompute_indicators`, taken at t−1). Outputs `sizing_2026-09/calendar_impact_{m5,m15}.txt`, +`burst_indicator_map_{m5,m15}.txt`. Burst = range ≥ 3× prior ATR14; lifts quoted are stable in +BOTH halves on both TFs unless noted. + +**Calendar.** The M5 bar containing a high-impact USD release averages 4.4× normal range; 54 % +of such releases produce a burst within the hour (base 9 %). By event: Non-Farm Payrolls (with +Unemployment Rate / Avg Hourly Earnings) 97 %, 9.3× bar, $11.9 mean 1-h move; CPI 91 %, 8.3×, +$9.0; FOMC decision/statement 87.5 %, 7.1×, $18.2 (press conference a further $20 hour); FOMC +projections 92 %; Retail Sales 64 %; PPI 54 %; Core PCE 50 %; Unemployment Claims 44 %; GDP +45–58 %; ISM PMIs 26 %; Powell speaks 33 %; Trump speaks 21 %; UoM sentiment ≈ 5 %. 20 % (M5) +/ 31 % (M15) of ALL bursts fall within ±30 min of a high-impact USD release (3.7–4.2 % of time); +in the 30 min AFTER one, next-bar burst odds are 8–17 % (7–16× base); 15 min before, 2–3 %. +Open trades that held through a high-impact release did BETTER (M5 +0.41R vs +0.16R; M15 +0.27 +vs +0.07), even when the first 15-min move went against them (M5 +0.09, M15 +0.22). Flattening +5 min before a release: −5.4R (M5) / −59.3R (M15); 0.5R protective stop through the hour: +−2.4R / −43.1R. **Events are the biggest single cause of violent bars and still not a reason +to touch an open trade.** + +**Indicator states before a burst (next-bar lift, M5 / M15).** Price OUTSIDE the Bollinger +bands: below 2.9× / 1.8–2.9×, above 2.1× / 2.2×, middle 0.2–0.8 band 0.7×. RSI < 30: 2.6–2.9× / +1.8–2.2×; RSI ≥ 70: 1.9–2.1× / 1.6–1.9×; RSI 40–60: 0.7× (calmest). Close outside the Keltner +channel: 1.4–2.2×; inside 0.76×. ADX RISING > 3: 1.4–1.5× / 1.2×; ADX FALLING < −3: 0.5–0.6× +(safest state); the ADX level itself carries nothing. MACD histogram negative and still falling +1.3× / 1.2–1.4×; histogram fading (either sign) 0.8×. Fully bearish 4-EMA stack 1.3–1.4× / +1.3×; "mostly bull" 0.8× on M5. Regime: transitioning 1.4–1.6× / 1.5–1.6×, ranging 0.88×. +Volatility regime: extreme 1.5–1.6× / 1.4–1.9×, low 0.6–0.8×. ROC beyond ±0.3 %: 1.2–1.9× +(>1 %: 1.3–10×, n small). Bollinger-width squeeze: 0.59× on M5 (LESS likely next bar — narrow +bands are not a warning); TTM squeeze (bands inside Keltner) 1.6× on M15 only. EMA200 distance, +day of month, first-Friday-as-a-day: nothing stable (first Friday matters only through its +12:30 slot). + +**Time slots (M5 next-bar burst %, base 0.86 %):** Fri 01:xx 3.4 %, Fri 12:xx 3.4 %, Tue/Wed +01:xx 3.2 %, Thu 12:xx 3.1 %, Fri 13:xx 3.1 %, Sun 22:xx reopen 3.0 %, Thu 01:xx 2.7 %, Wed +12:xx 2.6 %, Sun 23:xx 2.4 %, Tue 00:xx 2.3 % (all 2.5–3.9× base). Quietest: 03–04 and 16–17. + +**Direction of the burst (share going UP):** fully-bull stack 62 % (M5) / 50 % (M15); fully-bear +stack 26 % / 29 %; RSI < 30 → 22 % / 30 % up (i.e. 70–78 % DOWN); RSI ≥ 70 → 64 % / 52 % up; +MACD neg & falling → 30 % / 34 % up; trending_down regime → 26 % / 26 % up; trending_up → 60 % / +46 % up. Bearish states foretell a DOWN burst reliably on both TFs; bullish states foretell an +UP burst on M5 only. + +**Reading:** violent bars arrive when price is already at an extreme with momentum accelerating +(outside the bands, RSI extreme, ADX rising, MACD still falling, transitioning regime, extreme +volatility) and/or a release is minutes away — "the storm is already brewing", not "calm before +the storm". Calm mid-band bars with fading momentum and falling ADX are the safest. None of it +changes trade handling (protection tests above); it is display material for a warning line. + +## Addendum 2026-09-06 — violent-move WATCH rule (display-only feature, `violent_watch.py`) +Owner asked for a warning. Delivery: a line on signal alerts + a standalone heads-up to the owner +(once per hour per trigger set, daily cap), dark by default (`violent_watch` config block). +The first draft rule ("hot" = calendar | ≥3 indicator readings | clock & ≥2 readings) was +evaluated bar-by-bar on 3 years (`backtest/research/watch_rule_eval.py`, scratch variant +sweep): it flagged 27 % of bars hot at only ≈2× odds and would have sent ≈13 heads-ups/day on +M5 — rejected. Sweep of 11 variants (M5 / M15, both halves): +| hot definition | bars hot | next-bar lift h1/h2 | next-hour rate | hot hours/day | +|---|---|---|---|---| +| calendar only (hi-impact USD ±30 min) | 3.4 % | 6.7×/4.4× · 9.1×/6.4× | 31 % · 28 % | 2.6 | +| clock slot only | 18.8 % | 3.1×/2.0× · 3.6×/2.6× | 16 % · 13 % | 5.3 | +| ≥2 price-extreme readings | 14.6 % | 2.3×/2.2× · 1.8×/2.0× | 11 % · 6 % | 10.7 | +| **calendar ∨ (clock ∧ ≥1 price-extreme reading)** | **8.6 %** | **4.9×/3.1× · 5.9×/4.2×** | **21 % · 17 %** | **4.1 · 3.5** | +| draft (cal ∨ readings≥3 ∨ clock∧readings≥2) | 26.9 % | 2.4×/2.0× · 2.4×/2.1× | 13 % · 8 % | 13.0 | +**Shipped rule:** price-extreme readings = outside the Bollinger bands, RSI < 30 or ≥ 70, close +outside the Keltner channel (the neutral readings — ADX rising, extreme volatility, transitioning +— are dropped from the level logic; they are common and only ≈1.5×). `hot` = high-impact USD +release within 30 min or 30 min after, OR a hot clock slot with ≥ 1 price-extreme reading; `watch` += clock slot alone or ≥ 2 price-extreme readings (≈2× odds). Direction sentence from the bear/bull +polarity of the readings plus MACD acceleration and trend regime ("DOWN" on both TFs, "UP" on M5 +only). Heads-up only at `hot`. Expected ≈4 hot hours/day → ≤4 heads-ups/day after per-hour dedup. diff --git a/backtest/research/sizing_model.py b/backtest/research/sizing_model.py new file mode 100644 index 0000000..47cd7f7 --- /dev/null +++ b/backtest/research/sizing_model.py @@ -0,0 +1,961 @@ +"""Stage-D stake-sizing model driver (2026-09 campaign). + +Walk-forward fit + evaluation of a per-signal stake multiplier on a trade +book exported by ``gpu_backtest.py --export-features`` (63 base columns + +``stake_multiplier``/``r_sized`` + 65 ``f_*`` decision-time features). + +The protocol implemented here is the one pre-registered in +``backtest/research/sizing_campaign_2026-09.md`` and is NOT to be varied to +rescue a result: + +* expanding walk-forward, 12-month burn-in, calendar-quarter refits, a + ``purge_bars × bar_minutes`` gap between the end of train and the start of + the test quarter, standardisation fit on train only; +* models: ridge on winsorised R (primary), L2 logistic on ``R > 0`` + (secondary), a one-feature ``f_is_buy`` simplicity baseline, an + exhaustion-block-only ablation, and an optional lightgbm upper bound; +* size map ``m = round_0.05(clip(normaliser × (0.5 + clip((z−lo)/(hi−lo),0,1)), + 0.5, 1.5))`` with ``lo``/``hi``/``normaliser`` calibrated on TRAIN + predictions only; +* a label-shuffle null (training labels permuted, test labels untouched); +* six acceptance clauses, printed PASS/FAIL individually. + +Usage: + python backtest/research/sizing_model.py --book BOOK.csv --bar-minutes 5 \ + --features f_a,f_b,... [--model ridge|logistic] [--out DIR] \ + [--n-shuffle 50] [--skip-lgbm] \ + [--write-artifact PATH --tf 5min --confirm-tfs 15min,1h] + +Analysis tooling only — never imported by the live code. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import subprocess +import sys +import warnings +from pathlib import Path +from typing import Dict, List, Optional, Sequence, Tuple + +import numpy as np +import pandas as pd + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from position_sizing import FEATURE_ORDER, FEATURE_VERSION # noqa: E402 (read-only import) + +# ── Pre-registered constants (flag any change loudly) ────────────────── +WINSOR_LO, WINSOR_HI = -1.5, 4.0 +LAMBDA_GRID: Tuple[float, ...] = (1.0, 10.0, 100.0, 1000.0) +CV_FOLDS = 5 +MIN_TRAIN = 150 +MAP_Q_LO, MAP_Q_HI = 0.10, 0.90 +MAP_OUT_LO, MAP_OUT_HI = 0.5, 1.5 +MAP_CLIP_LO, MAP_CLIP_HI = 0.5, 1.5 +MAP_ROUND_TO = 0.05 +MEAN_MULT_TOL = 0.05 +DD_TOL = 1.10 +PF_FLOOR = 1.3 +NEWTON_MAX_ITER = 50 +NEWTON_TOL = 1e-8 + +#: The exhaustion-block ablation's fixed feature list. +EXHAUSTION_FEATURES: Tuple[str, ...] = ( + "f_swing_dist_atr", + "f_ema21_dist_atr", + "f_ema200_dist_atr", + "f_exhaustion_count", + "f_penalty_total", + "f_tag_bb_overextended", + "f_tag_rsi_extended", + "f_tag_no_pullback", + "f_tag_exhaustion", +) +ISBUY_FEATURES: Tuple[str, ...] = ("f_is_buy",) + +#: lightgbm never ships; running the full shuffle null through 200-tree fits +#: costs minutes for a number nobody acts on, so its null is capped here. +#: DEVIATION FROM THE PROTOCOL CONSTANT — printed in the summary when used. +LGBM_MAX_SHUFFLE = 10 + +LGBM_PARAMS = dict( + n_estimators=200, + learning_rate=0.03, + num_leaves=7, + min_child_samples=30, + subsample=0.8, + colsample_bytree=0.8, + reg_lambda=10, +) + +_LINES: List[str] = [] + + +def say(msg: str = "") -> None: + """Print and tee into the summary buffer.""" + print(msg) + _LINES.append(msg) + + +# ══════════════════════════════════════════════════════════════════════ +# Book loading +# ══════════════════════════════════════════════════════════════════════ +def load_book(path: Path) -> pd.DataFrame: + """Read a feature book, parse ``entry_time`` UTC, drop NaN-R rows.""" + df = pd.read_csv(path) + if "entry_time" not in df.columns or "r_multiple" not in df.columns: + raise SystemExit(f"{path}: not a trade book (no entry_time/r_multiple)") + df["entry_time"] = pd.to_datetime(df["entry_time"], utc=True, format="mixed") + n0 = len(df) + bad = ~np.isfinite(pd.to_numeric(df["r_multiple"], errors="coerce").to_numpy(float)) + if bad.any(): + say(f"WARNING: dropping {int(bad.sum())}/{n0} rows with non-finite r_multiple") + df = df.loc[~bad].copy() + df["r_multiple"] = df["r_multiple"].astype(float) + df = df.sort_values("entry_time", kind="mergesort").reset_index(drop=True) + return df + + +def resolve_features(df: pd.DataFrame, names: Sequence[str], label: str) -> List[str]: + """Validate a requested feature list against the book and FEATURE_ORDER.""" + missing = [n for n in names if n not in df.columns] + if missing: + raise SystemExit(f"{label}: columns absent from book: {', '.join(missing)}") + unknown = [n for n in names if n.startswith("f_") and n not in FEATURE_ORDER] + if unknown: + say(f"NOTE: {label}: {', '.join(unknown)} not in position_sizing.FEATURE_ORDER") + return list(names) + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def git_commit() -> str: + try: + return subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=PROJECT_ROOT, text=True).strip() + except Exception: # pragma: no cover - research tool + return "unknown" + + +# ══════════════════════════════════════════════════════════════════════ +# Walk-forward geometry +# ══════════════════════════════════════════════════════════════════════ +def quarter_starts(first: pd.Timestamp, last: pd.Timestamp) -> List[pd.Timestamp]: + """Every Jan/Apr/Jul/Oct 1 00:00 UTC in ``[first, last]`` inclusive.""" + out: List[pd.Timestamp] = [] + year = first.year + while True: + for month in (1, 4, 7, 10): + ts = pd.Timestamp(year=year, month=month, day=1, tz="UTC") + if ts < first: + continue + if ts > last: + return out + out.append(ts) + year += 1 + + +def quarter_label(ts: pd.Timestamp) -> str: + return f"{ts.year}Q{(ts.month - 1) // 3 + 1}" + + +def build_folds(times: np.ndarray, n_folds: int) -> List[np.ndarray]: + """Contiguous-by-time index blocks (rows are already time-sorted).""" + n = len(times) + edges = np.linspace(0, n, n_folds + 1).astype(int) + return [np.arange(edges[i], edges[i + 1]) for i in range(n_folds) + if edges[i + 1] > edges[i]] + + +# ══════════════════════════════════════════════════════════════════════ +# Standardisation +# ══════════════════════════════════════════════════════════════════════ +def fit_standardiser(X: np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """Return (mean, scale, keep) from a training matrix. + + NaNs are ignored when computing the mean (they are later filled with it). + ``scale`` is the population std with zeros replaced by 1.0; ``keep`` is + False for zero-variance-in-train columns, which are dropped from the fit + and carry a coefficient of 0. + """ + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=RuntimeWarning) + mean = np.nanmean(X, axis=0) + std = np.nanstd(X, axis=0) + mean = np.where(np.isfinite(mean), mean, 0.0) + std = np.where(np.isfinite(std), std, 0.0) + keep = std > 0 + scale = np.where(keep, std, 1.0) + return mean, scale, keep + + +def apply_standardiser(X: np.ndarray, mean: np.ndarray, scale: np.ndarray) -> np.ndarray: + """Fill NaN with the TRAIN mean, then standardise.""" + Z = np.array(X, dtype=float, copy=True) + bad = ~np.isfinite(Z) + if bad.any(): + Z[bad] = np.take(mean, np.where(bad)[1]) + return (Z - mean) / scale + + +# ══════════════════════════════════════════════════════════════════════ +# Models (numpy only) +# ══════════════════════════════════════════════════════════════════════ +def _solve(A: np.ndarray, b: np.ndarray) -> np.ndarray: + try: + return np.linalg.solve(A, b) + except np.linalg.LinAlgError: # pragma: no cover - degenerate design + return np.linalg.lstsq(A, b, rcond=None)[0] + + +def fit_ridge(X: np.ndarray, y: np.ndarray, lam: float) -> Tuple[np.ndarray, float]: + """Closed-form ridge with an UNPENALISED intercept. + + Centring X and y is exactly equivalent to appending a 1-column carrying + zero penalty. Returns ``(coef, intercept)`` with ``pred = intercept + X @ coef``. + """ + if X.shape[1] == 0: + return np.zeros(0), float(y.mean()) if len(y) else 0.0 + xm = X.mean(axis=0) + ym = float(y.mean()) + Xc = X - xm + w = _solve(Xc.T @ Xc + lam * np.eye(X.shape[1]), Xc.T @ (y - ym)) + return w, ym - float(xm @ w) + + +def fit_logistic(X: np.ndarray, y: np.ndarray, lam: float, + max_iter: int = NEWTON_MAX_ITER, + tol: float = NEWTON_TOL) -> Tuple[np.ndarray, float, float]: + """L2 logistic via Newton-Raphson, intercept unpenalised. + + Returns ``(coef, intercept, grad_norm)`` where ``grad_norm`` is the + infinity-norm of the penalised gradient at the returned point. + """ + n, p = X.shape + A = np.hstack([np.ones((n, 1)), X]) + pen = np.zeros(p + 1) + pen[1:] = lam + beta = np.zeros(p + 1) + # Warm-start the intercept at the empirical log-odds. + frac = float(np.clip(y.mean(), 1e-6, 1 - 1e-6)) + beta[0] = math.log(frac / (1 - frac)) + grad = np.zeros(p + 1) + for _ in range(max_iter): + eta = A @ beta + prob = 1.0 / (1.0 + np.exp(-np.clip(eta, -35, 35))) + grad = A.T @ (y - prob) - pen * beta + W = np.clip(prob * (1 - prob), 1e-10, None) + H = A.T @ (A * W[:, None]) + np.diag(pen) + step = _solve(H, grad) + beta = beta + step + if np.max(np.abs(step)) < tol: + break + eta = A @ beta + prob = 1.0 / (1.0 + np.exp(-np.clip(eta, -35, 35))) + grad = A.T @ (y - prob) - pen * beta + return beta[1:], float(beta[0]), float(np.max(np.abs(grad))) + + +def predict_linear(X: np.ndarray, coef: np.ndarray, intercept: float) -> np.ndarray: + if X.shape[1] == 0: + return np.full(len(X), intercept, dtype=float) + return intercept + X @ coef + + +def sigmoid(z: np.ndarray) -> np.ndarray: + return 1.0 / (1.0 + np.exp(-np.clip(z, -35, 35))) + + +def log_loss(y: np.ndarray, p: np.ndarray) -> float: + p = np.clip(p, 1e-12, 1 - 1e-12) + return float(-np.mean(y * np.log(p) + (1 - y) * np.log(1 - p))) + + +def choose_lambda(X: np.ndarray, y: np.ndarray, kind: str, + grid: Sequence[float] = LAMBDA_GRID, + n_folds: int = CV_FOLDS) -> Tuple[float, Dict[float, float]]: + """5-fold time-contiguous CV inside the training window.""" + folds = build_folds(np.arange(len(y)), n_folds) + if X.shape[1] == 0 or len(folds) < 2: + return float(grid[0]), {} + scores: Dict[float, float] = {} + for lam in grid: + vals: List[float] = [] + for hold in folds: + mask = np.ones(len(y), dtype=bool) + mask[hold] = False + if mask.sum() < 2: + continue + if kind == "logistic": + ytr = y[mask] + if ytr.min() == ytr.max(): # single-class fold + p = np.full(len(hold), float(np.clip(ytr.mean(), 1e-6, 1 - 1e-6))) + else: + c, b, _ = fit_logistic(X[mask], ytr, lam) + p = sigmoid(predict_linear(X[hold], c, b)) + vals.append(log_loss(y[hold], p)) + else: + c, b = fit_ridge(X[mask], y[mask], lam) + resid = y[hold] - predict_linear(X[hold], c, b) + vals.append(float(np.mean(resid ** 2))) + scores[float(lam)] = float(np.mean(vals)) if vals else float("inf") + best = min(scores, key=lambda k: scores[k]) + return best, scores + + +def lgbm_available() -> bool: + try: + import lightgbm # noqa: F401 + return True + except Exception: + return False + + +def fit_lgbm(X: np.ndarray, y: np.ndarray, seed: int): + import lightgbm as lgb + model = lgb.LGBMRegressor(random_state=seed, verbose=-1, **LGBM_PARAMS) + model.fit(X, y) + return model + + +# ══════════════════════════════════════════════════════════════════════ +# Size map +# ══════════════════════════════════════════════════════════════════════ +def round_to(x, step: float = MAP_ROUND_TO) -> np.ndarray: + """Round to the nearest ``step`` and kill float dust (0.05 → 2 dp).""" + decimals = max(0, int(round(-math.log10(step)))) + 1 + return np.round(np.round(np.asarray(x, dtype=float) / step) * step, decimals) + + +def calibrate_map(z_train: np.ndarray) -> Dict[str, float]: + """lo/hi/normaliser from TRAIN predictions only.""" + lo = float(np.quantile(z_train, MAP_Q_LO)) + hi = float(np.quantile(z_train, MAP_Q_HI)) + if not np.isfinite(lo) or not np.isfinite(hi) or hi <= lo: + return {"lo": lo, "hi": hi, "normalizer": 1.0, "degenerate": True} + raw = MAP_OUT_LO + (MAP_OUT_HI - MAP_OUT_LO) * np.clip((z_train - lo) / (hi - lo), 0, 1) + mean_raw = float(np.mean(raw)) + norm = 1.0 / mean_raw if mean_raw > 0 else 1.0 + return {"lo": lo, "hi": hi, "normalizer": norm, "degenerate": False} + + +def apply_map(z: np.ndarray, cal: Dict[str, float]) -> np.ndarray: + z = np.asarray(z, dtype=float) + if cal.get("degenerate"): + return np.ones(len(z)) + lo, hi, norm = cal["lo"], cal["hi"], cal["normalizer"] + raw = MAP_OUT_LO + (MAP_OUT_HI - MAP_OUT_LO) * np.clip((z - lo) / (hi - lo), 0, 1) + return round_to(np.clip(raw * norm, MAP_CLIP_LO, MAP_CLIP_HI)) + + +# ══════════════════════════════════════════════════════════════════════ +# Metrics +# ══════════════════════════════════════════════════════════════════════ +def max_drawdown(r: np.ndarray) -> float: + """0-prepended cumulative-R drawdown (gpu_backtest.compute_metrics).""" + r = np.asarray(r, dtype=float) + if len(r) == 0: + return 0.0 + cum = np.concatenate(([0.0], np.cumsum(r))) + return float((np.maximum.accumulate(cum) - cum).max()) + + +def profit_factor(r: np.ndarray) -> float: + r = np.asarray(r, dtype=float) + gp = float(r[r > 0].sum()) if (r > 0).any() else 0.0 + gl = float(-r[r < 0].sum()) if (r < 0).any() else 0.0 + return gp / gl if gl > 0 else float("inf") + + +def multiplier_hist(m: np.ndarray) -> pd.DataFrame: + """Counts in 0.1-wide bins from 0.5 to 1.5. + + ``np.histogram`` already closes the final bin on the right, so an + exactly-1.5 multiplier lands in [1.4, 1.5] once and the counts sum to + ``len(m)``. + """ + edges = np.round(np.arange(MAP_OUT_LO, MAP_OUT_HI + 1e-9, 0.1), 2) + counts, _ = np.histogram(np.asarray(m, dtype=float), bins=edges) + counts = counts.astype(int) + return pd.DataFrame({ + "bin_lo": edges[:-1], "bin_hi": edges[1:], "count": counts, + "pct": np.where(len(m), counts / max(len(m), 1) * 100, 0.0), + }) + + +# ══════════════════════════════════════════════════════════════════════ +# Walk-forward driver +# ══════════════════════════════════════════════════════════════════════ +def walk_forward(df: pd.DataFrame, features: Sequence[str], model: str, + bar_minutes: int, burn_in_months: int, purge_bars: int, + seed: int, y_train_override: Optional[np.ndarray] = None, + verbose: bool = True) -> Dict: + """One full expanding walk-forward pass. + + ``y_train_override`` (full-book length, aligned to ``df``) replaces the + TRAINING labels only — this is how the shuffle null is run. Test labels + always come from the real ``r_multiple``. + """ + times = df["entry_time"].to_numpy() + r_all = df["r_multiple"].to_numpy(float) + r_train_src = r_all if y_train_override is None else np.asarray(y_train_override, float) + X_all = df[list(features)].apply(pd.to_numeric, errors="coerce").to_numpy(float) \ + if features else np.zeros((len(df), 0)) + + book_start = df["entry_time"].iloc[0] + book_end = df["entry_time"].iloc[-1] + burn_end = book_start + pd.DateOffset(months=burn_in_months) + purge = pd.Timedelta(minutes=purge_bars * bar_minutes) + + qs = [q for q in quarter_starts(book_start, book_end) if q >= burn_end] + rows: List[Dict] = [] + per_q: List[Dict] = [] + coef_rows: List[Dict] = [] + skipped: List[str] = [] + dropped_seen: set = set() + + for q_start in qs: + q_end = q_start + pd.DateOffset(months=3) + train_mask = df["entry_time"] < (q_start - purge) + test_mask = (df["entry_time"] >= q_start) & (df["entry_time"] < q_end) + n_tr, n_te = int(train_mask.sum()), int(test_mask.sum()) + if n_tr < MIN_TRAIN or n_te == 0: + skipped.append(f"{quarter_label(q_start)} (train={n_tr}, test={n_te})") + continue + + tr = np.flatnonzero(train_mask.to_numpy()) + te = np.flatnonzero(test_mask.to_numpy()) + Xtr_raw, Xte_raw = X_all[tr], X_all[te] + mean, scale, keep = fit_standardiser(Xtr_raw) + dropped = [features[i] for i in np.flatnonzero(~keep)] + if verbose and dropped: + tag = (quarter_label(q_start), tuple(dropped)) + if tag not in dropped_seen: + dropped_seen.add(tag) + say(f" WARNING {quarter_label(q_start)}: zero variance in train, " + f"dropped from fit (coef 0): {', '.join(dropped)}") + Xtr = apply_standardiser(Xtr_raw, mean, scale)[:, keep] + Xte = apply_standardiser(Xte_raw, mean, scale)[:, keep] + + r_tr = r_train_src[tr] + lam: Optional[float] = None + if model == "logistic": + y_tr = (r_tr > 0).astype(float) + lam, _ = choose_lambda(Xtr, y_tr, "logistic") + coef, b, _ = fit_logistic(Xtr, y_tr, lam) + z_tr = sigmoid(predict_linear(Xtr, coef, b)) + z_te = sigmoid(predict_linear(Xte, coef, b)) + elif model == "lgbm": + y_tr = np.clip(r_tr, WINSOR_LO, WINSOR_HI) + gbm = fit_lgbm(Xtr, y_tr, seed) + z_tr = np.asarray(gbm.predict(Xtr), dtype=float) + z_te = np.asarray(gbm.predict(Xte), dtype=float) + coef, b = np.zeros(int(keep.sum())), 0.0 + else: # ridge / isbuy / exhaustion + y_tr = np.clip(r_tr, WINSOR_LO, WINSOR_HI) + lam, _ = choose_lambda(Xtr, y_tr, "ridge") + coef, b = fit_ridge(Xtr, y_tr, lam) + z_tr = predict_linear(Xtr, coef, b) + z_te = predict_linear(Xte, coef, b) + + cal = calibrate_map(z_tr) + m_te = apply_map(z_te, cal) + + if model != "lgbm": + full_coef = np.zeros(len(features)) + full_coef[np.flatnonzero(keep)] = coef + for name, c in zip(features, full_coef): + coef_rows.append({"quarter": quarter_label(q_start), "feature": name, + "coef_std": float(c), "lambda": lam, + "dropped": name in dropped}) + coef_rows.append({"quarter": quarter_label(q_start), "feature": "_intercept", + "coef_std": float(b), "lambda": lam, "dropped": False}) + + r_te = r_all[te] + for i, idx in enumerate(te): + rows.append({ + "entry_time": df["entry_time"].iloc[idx], + "r_multiple": float(r_te[i]), + "mult": float(m_te[i]), + "r_sized": float(r_te[i] * m_te[i]), + "quarter": quarter_label(q_start), + "z": float(z_te[i]), + }) + per_q.append({ + "quarter": quarter_label(q_start), + "q_start": q_start, "n_train": n_tr, "n_test": n_te, + "flat_r": float(r_te.sum()), "sized_r": float((r_te * m_te).sum()), + "mean_m": float(m_te.mean()), "lambda": lam, + "map_lo": cal["lo"], "map_hi": cal["hi"], "normalizer": cal["normalizer"], + }) + + oos = pd.DataFrame(rows) + return {"oos": oos, "quarters": pd.DataFrame(per_q), + "coefs": pd.DataFrame(coef_rows), "skipped": skipped, + "quarter_starts": qs} + + +def evaluate(oos: pd.DataFrame) -> Dict: + """Headline OOS numbers for one model's walk-forward output.""" + if oos.empty: + return {"n": 0, "flat_r": 0.0, "sized_r": 0.0, "lift": 0.0} + r = oos["r_multiple"].to_numpy(float) + rs = oos["r_sized"].to_numpy(float) + m = oos["mult"].to_numpy(float) + med = oos["entry_time"].median() + first = (oos["entry_time"] < med).to_numpy() + second = ~first + return { + "n": int(len(oos)), + "flat_r": float(r.sum()), "sized_r": float(rs.sum()), + "lift": float(rs.sum() - r.sum()), + "flat_pf": profit_factor(r), "sized_pf": profit_factor(rs), + "flat_dd": max_drawdown(r), "sized_dd": max_drawdown(rs), + "mean_m": float(m.mean()), "min_m": float(m.min()), "max_m": float(m.max()), + "median_time": med, + "n_h1": int(first.sum()), "n_h2": int(second.sum()), + "flat_h1": float(r[first].sum()), "flat_h2": float(r[second].sum()), + "sized_h1": float(rs[first].sum()), "sized_h2": float(rs[second].sum()), + } + + +def acceptance(ev: Dict, real_lift: float, shuffle_p95: float) -> Dict: + """The six pre-registered clauses.""" + c1 = ev["sized_h1"] > ev["flat_h1"] and ev["sized_h2"] > ev["flat_h2"] + c2 = ev["sized_r"] > ev["flat_r"] + c3 = ev["sized_dd"] <= DD_TOL * ev["flat_dd"] + c4 = (ev["sized_pf"] >= PF_FLOOR) and ev["sized_h1"] > 0 and ev["sized_h2"] > 0 + c5 = real_lift > shuffle_p95 if np.isfinite(shuffle_p95) else False + c6 = (abs(ev["mean_m"] - 1.0) <= MEAN_MULT_TOL + and ev["min_m"] >= MAP_CLIP_LO - 1e-9 and ev["max_m"] <= MAP_CLIP_HI + 1e-9) + clauses = {"1_both_halves": bool(c1), "2_overall": bool(c2), "3_drawdown": bool(c3), + "4_pf_and_halves": bool(c4), "5_beats_shuffle": bool(c5), + "6_multiplier_bounds": bool(c6)} + clauses["overall"] = bool(all(clauses.values())) + return clauses + + +def shuffle_null(df: pd.DataFrame, features: Sequence[str], model: str, + bar_minutes: int, burn_in_months: int, purge_bars: int, + n_shuffle: int, seed: int) -> np.ndarray: + """Lift distribution under permuted TRAINING labels (test labels real).""" + rng = np.random.default_rng(seed) + r_all = df["r_multiple"].to_numpy(float) + lifts: List[float] = [] + for _ in range(n_shuffle): + r_perm = r_all[rng.permutation(len(r_all))] + res = walk_forward(df, features, model, bar_minutes, burn_in_months, + purge_bars, seed, y_train_override=r_perm, verbose=False) + ev = evaluate(res["oos"]) + lifts.append(ev["lift"]) + return np.asarray(lifts, dtype=float) + + +# ══════════════════════════════════════════════════════════════════════ +# Final fit + artifact +# ══════════════════════════════════════════════════════════════════════ +def final_fit(df: pd.DataFrame, features: Sequence[str], model: str, + seed: int) -> Dict: + """Fit on ALL trades; calibrate the map on in-sample predictions.""" + X_raw = df[list(features)].apply(pd.to_numeric, errors="coerce").to_numpy(float) + r = df["r_multiple"].to_numpy(float) + mean, scale, keep = fit_standardiser(X_raw) + X = apply_standardiser(X_raw, mean, scale)[:, keep] + if model == "logistic": + y = (r > 0).astype(float) + lam, _ = choose_lambda(X, y, "logistic") + coef, b, gnorm = fit_logistic(X, y, lam) + z = sigmoid(predict_linear(X, coef, b)) + target = "r_positive" + else: + y = np.clip(r, WINSOR_LO, WINSOR_HI) + lam, _ = choose_lambda(X, y, "ridge") + coef, b = fit_ridge(X, y, lam) + gnorm = float("nan") + z = predict_linear(X, coef, b) + target = "r_winsorised" + full_coef = np.zeros(len(features)) + full_coef[np.flatnonzero(keep)] = coef + cal = calibrate_map(z) + return {"mean": mean, "scale": scale, "keep": keep, "coef": full_coef, + "intercept": float(b), "lambda": float(lam), "z": z, + "mult": apply_map(z, cal), "map": cal, "target": target, + "grad_norm": gnorm} + + +def predict_multiplier(feature_dict: Dict[str, float], artifact: Dict) -> float: + """Runtime formula: standardise → linear score → size map → 0.05 grid.""" + names = artifact["features"] + mean = artifact["mean"] + scale = artifact["scale"] + coef = artifact["coef"] + z = float(artifact["intercept"]) + for i, name in enumerate(names): + raw = feature_dict.get(name, mean[i]) + if raw is None or not np.isfinite(float(raw)): + raw = mean[i] + z += float(coef[i]) * (float(raw) - float(mean[i])) / float(scale[i]) + if artifact["model"] == "logistic": + z = 1.0 / (1.0 + math.exp(-max(-35.0, min(35.0, z)))) + mp = artifact["map"] + if mp["hi"] <= mp["lo"]: + return 1.0 + frac = min(1.0, max(0.0, (z - mp["lo"]) / (mp["hi"] - mp["lo"]))) + raw_m = mp["out_lo"] + (mp["out_hi"] - mp["out_lo"]) * frac + val = min(mp["clip_hi"], max(mp["clip_lo"], raw_m * mp["normalizer"])) + step = mp["round_to"] + return float(round(round(val / step) * step, 2)) + + +def build_artifact(fit: Dict, features: Sequence[str], model: str, df: pd.DataFrame, + ev: Dict, clauses: Dict, real_lift: float, shuffle: np.ndarray, + n_shuffle: int, book: Path, tf: str, confirm_tfs: List[str], + burn_in_months: int, purge_bars: int, spread: float) -> Dict: + return { + "schema_version": 1, + "feature_version": FEATURE_VERSION, + "pair": "XAU/USD", + "primary_tf": tf, + "confirmation_tfs": confirm_tfs, + "model": model, + "target": fit["target"], + "features": list(features), + "mean": [float(v) for v in fit["mean"]], + "scale": [float(v) for v in fit["scale"]], + "coef": [float(v) for v in fit["coef"]], + "intercept": float(fit["intercept"]), + "map": { + "kind": "clipped_linear_on_quantiles", + "lo": float(fit["map"]["lo"]), "hi": float(fit["map"]["hi"]), + "out_lo": MAP_OUT_LO, "out_hi": MAP_OUT_HI, + "normalizer": float(fit["map"]["normalizer"]), + "clip_lo": MAP_CLIP_LO, "clip_hi": MAP_CLIP_HI, + "round_to": MAP_ROUND_TO, + }, + "training": { + "window": [df["entry_time"].iloc[0].isoformat(), + df["entry_time"].iloc[-1].isoformat()], + "spread": spread, + "n_train_trades": int(len(df)), + "n_oos_trades": int(ev.get("n", 0)), + "burn_in_months": burn_in_months, + "purge_bars": purge_bars, + "refit": "calendar_quarter", + "final_fit": "full_window", + "lambda": float(fit["lambda"]), + "git_commit": git_commit(), + "book_sha256": sha256_file(book), + }, + "acceptance": { + **{k: v for k, v in clauses.items()}, + "flat_r": ev.get("flat_r"), "sized_r": ev.get("sized_r"), + "sized_pf": ev.get("sized_pf"), "flat_dd": ev.get("flat_dd"), + "sized_dd": ev.get("sized_dd"), + "sized_h1": ev.get("sized_h1"), "sized_h2": ev.get("sized_h2"), + "flat_h1": ev.get("flat_h1"), "flat_h2": ev.get("flat_h2"), + "real_lift": float(real_lift), + "shuffle_p95_lift": float(np.percentile(shuffle, 95)) if len(shuffle) else None, + "n_shuffle": int(n_shuffle), + "mean_multiplier": ev.get("mean_m"), + }, + } + + +# ══════════════════════════════════════════════════════════════════════ +# Reporting +# ══════════════════════════════════════════════════════════════════════ +def fmt(x, nd=2, width=0) -> str: + if x is None: + return "n/a".rjust(width) + if isinstance(x, float) and not np.isfinite(x): + return ("inf" if x > 0 else "-inf").rjust(width) + s = f"{x:.{nd}f}" if isinstance(x, (int, float, np.floating)) else str(x) + return s.rjust(width) if width else s + + +def print_quarters(q: pd.DataFrame) -> None: + say(f"{'quarter':<9}{'n_tr':>6}{'n_te':>6}{'flat R':>10}{'sized R':>10}" + f"{'mean m':>9}{'lambda':>9}") + for _, row in q.iterrows(): + say(f"{row['quarter']:<9}{int(row['n_train']):>6}{int(row['n_test']):>6}" + f"{fmt(row['flat_r'], 2, 10)}{fmt(row['sized_r'], 2, 10)}" + f"{fmt(row['mean_m'], 3, 9)}" + f"{('n/a' if row['lambda'] is None else fmt(row['lambda'], 0)):>9}") + + +def print_hist(m: np.ndarray) -> None: + h = multiplier_hist(m) + say(f"{'bin':<12}{'count':>8}{'pct':>8}") + for _, row in h.iterrows(): + bar = "#" * int(round(row["pct"] / 2)) + say(f"{row['bin_lo']:.1f}–{row['bin_hi']:.1f}{'':<4}{int(row['count']):>8}" + f"{row['pct']:>7.1f}% {bar}") + + +def print_coef_signs(coefs: pd.DataFrame, label: str) -> None: + """Mean standardised coefficient + sign stability across quarters.""" + if coefs.empty: + return + d = coefs[coefs["feature"] != "_intercept"] + say(f" standardised coefficients across quarters ({label}):") + say(f" {'feature':<24}{'mean':>10}{'min':>10}{'max':>10}{'+q':>5}{'-q':>5} sign-stable") + for name, grp in d.groupby("feature", sort=False): + c = grp["coef_std"].to_numpy(float) + pos, neg = int((c > 0).sum()), int((c < 0).sum()) + stable = "yes" if (pos == 0 or neg == 0) and (pos + neg) > 0 else "no" + say(f" {name:<24}{c.mean():>10.4f}{c.min():>10.4f}{c.max():>10.4f}" + f"{pos:>5}{neg:>5} {stable}") + + +def print_comparison(table: List[Dict]) -> None: + say(f"{'model':<12}{'n':>6}{'flat R':>10}{'sized R':>10}{'lift':>9}" + f"{'PF':>7}{'DD':>8}{'mean m':>8}{'p95 null':>10}{'rank%':>7} pass") + for row in table: + say(f"{row['model']:<12}{row['n']:>6}{fmt(row['flat_r'], 2, 10)}" + f"{fmt(row['sized_r'], 2, 10)}{fmt(row['lift'], 2, 9)}" + f"{fmt(row['sized_pf'], 2, 7)}{fmt(row['sized_dd'], 2, 8)}" + f"{fmt(row['mean_m'], 3, 8)}{fmt(row['shuffle_p95'], 2, 10)}" + f"{fmt(row['rank_pct'], 0, 7)} {'PASS' if row['pass'] else 'FAIL'}") + + +# ══════════════════════════════════════════════════════════════════════ +# main +# ══════════════════════════════════════════════════════════════════════ +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--book", required=True, type=Path) + ap.add_argument("--bar-minutes", required=True, type=int) + ap.add_argument("--features", required=True, + help="comma-separated f_* column names for the main model") + ap.add_argument("--model", default="ridge", choices=("ridge", "logistic")) + ap.add_argument("--burn-in-months", type=int, default=12) + ap.add_argument("--purge-bars", type=int, default=300) + ap.add_argument("--n-shuffle", type=int, default=50) + ap.add_argument("--seed", type=int, default=20260905) + ap.add_argument("--skip-lgbm", action="store_true") + ap.add_argument("--out", type=Path, default=None) + ap.add_argument("--write-artifact", type=Path, default=None) + ap.add_argument("--tf", default=None, help="primary TF label for the artifact") + ap.add_argument("--confirm-tfs", default="15min,1h") + ap.add_argument("--spread", type=float, default=0.65, + help="cost the book was produced at (recorded in the artifact)") + args = ap.parse_args() + + if args.write_artifact and not args.tf: + raise SystemExit("--write-artifact requires --tf") + + say("=" * 96) + say("Stage-D stake-sizing walk-forward — RESEARCH ONLY") + say("=" * 96) + say(f"book : {args.book}") + df = load_book(args.book) + say(f"trades : {len(df)} " + f"({df['entry_time'].iloc[0]} → {df['entry_time'].iloc[-1]})") + say(f"bar minutes : {args.bar_minutes} purge = {args.purge_bars} bars = " + f"{args.purge_bars * args.bar_minutes} min " + f"({args.purge_bars * args.bar_minutes / 1440:.2f} d)") + say(f"burn-in : {args.burn_in_months} months " + f"(train-only until {df['entry_time'].iloc[0] + pd.DateOffset(months=args.burn_in_months)})") + say(f"model / seed : {args.model} / {args.seed} shuffles: {args.n_shuffle}") + say(f"winsor / lambda : [{WINSOR_LO}, {WINSOR_HI}] / {list(LAMBDA_GRID)} " + f"CV folds {CV_FOLDS} (time-contiguous)") + say(f"map : m = round_{MAP_ROUND_TO}(clip(norm × (0.5 + clip((z−lo)/(hi−lo),0,1)" + f"), {MAP_CLIP_LO}, {MAP_CLIP_HI})) lo/hi = train q{MAP_Q_LO}/q{MAP_Q_HI}") + say("") + + main_feats = resolve_features(df, [f.strip() for f in args.features.split(",") if f.strip()], + "--features") + say(f"main features ({len(main_feats)}): {', '.join(main_feats)}") + + specs: List[Tuple[str, List[str], str]] = [(args.model, main_feats, args.model)] + for label, feats in (("isbuy", ISBUY_FEATURES), ("exhaustion", EXHAUSTION_FEATURES)): + have = [f for f in feats if f in df.columns] + if len(have) != len(feats): + say(f"NOTE: {label} baseline missing " + f"{', '.join(f for f in feats if f not in df.columns)} — using {len(have)}") + if have: + specs.append((label, have, "ridge")) + use_lgbm = (not args.skip_lgbm) and lgbm_available() + if not args.skip_lgbm and not use_lgbm: + say("lightgbm not installed — upper bound skipped") + if use_lgbm: + all_f = [c for c in df.columns if c.startswith("f_")] + specs.append(("lgbm", all_f, "lgbm")) + say(f"lightgbm upper bound: {len(all_f)} f_* columns") + say("") + + results: Dict[str, Dict] = {} + table: List[Dict] = [] + for label, feats, kind in specs: + say("-" * 96) + say(f"MODEL: {label} ({kind}, {len(feats)} feature(s))") + say("-" * 96) + res = walk_forward(df, feats, kind, args.bar_minutes, args.burn_in_months, + args.purge_bars, args.seed, verbose=True) + if res["skipped"]: + say(f" skipped quarters: {'; '.join(res['skipped'])}") + if res["oos"].empty: + say(" no OOS quarters — nothing to evaluate") + continue + ev = evaluate(res["oos"]) + n_sh = args.n_shuffle if kind != "lgbm" else min(args.n_shuffle, LGBM_MAX_SHUFFLE) + if kind == "lgbm" and n_sh != args.n_shuffle: + say(f" NOTE (protocol deviation, flagged): lgbm shuffle null capped at " + f"{n_sh} repeats (never ships)") + sh = shuffle_null(df, feats, kind, args.bar_minutes, args.burn_in_months, + args.purge_bars, n_sh, args.seed) if n_sh > 0 else np.array([]) + p95 = float(np.percentile(sh, 95)) if len(sh) else float("nan") + rank = float((sh < ev["lift"]).mean() * 100) if len(sh) else float("nan") + cl = acceptance(ev, ev["lift"], p95) + + say(f" OOS quarters : {len(res['quarters'])} " + f"({res['quarters']['quarter'].iloc[0]} → {res['quarters']['quarter'].iloc[-1]}), " + f"{ev['n']} trades") + say(f" flat R : {ev['flat_r']:+.2f} PF {fmt(ev['flat_pf'])} " + f"maxDD {ev['flat_dd']:.2f}") + say(f" sized R : {ev['sized_r']:+.2f} PF {fmt(ev['sized_pf'])} " + f"maxDD {ev['sized_dd']:.2f}") + say(f" lift : {ev['lift']:+.2f} R") + say(f" halves (split {ev['median_time']}): " + f"h1 flat {ev['flat_h1']:+.2f} / sized {ev['sized_h1']:+.2f} (n={ev['n_h1']}) | " + f"h2 flat {ev['flat_h2']:+.2f} / sized {ev['sized_h2']:+.2f} (n={ev['n_h2']})") + say(f" multiplier : mean {ev['mean_m']:.3f} min {ev['min_m']:.2f} " + f"max {ev['max_m']:.2f}") + say(f" shuffle null : n={len(sh)} mean {fmt(float(sh.mean()) if len(sh) else None)} " + f"p95 {fmt(p95)} real-lift percentile rank {fmt(rank, 0)}%") + say(" acceptance:") + for k, v in cl.items(): + if k == "overall": + continue + say(f" {k:<22} {'PASS' if v else 'FAIL'}") + say(f" {'OVERALL':<22} {'PASS' if cl['overall'] else 'FAIL'}") + say(" per-quarter:") + print_quarters(res["quarters"]) + print_coef_signs(res["coefs"], label) + say("") + + results[label] = {"res": res, "ev": ev, "shuffle": sh, "p95": p95, + "rank": rank, "clauses": cl, "features": feats, "kind": kind} + table.append({"model": label, "n": ev["n"], "flat_r": ev["flat_r"], + "sized_r": ev["sized_r"], "lift": ev["lift"], + "sized_pf": ev["sized_pf"], "sized_dd": ev["sized_dd"], + "mean_m": ev["mean_m"], "shuffle_p95": p95, "rank_pct": rank, + "pass": cl["overall"]}) + + if args.model not in results: + raise SystemExit("the main model produced no OOS quarters — check burn-in / " + "book window / MIN_TRAIN") + + say("=" * 96) + say("COMPARISON") + say("=" * 96) + print_comparison(table) + say("") + say(f"OOS multiplier histogram ({args.model}):") + print_hist(results[args.model]["res"]["oos"]["mult"].to_numpy()) + say("") + + main_label = args.model + beats = None + if main_label in results and "isbuy" in results: + beats = results[main_label]["ev"]["lift"] > results["isbuy"]["ev"]["lift"] + say(f"beats isbuy baseline: {'yes' if beats else 'no'} " + f"({results[main_label]['ev']['lift']:+.2f} R vs " + f"{results['isbuy']['ev']['lift']:+.2f} R)") + say(f"OVERALL ACCEPTANCE ({main_label}): " + f"{'PASS' if results[main_label]['clauses']['overall'] else 'FAIL'}") + say("") + + # ── artifact ────────────────────────────────────────────────────── + artifact = None + if args.write_artifact: + say("-" * 96) + say("FINAL FIT (all trades) + ARTIFACT") + say("-" * 96) + fit = final_fit(df, main_feats, args.model, args.seed) + say(f" lambda (CV on full window): {fit['lambda']:g}") + if np.isfinite(fit["grad_norm"]): + say(f" logistic |grad|_inf at convergence: {fit['grad_norm']:.3e}") + say(f" map lo/hi/normaliser: {fit['map']['lo']:.5f} / {fit['map']['hi']:.5f} / " + f"{fit['map']['normalizer']:.5f}") + artifact = build_artifact( + fit, main_feats, args.model, df, results[main_label]["ev"], + results[main_label]["clauses"], results[main_label]["ev"]["lift"], + results[main_label]["shuffle"], args.n_shuffle, args.book, args.tf, + [t.strip() for t in args.confirm_tfs.split(",") if t.strip()], + args.burn_in_months, args.purge_bars, args.spread) + # Self-check: the runtime formula must reproduce the in-sample map exactly. + rows = df[main_feats].to_dict("records") + repro = np.array([predict_multiplier(r, artifact) for r in rows]) + exact = bool(np.array_equal(repro, fit["mult"])) + say(f" predict_multiplier reproduces in-sample multipliers exactly: " + f"{'YES' if exact else 'NO'} (max |diff| " + f"{float(np.max(np.abs(repro - fit['mult']))):.3e})") + if not exact: + raise SystemExit("predict_multiplier does not reproduce the final fit — abort") + args.write_artifact.parent.mkdir(parents=True, exist_ok=True) + args.write_artifact.write_text(json.dumps(artifact, indent=2, default=str)) + say(f" wrote {args.write_artifact}") + say("") + + # ── outputs ─────────────────────────────────────────────────────── + if args.out: + out = args.out + out.mkdir(parents=True, exist_ok=True) + main_res = results[main_label] + main_res["res"]["oos"].to_csv(out / "oos_trades.csv", index=False) + main_res["res"]["quarters"].to_csv(out / "quarters.csv", index=False) + coefs = pd.concat( + [r["res"]["coefs"].assign(model=lbl) for lbl, r in results.items() + if not r["res"]["coefs"].empty], + ignore_index=True) if results else pd.DataFrame() + coefs.to_csv(out / "coefs_by_quarter.csv", index=False) + pd.DataFrame({ + "model": np.repeat([lbl for lbl in results], [len(r["shuffle"]) for r in results.values()]), + "lift": np.concatenate([r["shuffle"] for r in results.values()]) + if results else np.array([]), + }).to_csv(out / "shuffle_lifts.csv", index=False) + multiplier_hist(main_res["res"]["oos"]["mult"].to_numpy()).to_csv( + out / "multiplier_hist.csv", index=False) + payload = { + "book": str(args.book), "book_sha256": sha256_file(args.book), + "git_commit": git_commit(), "bar_minutes": args.bar_minutes, + "model": args.model, "features": main_feats, "seed": args.seed, + "n_shuffle": args.n_shuffle, "burn_in_months": args.burn_in_months, + "purge_bars": args.purge_bars, + "beats_isbuy": beats, + "models": { + lbl: { + "kind": r["kind"], "features": r["features"], + "evaluation": {k: (None if isinstance(v, float) and not np.isfinite(v) + else (str(v) if isinstance(v, pd.Timestamp) else v)) + for k, v in r["ev"].items()}, + "shuffle": {"n": int(len(r["shuffle"])), + "mean": float(r["shuffle"].mean()) if len(r["shuffle"]) else None, + "p95": None if not np.isfinite(r["p95"]) else r["p95"], + "real_lift_percentile": None if not np.isfinite(r["rank"]) + else r["rank"]}, + "acceptance": r["clauses"], + } for lbl, r in results.items() + }, + "artifact": artifact, + } + (out / "results.json").write_text(json.dumps(payload, indent=2, default=str)) + (out / "summary.txt").write_text("\n".join(_LINES) + "\n") + print(f"\nwrote outputs to {out}") + + +if __name__ == "__main__": + main() diff --git a/backtest/research/watch_rule_eval.py b/backtest/research/watch_rule_eval.py new file mode 100644 index 0000000..67f99fe --- /dev/null +++ b/backtest/research/watch_rule_eval.py @@ -0,0 +1,78 @@ +"""Evaluate the violent-move WATCH rule (as shipped in violent_watch.py) bar by bar on history. Analysis tooling only. +Level per bar from: calendar (high-impact USD within 30 min ahead / 30 min after), clock slots, indicator readings. +Reports share of bars per level, burst rate next bar / next hour per level (both halves), and hot-hours per day. + python backtest/research/watch_rule_eval.py PARQUET BAR_MINUTES +""" +import sys +from pathlib import Path +import numpy as np, pandas as pd +ROOT = Path(__file__).resolve().parents[2]; sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "backtest")) +from gpu_backtest import precompute_indicators +from config import TechnicalConfig + + +def wilder_atr(df, period=14): + pc = df["Close"].shift(1) + tr = pd.concat([df["High"] - df["Low"], (df["High"] - pc).abs(), (df["Low"] - pc).abs()], axis=1).max(axis=1) + return tr.ewm(alpha=1 / period, min_periods=period, adjust=False).mean() + + +def main(): + pq, bm = sys.argv[1], int(sys.argv[2]) + px = pd.read_parquet(pq); px = px[(px.index >= pd.Timestamp("2023-08-05", tz="UTC")) & (px.index < pd.Timestamp("2026-09-05", tz="UTC"))] + ic = precompute_indicators(px, TechnicalConfig()) + sh = lambda a: pd.Series(np.asarray(a, dtype=float), index=px.index).shift(1) + rsi, adxs, hist, bbp, kcu, kcl = sh(ic.rsi), sh(ic.adx_slope), sh(ic.macd_hist), sh(ic.bb_position), sh(ic.kc_upper), sh(ic.kc_lower) + ef, em, es, et, adx, atr1 = sh(ic.ema_fast), sh(ic.ema_medium), sh(ic.ema_slow), sh(ic.ema_trend), sh(ic.adx), sh(ic.atr) + c1 = px["Close"].shift(1); hp = hist.shift(1) + atr_prev = wilder_atr(px).shift(1); rng = px["High"] - px["Low"] + burst = (rng >= 3.0 * atr_prev).astype(int); H = max(1, 60 // bm) + next_hour = burst[::-1].rolling(H, min_periods=1).max()[::-1].astype(int) + align = np.sign(c1 - ef) + np.sign(ef - em) + np.sign(em - es) + np.sign(es - et) + regime = np.select([(adx >= 25) & (align > 0), (adx >= 25) & (align < 0), adx >= 25, adxs > 3.0], ["trending_up", "trending_down", "trending", "transitioning"], default="ranging") + atr_pct = atr1.rolling(50).apply(lambda w: (w[-1] > w).mean() * 100, raw=True); extreme = atr_pct > 80 + # readings + bear = ((bbp < 0).astype(int) + (rsi < 30).astype(int) + (c1 < kcl).astype(int) + ((hist < 0) & (hist < hp) & (hp < 0)).astype(int)) + bull = ((bbp > 1).astype(int) + (rsi >= 70).astype(int) + (c1 > kcu).astype(int) + ((hist > 0) & (hist > hp) & (hp > 0)).astype(int)) + neutral = (adxs > 3).astype(int) + extreme.astype(int) + (regime == "transitioning").astype(int) + readings = bear + bull + neutral + # clock (now = bar close time) + ct = px.index + pd.Timedelta(minutes=bm); wd = ct.weekday; hr = ct.hour + clock = ((wd <= 4) & np.isin(hr, [12, 13, 14])) | (hr == 1) | ((wd == 6) & np.isin(hr, [22, 23])) + # calendar + cal = pd.read_parquet(ROOT / "backtest/cache/calendar.parquet"); cal["ts"] = pd.to_datetime(cal.dateline, unit="s", utc=True) + hi = np.sort(cal[(cal.currency == "USD") & (cal.impactName.str.lower() == "high")].ts.values.astype("datetime64[ns]")) + t = ct.values.astype("datetime64[ns]"); nx = np.searchsorted(hi, t) + to_next = np.where(nx < len(hi), (hi[np.minimum(nx, len(hi) - 1)] - t) / np.timedelta64(1, "m"), 1e9) + since = np.where(nx - 1 >= 0, (t - hi[np.maximum(nx - 1, 0)]) / np.timedelta64(1, "m"), 1e9) + calendar = ((to_next > 0) & (to_next <= 30)) | (since <= 30) + # FINAL RULE (2026-09-06, chosen from the variant sweep recorded in sizing_campaign_2026-09.md): + # extreme = price-at-extreme readings only (outside Bollinger, RSI beyond 30/70, outside Keltner) + # hot = calendar | (clock & extreme >= 1) -> ~8.6% of bars, next-bar odds 3-5x normal, ~4 hot hours/day + # watch = clock | extreme >= 2 -> ~2x normal + extreme = ((bbp < 0) | (bbp > 1)).astype(int) + ((rsi < 30) | (rsi >= 70)).astype(int) + ((c1 < kcl) | (c1 > kcu)).astype(int) + level = np.where(calendar | (clock & (extreme >= 1)), "hot", np.where(clock | (extreme >= 2), "watch", "none")) + d = pd.DataFrame({"level": level, "burst": burst.values, "next_hour": next_hour.values, "readings": readings.values, "clock": clock, "calendar": calendar}, index=px.index) + d = d[d.index >= pd.Timestamp("2023-09-05", tz="UTC")] + half = np.where(np.arange(len(d)) < len(d) // 2, 1, 2) + b1, b2 = d.burst[half == 1].mean(), d.burst[half == 2].mean(); h1, h2 = d.next_hour[half == 1].mean(), d.next_hour[half == 2].mean() + print(f"{Path(pq).name}: {len(d)} bars; base next-bar {100*d.burst.mean():.2f}%, next-hour {100*d.next_hour.mean():.1f}%") + rows = [] + for lv in ("none", "watch", "hot"): + m = d.level == lv + rows.append(dict(level=lv, share_of_bars=100 * m.mean(), next_bar_pct=100 * d.burst[m].mean(), lift_h1=d.burst[m & (half == 1)].mean() / b1, lift_h2=d.burst[m & (half == 2)].mean() / b2, + next_hour_pct=100 * d.next_hour[m].mean(), hour_lift_h1=d.next_hour[m & (half == 1)].mean() / h1, hour_lift_h2=d.next_hour[m & (half == 2)].mean() / h2, + share_of_all_bursts=100 * d.burst[m].sum() / d.burst.sum())) + print(pd.DataFrame(rows).round(2).to_string(index=False)) + hot = d[d.level == "hot"] + kinds = pd.Series(np.where(hot.calendar, "calendar", "clock+extreme reading"), index=hot.index) + print("hot bars by cause:", (100 * kinds.value_counts(normalize=True)).round(1).to_dict()) + hot_hours = hot.index.floor("h").unique() + days = pd.Series(hot_hours.date).value_counts() + print(f"distinct HOT hours per calendar day: mean {days.mean():.1f}, median {days.median():.0f}, max {days.max()} (≈ heads-up messages/day before the daily cap)") + hb = d.burst.groupby(d.index.floor("h")).max(); hh = pd.Series(1, index=hot_hours) + print(f"of all hours with a violent bar, the hour was flagged HOT at its start or during: {100*hb[hb==1].index.isin(hot_hours).mean():.1f}%") + + +if __name__ == "__main__": + main() diff --git a/position_sizing.py b/position_sizing.py new file mode 100644 index 0000000..ce13786 --- /dev/null +++ b/position_sizing.py @@ -0,0 +1,622 @@ +""" +ForexBot2 — Position sizing (stake-multiplier overlay) +====================================================== + +Single source of truth for **decision-time features** and the resulting +**stake multiplier**. Everything here is designed to be called from +inside the signal path so that live and backtest see byte-identical +inputs (rule 4: live == backtest parity). + +Design contract +--------------- +* **Bounded lookbacks.** No feature may read further back than the live + fetch depth of its timeframe (``LIVE_DEPTH_BARS``). The primary frame + is hard-capped at ``MAX_LOOKBACK_BARS`` rows, confirmation frames at + 60 rows, and the burst scan at 200 rows, so a feature computed on a + full backtest frame equals the same feature computed on the shorter + live tail. +* **Closed bars only.** The frames handed in are exactly the frames + ``analyze_from_dataframes`` received: UTC tz-aware, oldest-first, + capitalized OHLCV columns, forming candle already dropped. The last + row is the decision bar; nothing peeks past it. +* **Never raises into a scan.** ``build_features`` and + ``stake_multiplier`` swallow every exception (logged at DEBUG) and + fall back to neutral defaults. A sizing failure can never break a + signal. +* **Advisory today.** ``stake_multiplier`` returns ``1.0`` in every case + at this stage — artifact evaluation lands in a later step. The guard + order is already implemented so the wiring is testable now. + +Dependencies are deliberately limited to numpy/pandas plus ``config`` +(the ``Timeframe`` enum): this module ships live and must not drag the +provider/news stack into any process that only wants features. + +VERDICT 2026-09-06 (backtest/research/sizing_campaign_2026-09.md): the walk-forward +stake-multiplier model FAILED the pre-registered acceptance rule on both M5 and +M15 (M5 a clean null; M15 lift inside the label-shuffle null and explained by +BUY-vs-SELL). NO artifact ships and nothing reads a multiplier live. This module +is research plumbing: `build_features` feeds the backtester's --export-features +flag; `stake_multiplier` returns 1.0 unless a validated artifact is ever added +under a NEW pre-registration. +""" + +from __future__ import annotations + +import json +import logging +import math +from pathlib import Path +from typing import Dict, Iterable, Optional, Sequence, Tuple + +import numpy as np +import pandas as pd + +from config import SignalStrength, Timeframe + +logger = logging.getLogger(__name__) + + +#: Bump whenever FEATURE_ORDER or any feature's definition changes. +#: A trained artifact carrying a different value is ignored. +FEATURE_VERSION = 1 + +#: Bars each timeframe actually holds in LIVE mode. Any feature whose +#: lookback exceeds its timeframe's depth here would be unreproducible +#: live, so the windows below stay far inside these numbers. +LIVE_DEPTH_BARS: Dict[Timeframe, int] = { + Timeframe.M5: 5000, + Timeframe.M15: 5000, + Timeframe.H1: 1460, + Timeframe.H4: 380, + Timeframe.D1: 300, +} + +#: Hard cap on any primary-timeframe window used in this module. +MAX_LOOKBACK_BARS = 500 + +#: Bars of each confirmation timeframe used by the cross-TF features. +CONFIRM_LOOKBACK_BARS = 60 + +#: Window for the "bars since the last volatility burst" feature. Also +#: the sentinel returned when no burst is present in the window. +BURST_LOOKBACK_BARS = 200 + +#: Burst threshold: bar range >= this multiple of the PRIOR bar's ATR. +BURST_ATR_MULTIPLE = 3.0 + +#: Candle duration in seconds. Deliberate local copy of +#: ``providers.base.BaseDataProvider.CANDLE_SECONDS`` — importing the +#: providers package here would pull requests/twelvedata into every +#: feature-only consumer. A test asserts the copies agree. +CANDLE_SECONDS: Dict[Timeframe, int] = { + Timeframe.M5: 300, + Timeframe.M15: 900, + Timeframe.H1: 3600, + Timeframe.H4: 14400, + Timeframe.D1: 86400, +} + +#: Grade -> ordinal used by ``f_quality_rank`` (D/F both fall to 0). +_QUALITY_RANK = {"A+": 4.0, "A": 3.0, "B": 2.0, "C": 1.0} + +_BULLISH = (SignalStrength.BUY, SignalStrength.STRONG_BUY) +_BEARISH = (SignalStrength.SELL, SignalStrength.STRONG_SELL) + + +FEATURE_ORDER: Tuple[str, ...] = ( + # ── Time ────────────────────────────────────────────────────── + "f_hour_utc", + "f_hour_sin", + "f_hour_cos", + "f_weekday", + "f_sess_asian", + "f_sess_london", + "f_sess_overlap", + "f_sess_ny", + "f_minutes_to_hour", + "f_day_of_month", + "f_first_friday", + # ── Volatility / setup geometry ─────────────────────────────── + "f_atr", + "f_atr_pct", + "f_atr_rel_500", + "f_range_pctile_500", + "f_bar_range_atr", + "f_close_pos_in_bar", + "f_risk_price", + "f_sl_atr", + "f_tp1_atr", + "f_rr_planned", + # ── Exhaustion block ────────────────────────────────────────── + "f_swing_dist_atr", + "f_ema21_dist_atr", + "f_ema200_dist_atr", + "f_ema200_dist_pct", + "f_exhaustion_count", + "f_tag_exhaustion", + "f_tag_no_pullback", + "f_tag_weak_pullback", + "f_tag_rsi_extended", + "f_tag_bb_overextended", + "f_tag_ema200_near", + "f_tag_ema200_far", + "f_penalty_total", + "f_momentum_fading", + # ── Signal quality ──────────────────────────────────────────── + "f_score_dir", + "f_confidence", + "f_quality_rank", + "f_ind_agree_frac", + "f_ind_oppose_frac", + "f_is_buy", + "f_strat_atr", + "f_strat_pb", + "f_strat_cp", + "f_strat_generic", + # ── Multi-timeframe ─────────────────────────────────────────── + "f_mtf_primary", + "f_mtf_c1", + "f_mtf_c2", + "f_mtf_agreement", + # ── Primary-TF indicators ───────────────────────────────────── + "f_rsi", + "f_rsi_dir", + "f_bb_pos", + "f_bb_pos_dir", + "f_adx", + "f_kc_ext_atr", + "f_roc_dir", + "f_macd_hist_dir", + # ── Price path ──────────────────────────────────────────────── + "f_ret_6_atr", + "f_ret_12_atr", + "f_ret_24_atr", + "f_ret_48_atr", + "f_consec_closes_dir", + "f_bars_since_burst", + # ── Cross-timeframe extension ───────────────────────────────── + "f_h1_ext_atr", + "f_c_other_ext_atr", +) + +#: Value each feature falls back to when it cannot be computed. Only +#: ``f_bars_since_burst`` has a non-zero sentinel (its "none in window" +#: cap); everything else degrades to a neutral 0.0. +_DEFAULTS: Dict[str, float] = {name: 0.0 for name in FEATURE_ORDER} +_DEFAULTS["f_bars_since_burst"] = float(BURST_LOOKBACK_BARS) + +#: Penalty tags recorded by SignalAggregator._compute_final_signal that +#: this module exposes as one-hot features. +_TAG_FEATURES = ( + ("f_tag_exhaustion", "exhaustion"), + ("f_tag_no_pullback", "no_pullback"), + ("f_tag_weak_pullback", "weak_pullback"), + ("f_tag_rsi_extended", "rsi_extended"), + ("f_tag_bb_overextended", "bb_overextended"), + ("f_tag_ema200_near", "ema200_near"), + ("f_tag_ema200_far", "ema200_far"), +) + + +# ══════════════════════════════════════════════════════════════════ +# Small helpers +# ══════════════════════════════════════════════════════════════════ + +def _finite(value, default: float = 0.0) -> float: + """Coerce to a finite Python float, falling back to ``default``.""" + try: + out = float(value) + except (TypeError, ValueError): + return float(default) + if not math.isfinite(out): + return float(default) + return out + + +def wilder_atr(df: pd.DataFrame, period: int = 14) -> pd.Series: + """Wilder ATR, identical convention to ``engine._compute_atr``. + + True Range, then an EWM with ``alpha = 1/period``, + ``min_periods = period`` and ``adjust=False``. + """ + high, low, close = df["High"], df["Low"], df["Close"] + tr1 = high - low + tr2 = (high - close.shift()).abs() + tr3 = (low - close.shift()).abs() + tr = pd.concat([tr1, tr2, tr3], axis=1).max(axis=1) + return tr.ewm(alpha=1.0 / period, min_periods=period, adjust=False).mean() + + +def decision_time(primary_df: pd.DataFrame, tf: Timeframe) -> pd.Timestamp: + """The clock every gate uses: last CLOSED bar label + its duration.""" + last = pd.Timestamp(primary_df.index[-1]) + return last + pd.Timedelta(seconds=CANDLE_SECONDS.get(tf, 3600)) + + +def session_label(hour_utc: int) -> str: + """Trading-session bucket for a UTC hour.""" + h = int(hour_utc) + if 3 <= h <= 6: + return "asian" + if 7 <= h <= 12: + return "london" + if 13 <= h <= 16: + return "overlap" + if 17 <= h <= 22: + return "ny" + return "off" + + +# ══════════════════════════════════════════════════════════════════ +# Artifact plumbing +# ══════════════════════════════════════════════════════════════════ + +#: path -> (mtime, artifact-or-None) +_ARTIFACT_CACHE: Dict[str, Tuple[float, Optional[dict]]] = {} + + +def artifact_path_for(primary_tf: Timeframe, base_dir) -> Path: + """Where the trained artifact for one primary timeframe lives.""" + return Path(base_dir) / f"xauusd_{primary_tf.value}.json" + + +def load_artifact(path) -> Optional[dict]: + """Load a sizing artifact (JSON object), cached on (path, mtime). + + Returns ``None`` when the file is missing, unreadable, not JSON, or + not a JSON object. Never raises. + """ + try: + p = Path(path) + mtime = p.stat().st_mtime + except Exception: + return None + + key = str(p) + cached = _ARTIFACT_CACHE.get(key) + if cached is not None and cached[0] == mtime: + return cached[1] + + data: Optional[dict] + try: + with p.open("r", encoding="utf-8") as fh: + loaded = json.load(fh) + data = loaded if isinstance(loaded, dict) else None + except Exception: + logger.debug("position_sizing: artifact %s is unreadable", key, + exc_info=True) + data = None + + _ARTIFACT_CACHE[key] = (mtime, data) + return data + + +# ══════════════════════════════════════════════════════════════════ +# Feature construction +# ══════════════════════════════════════════════════════════════════ + +def _direction(setup, report) -> float: + """+1.0 for a long, -1.0 for a short — from the TRADE, not the score.""" + direction = getattr(setup, "direction", None) + if direction is None: + return 1.0 if _finite(getattr(report, "final_score", 0.0)) >= 0 else -1.0 + return 1.0 if str(direction).upper() == "BUY" else -1.0 + + +def _ext_vs_sma(df: pd.DataFrame, d: float) -> float: + """Direction-signed distance from the 20-bar SMA in ATR units. + + Computed on the LAST ``CONFIRM_LOOKBACK_BARS`` rows only, so the + value is identical on a full history and on its live-depth tail. + """ + if df is None or len(df) < 21: + return 0.0 + tail = df.iloc[-CONFIRM_LOOKBACK_BARS:] + sma = tail["Close"].rolling(20).mean().iloc[-1] + atr = wilder_atr(tail, 14).iloc[-1] + atr_f = _finite(atr, 0.0) + if atr_f <= 0: + return 0.0 + return d * (_finite(tail["Close"].iloc[-1]) - _finite(sma)) / atr_f + + +def _fill_time(out: Dict[str, float], dt: pd.Timestamp) -> None: + hour = int(dt.hour) + out["f_hour_utc"] = float(hour) + out["f_hour_sin"] = float(math.sin(2.0 * math.pi * hour / 24.0)) + out["f_hour_cos"] = float(math.cos(2.0 * math.pi * hour / 24.0)) + weekday = int(dt.weekday()) + out["f_weekday"] = float(weekday) + label = session_label(hour) + out["f_sess_asian"] = 1.0 if label == "asian" else 0.0 + out["f_sess_london"] = 1.0 if label == "london" else 0.0 + out["f_sess_overlap"] = 1.0 if label == "overlap" else 0.0 + out["f_sess_ny"] = 1.0 if label == "ny" else 0.0 + out["f_minutes_to_hour"] = float((60 - int(dt.minute)) % 60) + out["f_day_of_month"] = float(int(dt.day)) + out["f_first_friday"] = 1.0 if (weekday == 4 and int(dt.day) <= 7) else 0.0 + + +def _fill_geometry(out: Dict[str, float], P: pd.DataFrame, setup, + atr: float, atr_ok: bool, norm, d: float) -> None: + out["f_atr"] = max(0.0, _finite(atr)) + ranges = (P["High"] - P["Low"]).astype(float) + last_range = _finite(ranges.iloc[-1]) + mean_range = _finite(ranges.mean()) + out["f_atr_rel_500"] = (_finite(atr) / mean_range) if mean_range > 0 else 0.0 + n = int(len(ranges)) + out["f_range_pctile_500"] = ( + float((ranges.to_numpy() < last_range).sum()) / n if n else 0.0) + out["f_bar_range_atr"] = norm(last_range) + + high = _finite(P["High"].iloc[-1]) + low = _finite(P["Low"].iloc[-1]) + close = _finite(P["Close"].iloc[-1]) + span = high - low + if span > 0: + out["f_close_pos_in_bar"] = ( + (close - low) / span if d > 0 else (high - close) / span) + else: + out["f_close_pos_in_bar"] = 0.5 + + entry = _finite(getattr(setup, "entry_price", None)) + stop = _finite(getattr(setup, "stop_loss", None)) + tp1 = _finite(getattr(setup, "take_profit_1", None)) + risk = abs(entry - stop) + out["f_risk_price"] = risk + out["f_sl_atr"] = norm(risk) + out["f_tp1_atr"] = norm(abs(tp1 - entry)) + out["f_rr_planned"] = _finite(getattr(setup, "risk_reward", None)) + + +def _fill_exhaustion(out: Dict[str, float], report, KL: Dict[str, float], + px: float, d: float, norm) -> None: + if d > 0: + swings = [KL[k] for k in ("swing_low_1", "swing_low_2", "swing_low_3") + if KL.get(k) is not None] + out["f_swing_dist_atr"] = norm(px - min(swings)) if swings else 0.0 + else: + swings = [KL[k] for k in ("swing_high_1", "swing_high_2", "swing_high_3") + if KL.get(k) is not None] + out["f_swing_dist_atr"] = norm(max(swings) - px) if swings else 0.0 + + ema21 = KL.get("ema_medium") + out["f_ema21_dist_atr"] = ( + norm(d * (px - _finite(ema21))) if ema21 is not None else 0.0) + + ema200 = KL.get("ema_trend") + ema200_f = _finite(ema200, 0.0) if ema200 is not None else 0.0 + if ema200 is not None and ema200_f != 0.0: + out["f_ema200_dist_atr"] = norm(d * (px - ema200_f)) + out["f_ema200_dist_pct"] = d * (px - ema200_f) / ema200_f * 100.0 + else: + out["f_ema200_dist_atr"] = 0.0 + out["f_ema200_dist_pct"] = 0.0 + + out["f_exhaustion_count"] = _finite(getattr(report, "exhaustion_count", 0)) + tags = getattr(report, "penalty_tags", None) or {} + for feature, tag in _TAG_FEATURES: + out[feature] = 1.0 if tag in tags else 0.0 + out["f_penalty_total"] = _finite(getattr(report, "penalty_total", 0.0)) + out["f_momentum_fading"] = 1.0 if getattr(report, "momentum_fading", False) else 0.0 + + +def _fill_quality(out: Dict[str, float], report, setup, IND: dict, + d: float) -> None: + out["f_score_dir"] = d * _finite(getattr(report, "final_score", 0.0)) + out["f_confidence"] = _finite(getattr(report, "final_confidence", 0.0)) + out["f_quality_rank"] = _QUALITY_RANK.get( + getattr(report, "signal_quality", ""), 0.0) + + agree = oppose = total = 0 + for name, ind in (IND or {}).items(): + if str(name).startswith("_"): + continue + if _finite(getattr(ind, "weight", 0.0)) == 0.0: + continue + total += 1 + signal = getattr(ind, "signal", None) + if d > 0: + if signal in _BULLISH: + agree += 1 + elif signal in _BEARISH: + oppose += 1 + else: + if signal in _BEARISH: + agree += 1 + elif signal in _BULLISH: + oppose += 1 + out["f_ind_agree_frac"] = (agree / total) if total else 0.0 + out["f_ind_oppose_frac"] = (oppose / total) if total else 0.0 + + out["f_is_buy"] = 1.0 if d > 0 else 0.0 + strategy = str(getattr(setup, "strategy", "") or "") + # The ATR path stamps "XAU_ATR"; "XAU" is accepted for legacy setups. + out["f_strat_atr"] = 1.0 if strategy in ("XAU_ATR", "XAU") else 0.0 + out["f_strat_pb"] = 1.0 if strategy == "XAU_PB" else 0.0 + out["f_strat_cp"] = 1.0 if strategy == "XAU_CP" else 0.0 + out["f_strat_generic"] = 1.0 if strategy == "GENERIC" else 0.0 + + +def _fill_mtf(out: Dict[str, float], report, primary_tf: Timeframe, + confirmation_tfs: Sequence[Timeframe], d: float) -> None: + scores = getattr(report, "mtf_scores", None) or {} + out["f_mtf_primary"] = d * _finite(scores.get(primary_tf.value, 0.0)) + confirms = list(confirmation_tfs or []) + for idx, key in ((0, "f_mtf_c1"), (1, "f_mtf_c2")): + if idx < len(confirms): + out[key] = d * _finite(scores.get(confirms[idx].value, 0.0)) + else: + out[key] = 0.0 + out["f_mtf_agreement"] = 1.0 if getattr(report, "mtf_agreement", False) else 0.0 + + +def _fill_indicators(out: Dict[str, float], IND: dict, d: float, norm) -> None: + IND = IND or {} + + def _val(name, default): + ind = IND.get(name) + return _finite(getattr(ind, "value", None), default) if ind else default + + rsi = _val("rsi", 50.0) + out["f_rsi"] = rsi + out["f_rsi_dir"] = d * (rsi - 50.0) + bb_pos = _val("bollinger", 0.5) # bollinger.value = bb_position + out["f_bb_pos"] = bb_pos + out["f_bb_pos_dir"] = d * (bb_pos - 0.5) + out["f_adx"] = _val("adx", 0.0) + # keltner.value = close - kc_mid (signed extension from the mid line) + out["f_kc_ext_atr"] = norm(d * _val("keltner", 0.0)) + out["f_roc_dir"] = d * _val("roc", 0.0) + # macd.value = histogram + out["f_macd_hist_dir"] = d * _val("macd", 0.0) + + +def _fill_path(out: Dict[str, float], P: pd.DataFrame, d: float, norm) -> None: + close = P["Close"].astype(float).to_numpy() + n = len(close) + for lag, key in ((6, "f_ret_6_atr"), (12, "f_ret_12_atr"), + (24, "f_ret_24_atr"), (48, "f_ret_48_atr")): + out[key] = norm(d * (close[-1] - close[-1 - lag])) if n >= lag + 1 else 0.0 + + run = 0.0 + if n >= 2: + diffs = np.diff(close) + last = diffs[-1] + if last > 0: + sign = 1.0 + elif last < 0: + sign = -1.0 + else: + sign = 0.0 + if sign != 0.0: + count = 0 + for value in diffs[::-1]: + if (value > 0 and sign > 0) or (value < 0 and sign < 0): + count += 1 + else: + break + run = sign * count + out["f_consec_closes_dir"] = max(-20.0, min(20.0, d * run)) + + out["f_bars_since_burst"] = float(BURST_LOOKBACK_BARS) + prev_atr = wilder_atr(P, 14).shift(1).to_numpy(dtype=float) + ranges = (P["High"] - P["Low"]).astype(float).to_numpy() + start = max(0, n - BURST_LOOKBACK_BARS) + for i in range(n - 1, start - 1, -1): + threshold = prev_atr[i] + if not math.isfinite(threshold) or threshold <= 0: + continue + if ranges[i] >= BURST_ATR_MULTIPLE * threshold: + out["f_bars_since_burst"] = float((n - 1) - i) + break + + +def _fill_cross_tf(out: Dict[str, float], dfs: Dict[Timeframe, pd.DataFrame], + confirmation_tfs: Sequence[Timeframe], d: float) -> None: + out["f_h1_ext_atr"] = _ext_vs_sma(dfs.get(Timeframe.H1), d) + out["f_c_other_ext_atr"] = 0.0 + for tf in (confirmation_tfs or []): + if tf is Timeframe.H1: + continue + df = dfs.get(tf) + if df is None or len(df) < 21: + continue + out["f_c_other_ext_atr"] = _ext_vs_sma(df, d) + break + + +def build_features( + report, + setup, + dfs: Dict[Timeframe, pd.DataFrame], + primary_tf: Timeframe, + confirmation_tfs: Sequence[Timeframe], +) -> Dict[str, float]: + """Decision-time feature vector for one signal. + + Returns a dict whose keys are exactly ``FEATURE_ORDER`` (in order), + every value a finite Python float. Degenerate inputs fall back to + the neutral defaults instead of raising — this runs inside a scan. + """ + out: Dict[str, float] = dict(_DEFAULTS) + + try: + P = dfs.get(primary_tf) + if P is None or len(P) == 0: + return {k: out[k] for k in FEATURE_ORDER} + P = P.iloc[-MAX_LOOKBACK_BARS:] + + d = _direction(setup, report) + primary = getattr(report, "primary_analysis", None) + KL = dict(getattr(primary, "key_levels", None) or {}) + IND = getattr(primary, "indicators", None) or {} + + atr_raw = KL.get("atr") + atr = _finite(atr_raw, 0.0) + atr_ok = atr > 0 + atr_div = atr if atr_ok else 1e-9 + + def norm(value: float) -> float: + """ATR-normalise, or 0.0 when ATR is missing/non-positive.""" + return (_finite(value) / atr_div) if atr_ok else 0.0 + + px = _finite(P["Close"].iloc[-1]) + out["f_atr_pct"] = _finite(KL.get("atr_pct"), 0.0) + + for step, args in ( + (_fill_time, (out, decision_time(P, primary_tf))), + (_fill_geometry, (out, P, setup, atr, atr_ok, norm, d)), + (_fill_exhaustion, (out, report, KL, px, d, norm)), + (_fill_quality, (out, report, setup, IND, d)), + (_fill_mtf, (out, report, primary_tf, confirmation_tfs, d)), + (_fill_indicators, (out, IND, d, norm)), + (_fill_path, (out, P, d, norm)), + (_fill_cross_tf, (out, dfs, confirmation_tfs, d)), + ): + try: + step(*args) + except Exception: + logger.debug("position_sizing: %s failed", step.__name__, + exc_info=True) + except Exception: + logger.debug("position_sizing.build_features failed", exc_info=True) + + return {k: _finite(out.get(k), _DEFAULTS[k]) for k in FEATURE_ORDER} + + +# ══════════════════════════════════════════════════════════════════ +# Stake multiplier +# ══════════════════════════════════════════════════════════════════ + +def stake_multiplier( + report, + setup, + dfs: Dict[Timeframe, pd.DataFrame], + primary_tf: Timeframe, + confirmation_tfs: Sequence[Timeframe], + artifact_dir=None, +) -> float: + """Stake multiplier for a taken setup. Always ``1.0`` at this stage. + + The guard order is already final — no artifact directory, a missing + artifact file, an artifact built for a different ``FEATURE_VERSION``, + or ANY exception all yield the neutral ``1.0``. Artifact evaluation + lands in a later step; until then this is wiring, not a decision. + """ + try: + if not artifact_dir: + return 1.0 + artifact = load_artifact(artifact_path_for(primary_tf, artifact_dir)) + if artifact is None: + return 1.0 + if artifact.get("feature_version") != FEATURE_VERSION: + logger.debug( + "position_sizing: artifact feature_version %r != %d — neutral", + artifact.get("feature_version"), FEATURE_VERSION) + return 1.0 + # Step 1: evaluation not implemented yet. + return 1.0 + except Exception: + logger.debug("position_sizing.stake_multiplier failed", exc_info=True) + return 1.0 diff --git a/signal_aggregator.py b/signal_aggregator.py index baad38c..0ebf864 100755 --- a/signal_aggregator.py +++ b/signal_aggregator.py @@ -70,6 +70,10 @@ class TradeSetup: # platform defaults; 0.0 means trailing is genuinely disabled. trail_arm_r: Optional[float] = None trail_dist_r: Optional[float] = None + # Post-decision stake multiplier from position_sizing (advisory + # overlay). None = unsized — every gate/exit path ignores it, and + # the backtester's R accounting is unaffected unless it opts in. + stake_multiplier: Optional[float] = None @dataclass @@ -128,6 +132,13 @@ class SignalReport: market_regime: str = "" volatility_regime: str = "" exhaustion_count: int = 0 + # Record-keeping only (no gate reads these): the penalty tags that + # fired in _compute_final_signal (tag -> RAW pre-decay amount), the + # decayed total actually subtracted from confidence, and whether the + # momentum-fading half of the exhaustion block was true. + penalty_tags: Dict[str, float] = field(default_factory=dict) + penalty_total: float = 0.0 + momentum_fading: bool = False class SignalAggregator: @@ -1552,6 +1563,12 @@ class SignalAggregator: if total_penalty >= MAX_PENALTY: break + # Record-keeping for position_sizing / signal-log forensics. + # Purely additive — nothing below reads these fields. + report.penalty_tags = {name: float(amount) for name, amount in penalties} + report.penalty_total = float(total_penalty) + report.momentum_fading = bool(momentum_fading) + confidence -= total_penalty confidence = max(0.0, min(95.0, confidence)) diff --git a/test_all.py b/test_all.py index e6b34dd..07c5ba6 100644 --- a/test_all.py +++ b/test_all.py @@ -9229,6 +9229,445 @@ def _(): os.unlink(path) +# ══════════════════════════════════════════════════════════════════════ +# MODULE 29: stake-multiplier sizing — features + record-keeping +# ══════════════════════════════════════════════════════════════════════ + +print("\n═══ Module 29: stake-multiplier sizing ═══") + +import position_sizing as _ps + +#: Every tag name that _compute_final_signal can append to `penalties`. +_M29_KNOWN_TAGS = { + "mtf_disagreement", "news_conflict", "event_caution", "counter_trend", + "extreme_vol", "ema200_near", "ema200_far", "exhaustion", + "no_pullback", "weak_pullback", "rsi_extended", "bb_overextended", +} + + +def _m29_ramp(n=300, start=2400.0, step=1.2, noise=0.25, seed=1, freq="5min"): + """Steep, low-noise uptrend — rides the upper Bollinger band, so the + bb_overextended / no_pullback / rsi_extended penalties fire.""" + rng = np.random.default_rng(seed) + idx = pd.date_range(end=pd.Timestamp("2026-08-19 12:00", tz="UTC"), + periods=n, freq=freq) + close = start + np.arange(n) * step + rng.normal(0, noise, n) + high = close + np.abs(rng.normal(0, noise, n)) + 0.2 + low = close - np.abs(rng.normal(0, noise, n)) - 0.2 + open_ = close - rng.normal(0, noise, n) + df = pd.DataFrame({"Open": open_, "High": high, "Low": low, + "Close": close, "Volume": np.full(n, 1000.0)}, + index=idx) + df.index.name = "Datetime" + return df + + +_M29_CACHE = {} + + +def _m29_live(n=1500): + """One REAL analyze_from_dataframes run (M5 primary) + a synthetic + BUY setup on its close. Cached — the replay is the slow part.""" + key = ("live", n) + if key not in _M29_CACHE: + cfg = PlatformConfig() + cfg.primary_timeframe = Timeframe.M5 + cfg.confirmation_timeframes = [Timeframe.M15, Timeframe.H1] + dfs = { + Timeframe.M5: _m29_ramp(n=n, seed=5), + Timeframe.M15: _m29_ramp(n=400, seed=55, freq="15min"), + Timeframe.H1: _m29_ramp(n=200, seed=95, freq="1h"), + } + report = SignalAggregator(cfg).analyze_from_dataframes( + "XAU/USD", dfs, use_news=False) + px = report.current_price + setup = TradeSetup( + pair="XAU/USD", display_name="XAU/USD", direction="BUY", + entry_price=px, stop_loss=px - 6.0, take_profit_1=px + 12.0, + take_profit_2=px + 20.0, risk_reward=2.0, risk_pips=60.0, + reward_pips=120.0, pip_value=0.10, strategy="XAU_ATR") + _M29_CACHE[key] = (cfg, dfs, report, setup) + return _M29_CACHE[key] + + +def _m29_fixture(direction="BUY", score=42.0): + """Hand-built report/setup/frames — no engine run, full control over + key_levels, indicators and MTF scores. Levels are pinned RELATIVE to + the frame's last close, which is the price build_features reads.""" + dfs = { + Timeframe.M5: _m29_ramp(n=300, seed=7), + Timeframe.M15: _m29_ramp(n=120, seed=8, freq="15min"), + Timeframe.H1: _m29_ramp(n=120, seed=9, freq="1h"), + } + px = float(dfs[Timeframe.M5]["Close"].iloc[-1]) + report = SignalReport(symbol="XAU/USD", display_name="XAU/USD", + timestamp="2026-08-19T12:00:00+00:00") + report.final_score = score + report.final_confidence = 61.0 + report.signal_quality = "A" + report.mtf_agreement = True + report.mtf_scores = {Timeframe.M5.value: 30.0, Timeframe.M15.value: 22.0, + Timeframe.H1.value: 14.0} + report.current_price = px + report.exhaustion_count = 2 + report.penalty_tags = {"no_pullback": 8.0, "rsi_extended": 5.0} + report.penalty_total = 10.5 + report.momentum_fading = True + report.primary_analysis = TechnicalAnalysis( + symbol="XAU/USD", timeframe=Timeframe.M5.value, + timestamp=report.timestamp, + indicators={ + "rsi": IndicatorSignal("RSI", 68.0, SignalStrength.BUY, 70.0, "", 1.5), + "bollinger": IndicatorSignal("BB", 0.9, SignalStrength.SELL, 60.0, "", 1.5), + "adx": IndicatorSignal("ADX", 31.0, SignalStrength.BUY, 60.0, "", 0.5), + "keltner": IndicatorSignal("KC", 4.0, SignalStrength.BUY, 60.0, "", 1.5), + "roc": IndicatorSignal("ROC", 0.8, SignalStrength.BUY, 60.0, "", 1.5), + "macd": IndicatorSignal("MACD", 1.25, SignalStrength.BUY, 60.0, "", 1.5), + "atr": IndicatorSignal("ATR", 5.0, SignalStrength.NEUTRAL, 50.0, "", 0.0), + }, + composite_signal=SignalStrength.BUY, composite_score=score, + confidence=61.0, market_regime="trending_up", volatility_regime="normal", + key_levels={ + "atr": 5.0, "atr_pct": 0.2, + "ema_medium": px - 10.0, "ema_trend": px - 50.0, + "swing_low_1": px - 30.0, "swing_low_2": px - 40.0, + "swing_low_3": px - 50.0, + "swing_high_1": px + 10.0, "swing_high_2": px + 15.0, + "swing_high_3": px + 20.0, + }, + ) + setup = TradeSetup( + pair="XAU/USD", display_name="XAU/USD", direction=direction, + entry_price=px, stop_loss=px - 6.0 if direction == "BUY" else px + 6.0, + take_profit_1=px + 12.0 if direction == "BUY" else px - 12.0, + take_profit_2=px + 20.0 if direction == "BUY" else px - 20.0, + risk_reward=2.0, risk_pips=60.0, reward_pips=120.0, pip_value=0.10, + strategy="XAU_PB") + return report, setup, dfs + + +@test("sizing: SignalReport records penalty tags incl. bb_overextended") +def _(): + _cfg, _dfs, report, _setup = _m29_live() + assert report.penalty_tags, "penalty tags must be recorded" + assert "bb_overextended" in report.penalty_tags, \ + f"bb_overextended must be recorded, got {report.penalty_tags}" + assert report.penalty_tags["bb_overextended"] == 8.0, "raw pre-decay amount" + # The tag is invisible in `warnings` — that is exactly why the + # record-keeping dict exists. + assert not any("bb_overextended" in w for w in report.warnings), \ + f"bb_overextended must not surface as a warning: {report.warnings}" + unknown = set(report.penalty_tags) - _M29_KNOWN_TAGS + assert not unknown, f"unknown penalty tags recorded: {unknown}" + + +@test("sizing: penalty_total equals the decayed sum subtracted from confidence") +def _(): + _cfg, _dfs, report, _setup = _m29_live() + assert len(report.penalty_tags) >= 2, \ + f"need >=2 penalties to exercise the decay, got {report.penalty_tags}" + max_penalty, total, decay = 30.0, 0.0, 1.0 + for _name, amount in sorted(report.penalty_tags.items(), + key=lambda kv: kv[1], reverse=True): + applied = amount * decay + if total + applied > max_penalty: + applied = max_penalty - total + total += applied + decay *= 0.5 + if total >= max_penalty: + break + assert abs(total - report.penalty_total) < 1e-9, \ + f"recomputed {total} != recorded {report.penalty_total}" + + +@test("sizing: build_features returns every FEATURE_ORDER key, all finite") +def _(): + cfg, dfs, report, setup = _m29_live() + feats = _ps.build_features(report, setup, dfs, Timeframe.M5, + cfg.confirmation_timeframes) + assert list(feats.keys()) == list(_ps.FEATURE_ORDER), \ + "keys must be exactly FEATURE_ORDER, in order" + assert len(_ps.FEATURE_ORDER) == len(set(_ps.FEATURE_ORDER)), \ + "FEATURE_ORDER must not contain duplicates" + for name, value in feats.items(): + assert type(value) is float, f"{name} is {type(value)}, not float" + assert np.isfinite(value), f"{name} is not finite: {value}" + assert all(n.startswith("f_") for n in _ps.FEATURE_ORDER) + + +@test("sizing: exhaustion features are signed by TRADE direction, not score sign") +def _(): + report, setup, dfs = _m29_fixture(direction="SELL", score=42.0) + assert report.final_score > 0 and setup.direction == "SELL" + feats = _ps.build_features(report, setup, dfs, Timeframe.M5, + [Timeframe.M15, Timeframe.H1]) + # SELL must measure to the HIGHEST tracked swing high (px + 20), not + # the lowest swing low (px - 50), and the EMA21 distance (px - 10) + # must carry the SELL sign. ATR = 5. + assert abs(feats["f_swing_dist_atr"] - 20.0 / 5.0) < 1e-9, \ + f"expected swing-high distance 4.0, got {feats['f_swing_dist_atr']}" + assert abs(feats["f_ema21_dist_atr"] - (-10.0 / 5.0)) < 1e-9, \ + f"expected SELL-signed EMA21 distance -2.0, got {feats['f_ema21_dist_atr']}" + assert feats["f_score_dir"] < 0, "a SELL on a positive score scores negative" + assert feats["f_is_buy"] == 0.0 + + +@test("sizing: direction-signed features flip between a BUY and its mirrored SELL") +def _(): + report, buy, dfs = _m29_fixture(direction="BUY") + _r2, sell, _d2 = _m29_fixture(direction="SELL") + a = _ps.build_features(report, buy, dfs, Timeframe.M5, + [Timeframe.M15, Timeframe.H1]) + b = _ps.build_features(report, sell, dfs, Timeframe.M5, + [Timeframe.M15, Timeframe.H1]) + for name in ("f_ema21_dist_atr", "f_ret_12_atr", "f_rsi_dir", + "f_score_dir", "f_mtf_primary"): + assert a[name] != 0.0, f"{name} must be non-zero for the flip test" + assert abs(a[name] + b[name]) < 1e-9, \ + f"{name} must flip sign: {a[name]} vs {b[name]}" + for name in ("f_atr", "f_hour_utc", "f_atr_pct", "f_weekday", + "f_penalty_total"): + assert a[name] == b[name], f"{name} must be direction-independent" + + +@test("sizing: decision time is last closed label + candle duration") +def _(): + from providers.base import BaseDataProvider as _BDP + for tf in _ps.CANDLE_SECONDS: + assert _ps.CANDLE_SECONDS[tf] == _BDP.CANDLE_SECONDS[tf], \ + f"CANDLE_SECONDS copy diverged on {tf}" + for tf, freq in ((Timeframe.M5, "5min"), (Timeframe.M15, "15min"), + (Timeframe.H1, "1h"), (Timeframe.H4, "4h")): + df = _m29_ramp(n=30, freq=freq) + expected = df.index[-1] + pd.Timedelta(seconds=_ps.CANDLE_SECONDS[tf]) + assert _ps.decision_time(df, tf) == expected, tf + # The decision clock, not the bar label, drives the time features. + df = _m29_ramp(n=30, freq="1h") + report, setup, _dfs = _m29_fixture() + feats = _ps.build_features(report, setup, {Timeframe.H1: df}, + Timeframe.H1, []) + assert feats["f_hour_utc"] == float( + (df.index[-1] + pd.Timedelta(hours=1)).hour) + assert _ps.session_label(4) == "asian" and _ps.session_label(9) == "london" + assert _ps.session_label(14) == "overlap" and _ps.session_label(20) == "ny" + assert _ps.session_label(1) == "off" and _ps.session_label(23) == "off" + + +@test("sizing: features identical on a full frame and on its live-depth tail") +def _(): + cfg, dfs, report, setup = _m29_live() + assert len(dfs[Timeframe.M5]) > _ps.MAX_LOOKBACK_BARS, "need a long frame" + full = _ps.build_features(report, setup, dfs, Timeframe.M5, + cfg.confirmation_timeframes) + tail = _ps.build_features( + report, setup, + {Timeframe.M5: dfs[Timeframe.M5].iloc[-_ps.MAX_LOOKBACK_BARS:], + Timeframe.M15: dfs[Timeframe.M15].iloc[-_ps.CONFIRM_LOOKBACK_BARS:], + Timeframe.H1: dfs[Timeframe.H1].iloc[-_ps.CONFIRM_LOOKBACK_BARS:]}, + Timeframe.M5, cfg.confirmation_timeframes) + assert full == tail, { + k: (full[k], tail[k]) for k in full if full[k] != tail[k]} + + +@test("sizing: stake_multiplier returns 1.0 with no artifact dir, missing file, or stale feature_version") +def _(): + cfg, dfs, report, setup = _m29_live() + args = (report, setup, dfs, Timeframe.M5, cfg.confirmation_timeframes) + assert _ps.stake_multiplier(*args) == 1.0, "default (no dir) must be neutral" + assert _ps.stake_multiplier(*args, artifact_dir=None) == 1.0 + with tempfile.TemporaryDirectory() as tmp: + assert _ps.stake_multiplier(*args, artifact_dir=tmp) == 1.0, \ + "missing artifact file must be neutral" + path = _ps.artifact_path_for(Timeframe.M5, tmp) + assert path.name == "xauusd_5min.json", path.name + path.write_text(json.dumps({"feature_version": 999, "weights": {}})) + assert _ps.load_artifact(path)["feature_version"] == 999 + assert _ps.stake_multiplier(*args, artifact_dir=tmp) == 1.0, \ + "stale feature_version must be neutral" + path.write_text("{not json") + # mtime granularity: force a cache miss so the invalid body is read + os.utime(path, (0, 0)) + assert _ps.load_artifact(path) is None, "invalid JSON -> None" + assert _ps.stake_multiplier(*args, artifact_dir=tmp) == 1.0 + assert _ps.load_artifact(os.path.join(tmp, "nope.json")) is None + + +@test("sizing: build_features never raises on degenerate input") +def _(): + report, setup, _dfs = _m29_fixture() + tiny = _m29_ramp(n=3, seed=3) + + def _check(feats): + assert list(feats.keys()) == list(_ps.FEATURE_ORDER) + for name, value in feats.items(): + assert type(value) is float and np.isfinite(value), (name, value) + return feats + + # atr = 0 -> every ATR-normalised feature collapses to 0.0 + report.primary_analysis.key_levels = {"atr": 0.0} + f = _check(_ps.build_features(report, setup, {Timeframe.M5: tiny}, + Timeframe.M5, [Timeframe.M15])) + for name in ("f_sl_atr", "f_tp1_atr", "f_swing_dist_atr", + "f_ema21_dist_atr", "f_bar_range_atr", "f_ret_6_atr"): + assert f[name] == 0.0, f"{name} must be 0.0 when ATR is unusable" + assert f["f_bars_since_burst"] == 200.0, "sentinel when no burst in window" + assert f["f_rsi"] == 68.0 and f["f_bb_pos"] == 0.9, \ + "un-normalised indicator readings survive an unusable ATR" + + # empty key_levels + no indicators + no confirmation frames + report.primary_analysis.key_levels = {} + report.primary_analysis.indicators = {} + f = _check(_ps.build_features(report, setup, {Timeframe.M5: tiny}, + Timeframe.M5, [])) + assert f["f_rsi"] == 50.0 and f["f_bb_pos"] == 0.5, \ + "missing indicators fall back to their neutral readings" + assert f["f_ind_agree_frac"] == 0.0 and f["f_ind_oppose_frac"] == 0.0 + # no primary analysis at all + report.primary_analysis = None + _check(_ps.build_features(report, setup, {Timeframe.M5: tiny}, + Timeframe.M5, [Timeframe.M15, Timeframe.H1])) + # no frames, no setup + _check(_ps.build_features(report, None, {}, Timeframe.M5, [])) + _check(_ps.build_features(None, None, {Timeframe.M5: tiny.iloc[:0]}, + Timeframe.M5, None)) + + +@test("sizing: TradeSetup.stake_multiplier defaults to None (no sizing overlay ships)") +def _(): + # 2026-09-06 verdict: the stake-multiplier model FAILED the pre-registered + # rule on M5 and M15, so no SizingConfig / artifact exists. The field is + # reserved (None = unsized) and the backtester reads it as 1.0. + ts = TradeSetup(pair="XAU/USD", display_name="XAU/USD", direction="BUY", + entry_price=2450.0, stop_loss=2444.0, take_profit_1=2462.0, + take_profit_2=2470.0, risk_reward=2.0, risk_pips=60.0, + reward_pips=120.0, pip_value=0.10) + assert ts.stake_multiplier is None + assert not hasattr(PlatformConfig(), "sizing"), "rule 8: no unread config section" + +_M29_PINNED_COLS = [ + "pair", "strategy", "quality", "regime", "vol_regime", "direction", + "entry_time", "entry_weekday", "entry", "sl", "tp1", "rr_planned", "score", + "confidence", "indicators_agree", "indicators_oppose", "indicators_neutral", + "indicators_total", "entry_reasons", "warnings", "exit_time", "exit", + "exit_reason", "bars_held", "r_multiple", "mfe_r", "mae_r", +] +_M29_IND_NAMES = ["ema_trend", "rsi", "bollinger", "adx", "macd", "roc", + "keltner", "candlestick", "divergence"] + + +def _m29_trade(r_multiple=1.0, stake=1.0, features=None): + """A resolved backtest Trade with a full 9-indicator breakdown.""" + from gpu_backtest import Trade + t = Trade(pair="XAU/USD", + entry_time=pd.Timestamp("2026-01-05 10:00", tz="UTC"), + entry_price=3300.0, sl=3292.0, tp1=3312.0, tp2=3320.0, + direction="BUY", pip_value=0.10, risk_pips=80.0, + reward_pips=120.0, rr_planned=1.5, quality="B", + final_score=30.0, final_confidence=60.0, + market_regime="trending_up", volatility_regime="normal", + strategy_tag="GENERIC", + entry_reasons=["r1"], entry_warnings=["w1"]) + t.indicator_breakdown = { + name: {"signal": "BUY", "value": 1.0 + i, "confidence": 60.0, + "agrees": True} + for i, name in enumerate(_M29_IND_NAMES) + } + t.indicators_agreeing, t.indicators_total = 9, 9 + t.exit_time = pd.Timestamp("2026-01-05 12:00", tz="UTC") + t.exit_price = 3312.0 + t.exit_reason = "TP1" + t.bars_held = 24 + t.r_multiple = r_multiple + t.mfe_r, t.mae_r = 1.8, 0.3 + if features is not None: + t.features = features + t.stake_multiplier = stake + return t + + +@test("sizing: Trade defaults keep features empty and stake_multiplier 1.0") +def _(): + from gpu_backtest import Trade + bare = Trade(pair="XAU/USD", + entry_time=pd.Timestamp("2026-01-05 10:00", tz="UTC"), + entry_price=3300.0, sl=3292.0, tp1=3312.0, tp2=3320.0, + direction="BUY", pip_value=0.10, risk_pips=80.0, + reward_pips=120.0, rr_planned=1.5, quality="B", + final_score=30.0, final_confidence=60.0, + market_regime="trending_up") + assert bare.features == {}, "an unsized trade carries no features" + assert bare.stake_multiplier == 1.0, "neutral stake by default" + # Two Trades must not share one features dict (mutable default trap). + bare.features["f_hour_utc"] = 9.0 + assert Trade(pair="XAU/USD", + entry_time=pd.Timestamp("2026-01-05 10:00", tz="UTC"), + entry_price=3300.0, sl=3292.0, tp1=3312.0, tp2=3320.0, + direction="BUY", pip_value=0.10, risk_pips=80.0, + reward_pips=120.0, rr_planned=1.5, quality="B", + final_score=30.0, final_confidence=60.0, + market_regime="trending_up").features == {} + + +@test("sizing: trades_to_dataframe default export keeps the pinned column set") +def _(): + from gpu_backtest import trades_to_dataframe + feats_a = {name: float(i) for i, name in enumerate(_ps.FEATURE_ORDER)} + trades = [ + _m29_trade(r_multiple=1.5, stake=1.25, features=feats_a), + _m29_trade(r_multiple=-1.0, stake=0.75, features=dict(feats_a)), + ] + + off = trades_to_dataframe(trades) + assert len(off.columns) == 63, \ + f"pinned books are 63 columns, got {len(off.columns)}" + assert list(off.columns[:27]) == _M29_PINNED_COLS, list(off.columns[:27]) + assert not any(c.startswith("f_") or c in ("stake_multiplier", "r_sized") + for c in off.columns), "feature columns must stay off by default" + + on = trades_to_dataframe(trades, include_features=True) + expected = (_M29_PINNED_COLS + list(off.columns[27:]) + + ["stake_multiplier", "r_sized"] + list(_ps.FEATURE_ORDER)) + assert len(on.columns) == 63 + 2 + len(_ps.FEATURE_ORDER), \ + f"expected {63 + 2 + len(_ps.FEATURE_ORDER)} columns, got {len(on.columns)}" + assert list(on.columns) == expected, \ + [c for c in zip(list(on.columns), expected) if c[0] != c[1]][:5] + # The default export must be a strict prefix — pinned books stay diffable. + pd.testing.assert_frame_equal(off, on[off.columns]) + for i in range(len(trades)): + assert on["r_sized"].iloc[i] == ( + on["r_multiple"].iloc[i] * on["stake_multiplier"].iloc[i]) + assert on["f_hour_utc"].iloc[i] == feats_a["f_hour_utc"] + # A trade with no features (flag on, but nothing computed) → NaN, not 0. + partial = trades_to_dataframe([_m29_trade()], include_features=True) + assert np.isnan(partial["f_atr"].iloc[0]), "missing features export as NaN" + assert partial["stake_multiplier"].iloc[0] == 1.0 + # 2026-09-05 regression: when the FIRST trade lacks an indicator that a + # later trade carries, pandas appends that indicator's columns at first + # appearance. The extras must still land after every base column so the + # default export stays a strict prefix (seen on the 3yr M5 book, whose + # first trade had no candlestick pattern). + first, second = _m29_trade(features=feats_a), _m29_trade(features=dict(feats_a)) + for suffix in ("candlestick",): + first.indicator_breakdown.pop(suffix) + off2 = trades_to_dataframe([first, second]) + on2 = trades_to_dataframe([first, second], include_features=True) + assert list(on2.columns[:len(off2.columns)]) == list(off2.columns), \ + "extras must follow ALL base columns even when a later trade adds one" + assert list(on2.columns[len(off2.columns):]) == ( + ["stake_multiplier", "r_sized"] + list(_ps.FEATURE_ORDER)) + pd.testing.assert_frame_equal(off2, on2[off2.columns]) + + +@test("sizing: --export-features parses and defaults off") +def _(): + from gpu_backtest import _parse_args + assert _parse_args(["--pair", "XAUUSD"]).export_features is False + args = _parse_args(["--pair", "XAUUSD", "--export-features", + "--export-csv", "/tmp/x.csv"]) + assert args.export_features is True + assert args.export_csv == "/tmp/x.csv" + + # ══════════════════════════════════════════════════════════════════════ # SUMMARY # ══════════════════════════════════════════════════════════════════════