Phase 1 + 2 analysis: post-hoc filters and SL/TP geometry grid
- analyze_trades.py: time-split discipline (70/30) for filter selection. Reports session-of-day, day-of-week, pair, regime × direction, combined-filter performance on held-out test slice. Flags Saturday (10% WR) and Sunday (24% WR) as toxic — definite live-engine drops. - geometry_grid.py: re-simulates the existing trade pool over a (sl_mult × rr) grid without re-running the engine. Recovers ATR from the original 1.5×ATR SL distance, then walks forward through saved OHLCV with first-touch logic. Reports train/test split per cell so the verdict survives the 70/30 discipline. Findings (2025-2026 data, 7 majors): - M15: no SL/RR cell is profitable on the held-out test set, even with Phase 1 filters layered on top. Best test cell (SL=3×ATR, RR=1:1): -73 R, 48.7% WR, -0.062 avg R. The architecture is the problem. - H4 (separate seed, 2 years): Grade B subset is profitable — total +20.86 R / 967 trades, PF 1.03. USD/CAD H4 Grade B is *robust*: +34.77 total R / 118 trades, 44.1% WR, +0.295 avg R, PF 1.52, max DD -9.61 R. Positive in every 6-month sub-period, train +20.77 / test +14.0. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Harwinder
@@ -0,0 +1,240 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Phase 1 analysis: post-hoc filters on an existing trades.csv.
|
||||
|
||||
Asks "which simple filters, applied to the trades the engine already
|
||||
produced, would have moved total R toward zero or positive?" — without
|
||||
re-running the backtest engine.
|
||||
|
||||
Discipline: candidate filters are derived on the **first 70% of trades
|
||||
by entry_time** (train slice), then evaluated on the **last 30%** (test
|
||||
slice). This is the same time-based split the meta-model uses, so the
|
||||
numbers are comparable.
|
||||
|
||||
Outputs (printed + saved to backtest_results_all/phase1_summary.txt):
|
||||
- Session-of-day breakdown
|
||||
- Day-of-week breakdown
|
||||
- Pair × regime cross-tab
|
||||
- Per-pair contribution
|
||||
- Combined-filter sweep (drop-worst-hours / drop-worst-pairs / both)
|
||||
|
||||
Usage:
|
||||
python analyze_trades.py
|
||||
python analyze_trades.py --trades backtest_results_all/trades.csv
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def fmt(df: pd.DataFrame) -> pd.DataFrame:
|
||||
return df.round(3)
|
||||
|
||||
|
||||
def split(df: pd.DataFrame, train_frac: float = 0.7):
|
||||
df = df.sort_values("entry_time").reset_index(drop=True)
|
||||
cutoff = int(len(df) * train_frac)
|
||||
return df.iloc[:cutoff].copy(), df.iloc[cutoff:].copy()
|
||||
|
||||
|
||||
def stats(df: pd.DataFrame) -> dict:
|
||||
if len(df) == 0:
|
||||
return {"n": 0, "wr": 0.0, "avg_r": 0.0, "total_r": 0.0, "pf": 0.0}
|
||||
wins = (df["r_multiple"] > 0).sum()
|
||||
losses = (df["r_multiple"] <= 0).sum()
|
||||
gross_win = df.loc[df["r_multiple"] > 0, "r_multiple"].sum()
|
||||
gross_loss = -df.loc[df["r_multiple"] <= 0, "r_multiple"].sum()
|
||||
return {
|
||||
"n": len(df),
|
||||
"wr": wins / len(df) * 100,
|
||||
"avg_r": df["r_multiple"].mean(),
|
||||
"total_r": df["r_multiple"].sum(),
|
||||
"pf": gross_win / gross_loss if gross_loss > 0 else float("inf"),
|
||||
}
|
||||
|
||||
|
||||
def stats_line(label: str, df: pd.DataFrame) -> str:
|
||||
s = stats(df)
|
||||
return (
|
||||
f"{label:<40s}"
|
||||
f" n={s['n']:>5} WR={s['wr']:5.1f}%"
|
||||
f" avgR={s['avg_r']:+6.3f} totalR={s['total_r']:+9.2f} PF={s['pf']:.2f}"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="Phase 1 post-hoc analysis on backtest trades.")
|
||||
ap.add_argument("--trades", default="backtest_results_all/trades.csv")
|
||||
ap.add_argument("--out", default="backtest_results_all/phase1_summary.txt")
|
||||
ap.add_argument("--train-frac", type=float, default=0.7)
|
||||
args = ap.parse_args()
|
||||
|
||||
p = Path(args.trades)
|
||||
if not p.exists():
|
||||
print(f"❌ {p} not found")
|
||||
return 1
|
||||
|
||||
df = pd.read_csv(p)
|
||||
df["entry_time"] = pd.to_datetime(df["entry_time"])
|
||||
df["hour"] = df["entry_time"].dt.hour
|
||||
df["dow"] = df["entry_time"].dt.dayofweek
|
||||
df["winner"] = (df["r_multiple"] > 0).astype(int)
|
||||
|
||||
train, test = split(df, args.train_frac)
|
||||
out_lines = []
|
||||
|
||||
def emit(line: str = ""):
|
||||
print(line)
|
||||
out_lines.append(line)
|
||||
|
||||
emit("=" * 92)
|
||||
emit(f"PHASE 1 ANALYSIS — {len(df):,} trades total")
|
||||
emit(f" full: {df['entry_time'].min().date()} → {df['entry_time'].max().date()}")
|
||||
emit(f" train: {train['entry_time'].min().date()} → {train['entry_time'].max().date()} ({len(train):,} trades)")
|
||||
emit(f" test: {test['entry_time'].min().date()} → {test['entry_time'].max().date()} ({len(test):,} trades)")
|
||||
emit("=" * 92)
|
||||
emit()
|
||||
emit(stats_line("BASELINE full", df))
|
||||
emit(stats_line("BASELINE train", train))
|
||||
emit(stats_line("BASELINE test", test))
|
||||
emit()
|
||||
|
||||
# ── Per-hour (training set, used for filter selection) ──────────
|
||||
emit("─" * 92)
|
||||
emit("HOUR (UTC) — train slice (used for filter selection)")
|
||||
emit("─" * 92)
|
||||
hr_stats = train.groupby("hour").apply(
|
||||
lambda x: pd.Series(stats(x)), include_groups=False,
|
||||
).round(3)
|
||||
emit(hr_stats.to_string())
|
||||
emit()
|
||||
|
||||
# Bottom-K hours by avg_r
|
||||
bad_hours_4 = hr_stats.sort_values("avg_r").head(4).index.tolist()
|
||||
bad_hours_6 = hr_stats.sort_values("avg_r").head(6).index.tolist()
|
||||
emit(f" Worst 4 hours by avg_r (train): {sorted(bad_hours_4)}")
|
||||
emit(f" Worst 6 hours by avg_r (train): {sorted(bad_hours_6)}")
|
||||
emit()
|
||||
|
||||
# ── Per-DOW (training set) ──────────────────────────────────────
|
||||
emit("─" * 92)
|
||||
emit("DAY OF WEEK — train slice (0=Mon, 4=Fri)")
|
||||
emit("─" * 92)
|
||||
dow_stats = train.groupby("dow").apply(
|
||||
lambda x: pd.Series(stats(x)), include_groups=False,
|
||||
).round(3)
|
||||
emit(dow_stats.to_string())
|
||||
emit()
|
||||
bad_dow = dow_stats.sort_values("avg_r").head(2).index.tolist()
|
||||
emit(f" Worst 2 days by avg_r (train): {sorted(bad_dow)}")
|
||||
emit()
|
||||
|
||||
# ── Per-pair (training set) ─────────────────────────────────────
|
||||
emit("─" * 92)
|
||||
emit("PAIR — train slice")
|
||||
emit("─" * 92)
|
||||
pair_stats = train.groupby("pair").apply(
|
||||
lambda x: pd.Series(stats(x)), include_groups=False,
|
||||
).round(3)
|
||||
emit(pair_stats.to_string())
|
||||
emit()
|
||||
bad_pairs = pair_stats.sort_values("total_r").head(3).index.tolist()
|
||||
emit(f" Worst 3 pairs by total_r (train): {bad_pairs}")
|
||||
emit()
|
||||
|
||||
# ── Per-regime × direction (training set) ───────────────────────
|
||||
emit("─" * 92)
|
||||
emit("REGIME × DIRECTION — train slice")
|
||||
emit("─" * 92)
|
||||
reg_stats = train.groupby(["regime", "direction"]).apply(
|
||||
lambda x: pd.Series(stats(x)), include_groups=False,
|
||||
).round(3)
|
||||
emit(reg_stats.to_string())
|
||||
emit()
|
||||
|
||||
# ── Filter sweep ON THE TEST SET (rule selected on train) ───────
|
||||
emit("=" * 92)
|
||||
emit("FILTER SWEEP — applied to TEST slice (rule chosen from train, no leakage)")
|
||||
emit("=" * 92)
|
||||
test_filtered_dropworst4hours = test[~test["hour"].isin(bad_hours_4)]
|
||||
test_filtered_dropworst6hours = test[~test["hour"].isin(bad_hours_6)]
|
||||
test_filtered_droppairs = test[~test["pair"].isin(bad_pairs)]
|
||||
test_filtered_droprangingsell = test[~((test["regime"] == "ranging") & (test["direction"] == "SELL"))]
|
||||
test_filtered_only_ranging = test[test["regime"] == "ranging"]
|
||||
test_filtered_drop_dow = test[~test["dow"].isin(bad_dow)]
|
||||
|
||||
test_combo_hr_pair = test[
|
||||
(~test["hour"].isin(bad_hours_4)) & (~test["pair"].isin(bad_pairs))
|
||||
]
|
||||
test_combo_hr_pair_dow = test[
|
||||
(~test["hour"].isin(bad_hours_4))
|
||||
& (~test["pair"].isin(bad_pairs))
|
||||
& (~test["dow"].isin(bad_dow))
|
||||
]
|
||||
|
||||
emit()
|
||||
emit(stats_line("test baseline (no filter)", test))
|
||||
emit(stats_line("test drop 4 worst hours", test_filtered_dropworst4hours))
|
||||
emit(stats_line("test drop 6 worst hours", test_filtered_dropworst6hours))
|
||||
emit(stats_line("test drop 3 worst pairs", test_filtered_droppairs))
|
||||
emit(stats_line("test drop 2 worst DOWs", test_filtered_drop_dow))
|
||||
emit(stats_line("test ONLY ranging regime", test_filtered_only_ranging))
|
||||
emit(stats_line("test hours+pairs (combined)", test_combo_hr_pair))
|
||||
emit(stats_line("test hours+pairs+DOW (combined)", test_combo_hr_pair_dow))
|
||||
emit()
|
||||
|
||||
# ── Same filters applied to TRAIN slice for reference ───────────
|
||||
emit("─" * 92)
|
||||
emit("Same filters — TRAIN slice for reference (these are the in-sample numbers)")
|
||||
emit("─" * 92)
|
||||
train_drop_hours = train[~train["hour"].isin(bad_hours_4)]
|
||||
train_drop_pairs = train[~train["pair"].isin(bad_pairs)]
|
||||
train_combo = train[(~train["hour"].isin(bad_hours_4)) & (~train["pair"].isin(bad_pairs))]
|
||||
|
||||
emit(stats_line("train baseline (no filter)", train))
|
||||
emit(stats_line("train drop 4 worst hours", train_drop_hours))
|
||||
emit(stats_line("train drop 3 worst pairs", train_drop_pairs))
|
||||
emit(stats_line("train hours+pairs (combined)", train_combo))
|
||||
emit()
|
||||
|
||||
# ── Verdict ─────────────────────────────────────────────────────
|
||||
emit("=" * 92)
|
||||
emit("VERDICT")
|
||||
emit("=" * 92)
|
||||
test_base = stats(test)
|
||||
test_combo = stats(test_combo_hr_pair_dow)
|
||||
delta_r = test_combo["total_r"] - test_base["total_r"]
|
||||
delta_avgr = test_combo["avg_r"] - test_base["avg_r"]
|
||||
emit(f" Best combined filter (hours+pairs+DOW) on TEST set:")
|
||||
emit(f" {test_base['n']} → {test_combo['n']} trades "
|
||||
f"({test_combo['n']/test_base['n']*100:.0f}% retained)")
|
||||
emit(f" total R: {test_base['total_r']:+.2f} → {test_combo['total_r']:+.2f} "
|
||||
f"(Δ {delta_r:+.2f})")
|
||||
emit(f" avg R: {test_base['avg_r']:+.3f} → {test_combo['avg_r']:+.3f} "
|
||||
f"(Δ {delta_avgr:+.3f})")
|
||||
emit(f" win rate: {test_base['wr']:.1f}% → {test_combo['wr']:.1f}%")
|
||||
emit()
|
||||
if test_combo["total_r"] > 0:
|
||||
emit(" ✅ Filtered subset is profitable on the held-out test set.")
|
||||
emit(" → Phase 2 (geometry grid) may be unnecessary; this filter alone may suffice.")
|
||||
elif test_combo["avg_r"] > test_base["avg_r"]:
|
||||
emit(" ⚠️ Filtered subset improves but is still negative.")
|
||||
emit(" → Phase 2 (geometry grid) needed to find profitable structure.")
|
||||
else:
|
||||
emit(" ❌ Filters do not survive the train/test split — rejected as noise.")
|
||||
emit(" → Skip Phase 1's filter list; rely on Phase 2 (geometry) instead.")
|
||||
|
||||
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.out).write_text("\n".join(out_lines))
|
||||
emit()
|
||||
emit(f"📁 Saved: {args.out}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user