backtest cleanup: drop four research/one-time scripts; books and stale cache moved out (owner decisions 2026-09-15)
Discussion-first cleanup of backtest/ and scripts/. Nothing in the signal
core or the backtester changed.
Removed (last commit containing them: 931b265):
- backtest/diagnose_filters.py + its two Module 18 tests
- scripts/shadow_scan.py (Phase 4 shadow scan, never run)
- scripts/backfill_news_archive.py, scripts/migrate_signal_log_news.py
(one-time scripts; the interning test keeps its live-path half, the
prune test goes)
Kept: the replay-recording stack (replay_record.py, replay_scorecard.py,
ff_calendar_scraper.py, cache/calendar.parquet, Module 33).
Local gitignored data was moved to ~/Development/ForexBot2_trash_2026-09-15
(346 MB) for the owner to delete: all 35 pinned backtest/*.csv books, the
three research output folders, the BTC/ETH cache snapshots, the Tiingo
parquets and 13 uncited Oanda windows. The three research-folder ignore
rules are dropped with the folders.
Verification: all five gate books regenerate offline from the pruned cache
with the recorded MD5s (M5 W2 29120a1e, M15 W2 d185f099, M15 OOS 6f8f24b4,
H1 W3 3b45d12b, H4 W3 e75adae1; NumPy CPU, 0 cache fallbacks). Suite
506 passed, 0 failed, 33 modules.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Harwinder Sekhon
@@ -1,627 +0,0 @@
|
||||
"""
|
||||
ForexBot2 — Filter Rejection Diagnostic
|
||||
========================================
|
||||
Replays the same backtest loop as gpu_backtest.py but instead of
|
||||
collecting trades, counts how many M15 bars are rejected by each
|
||||
gate in the signal pipeline.
|
||||
|
||||
Produces a summary table showing where potential trades are being
|
||||
lost, helping tune filter thresholds.
|
||||
|
||||
Usage:
|
||||
python backtest/diagnose_filters.py --pair XAUUSD --days 365
|
||||
python backtest/diagnose_filters.py --pair XAUUSD --days 365 --sample-every 1
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
# Ensure project root on sys.path.
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from config import ( # noqa: E402
|
||||
PAIR_INFO,
|
||||
PlatformConfig,
|
||||
SignalStrength,
|
||||
Timeframe,
|
||||
)
|
||||
from signal_aggregator import SignalAggregator, SignalReport # noqa: E402
|
||||
|
||||
# Reuse GPU backtest infrastructure (data loading, indicator pre-compute, etc.)
|
||||
from gpu_backtest import ( # noqa: E402
|
||||
BAR_SECONDS,
|
||||
GPUTechnicalEngine,
|
||||
PAIR_ALIASES,
|
||||
IndicatorCache,
|
||||
fetch_history,
|
||||
precompute_indicators,
|
||||
_is_xau_blocked_by_calendar,
|
||||
)
|
||||
from providers import make_data_provider # noqa: E402
|
||||
|
||||
logger = logging.getLogger("diagnose_filters")
|
||||
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════
|
||||
# OUTCOME CLASSIFICATION
|
||||
# ═════════════════════════════════════════════════════════════════════
|
||||
|
||||
def classify_bar(
|
||||
report: SignalReport,
|
||||
is_xau: bool,
|
||||
) -> Tuple[str, List[str]]:
|
||||
"""Classify a bar's outcome and list which filters killed it.
|
||||
|
||||
Returns:
|
||||
(outcome, [filter_names])
|
||||
|
||||
Outcomes:
|
||||
"TRADE" — quality C+ and trade_setup exists
|
||||
"QUALITY_OK_NO_SETUP" — quality C+ but trade_setup is None
|
||||
"LOW_QUALITY" — signal reached BUY/SELL but quality D or F
|
||||
"NEUTRAL" — signal stayed NEUTRAL (score didn't reach +/-30)
|
||||
|
||||
Filter names are extracted from report.warnings.
|
||||
"""
|
||||
filters_hit: List[str] = []
|
||||
|
||||
# Known filter tags, matched against the exact warning strings the
|
||||
# aggregator emits (refreshed 2026-07-30 for gates 6d-6i). ORDER
|
||||
# MATTERS: most-specific prefix first, and each warning maps to at
|
||||
# most ONE tag (break on first hit) — substring families like
|
||||
# "M5 FILTER: SELL quality" vs "M5 FILTER" and "TREND EXHAUSTION"
|
||||
# vs "EXHAUSTION" would otherwise double-count.
|
||||
filter_tags = [
|
||||
# M5 noise gates (6i, _apply_m5_filters)
|
||||
("M5 FILTER: |score|", "M5_SCORE"),
|
||||
("M5 FILTER: no MTF agreement", "M5_MTF"),
|
||||
("M5 FILTER: no confirmation TF", "M5_CONF"),
|
||||
("M5 FILTER: SELL quality", "M5_SELL_QUALITY"),
|
||||
("M5 FILTER: SELL blocked — RSI", "M5_SELL_RSI"),
|
||||
("M5 FILTER: '", "M5_VOL"),
|
||||
("M5 FILTER", "M5_OTHER"),
|
||||
# XAU SELL gates (6e/6g/6g2/6g3)
|
||||
("SELL FILTER: quality", "SELL_QUALITY"),
|
||||
("SELL FILTER: volatility", "SELL_VOL"),
|
||||
("SELL FILTER: regime", "SELL_REGIME"),
|
||||
("UTC entry blocked for shorts", "SELL_HOUR"),
|
||||
# Hard gates
|
||||
("MTF OPPOSITION", "MTF_OPPOSITION"),
|
||||
("EXHAUSTED TREND FILTER", "EXHAUSTED_TREND"),
|
||||
("BIG CANDLE FILTER", "BIG_CANDLE"),
|
||||
("WEEKEND FILTER", "WEEKEND_FILTER"),
|
||||
("SESSION FILTER", "SESSION_FILTER"),
|
||||
("DAY FILTER", "DAY_FILTER"),
|
||||
("LOW ATR FILTER", "LOW_ATR_FILTER"),
|
||||
("REGIME FILTER", "REGIME_FILTER"),
|
||||
# Soft penalties / context warnings
|
||||
("TREND EXHAUSTION", "TREND_EXHAUSTION"),
|
||||
("EXHAUSTION", "EXHAUSTION_PENALTY"),
|
||||
("NO PULLBACK", "NO_PULLBACK_PENALTY"),
|
||||
("Weak pullback", "WEAK_PULLBACK_PENALTY"),
|
||||
("Extended — RSI", "RSI_EXTENDED_PENALTY"),
|
||||
("Counter-trend", "COUNTER_TREND_PENALTY"),
|
||||
("EXTREME volatility", "EXTREME_VOL_PENALTY"),
|
||||
("NEAR EMA200", "NEAR_EMA200_PENALTY"),
|
||||
("FAR FROM EMA200", "FAR_EMA200_PENALTY"),
|
||||
("EVENT RISK", "EVENT_RISK"),
|
||||
("Timeframes CONFLICT", "MTF_CONFLICT"),
|
||||
]
|
||||
|
||||
for w in report.warnings:
|
||||
for tag, name in filter_tags:
|
||||
if tag in w:
|
||||
filters_hit.append(name)
|
||||
break
|
||||
|
||||
# Determine outcome
|
||||
non_neutral = report.final_signal not in (SignalStrength.NEUTRAL,)
|
||||
quality_ok = report.signal_quality in ("A+", "A", "B", "C")
|
||||
|
||||
if quality_ok and report.trade_setup is not None:
|
||||
return "TRADE", filters_hit
|
||||
elif quality_ok and report.trade_setup is None:
|
||||
return "QUALITY_OK_NO_SETUP", filters_hit
|
||||
elif non_neutral and not quality_ok:
|
||||
return "LOW_QUALITY", filters_hit
|
||||
else:
|
||||
return "NEUTRAL", filters_hit
|
||||
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════
|
||||
# SCORE DISTRIBUTION TRACKER
|
||||
# ═════════════════════════════════════════════════════════════════════
|
||||
|
||||
class ScoreTracker:
|
||||
"""Tracks score/confidence distributions for analysis."""
|
||||
|
||||
def __init__(self):
|
||||
self.scores: List[float] = []
|
||||
self.confidences: List[float] = []
|
||||
self.abs_scores: List[float] = []
|
||||
# How many bars had score in various ranges
|
||||
self.score_bins = Counter()
|
||||
self.conf_bins = Counter()
|
||||
# Track "near miss" bars — score close to thresholds
|
||||
self.near_miss_score: int = 0 # score 20-29 (almost BUY/SELL)
|
||||
self.near_miss_conf: int = 0 # confidence 45-54 (almost passing)
|
||||
|
||||
def record(self, score: float, confidence: float):
|
||||
self.scores.append(score)
|
||||
self.confidences.append(confidence)
|
||||
self.abs_scores.append(abs(score))
|
||||
|
||||
# Score bins
|
||||
a = abs(score)
|
||||
if a < 10:
|
||||
self.score_bins["0-9"] += 1
|
||||
elif a < 20:
|
||||
self.score_bins["10-19"] += 1
|
||||
elif a < 30:
|
||||
self.score_bins["20-29"] += 1
|
||||
self.near_miss_score += 1
|
||||
elif a < 40:
|
||||
self.score_bins["30-39"] += 1
|
||||
elif a < 50:
|
||||
self.score_bins["40-49"] += 1
|
||||
else:
|
||||
self.score_bins["50+"] += 1
|
||||
|
||||
# Confidence bins
|
||||
if confidence < 30:
|
||||
self.conf_bins["0-29"] += 1
|
||||
elif confidence < 45:
|
||||
self.conf_bins["30-44"] += 1
|
||||
elif confidence < 55:
|
||||
self.conf_bins["45-54"] += 1
|
||||
self.near_miss_conf += 1
|
||||
elif confidence < 65:
|
||||
self.conf_bins["55-64"] += 1
|
||||
elif confidence < 75:
|
||||
self.conf_bins["65-74"] += 1
|
||||
else:
|
||||
self.conf_bins["75+"] += 1
|
||||
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════
|
||||
# MAIN DIAGNOSTIC LOOP
|
||||
# ═════════════════════════════════════════════════════════════════════
|
||||
|
||||
def run_diagnosis(
|
||||
pair: str,
|
||||
start: datetime,
|
||||
end: datetime,
|
||||
sample_every: int = 4,
|
||||
use_cache: bool = True,
|
||||
primary_tf: Timeframe = Timeframe.M15,
|
||||
confirm_tfs: Optional[List[Timeframe]] = None,
|
||||
warmup_bars: int = 250,
|
||||
) -> None:
|
||||
if confirm_tfs is None:
|
||||
confirm_tfs = [Timeframe.H1, Timeframe.H4]
|
||||
|
||||
pair_info = PAIR_INFO[pair]
|
||||
display = pair_info["display"]
|
||||
is_xau = display == "XAU/USD"
|
||||
|
||||
platform_cfg = PlatformConfig(
|
||||
symbol=pair,
|
||||
display_name=display,
|
||||
primary_timeframe=primary_tf,
|
||||
confirmation_timeframes=list(confirm_tfs),
|
||||
)
|
||||
|
||||
provider = make_data_provider(platform_cfg)
|
||||
|
||||
# ── Load data ──
|
||||
timeframes = [primary_tf] + list(confirm_tfs)
|
||||
pad_days = max(60, warmup_bars // 4)
|
||||
fetch_start = start - timedelta(days=pad_days)
|
||||
|
||||
dfs_full: Dict[Timeframe, pd.DataFrame] = {}
|
||||
for tf in timeframes:
|
||||
df = fetch_history(
|
||||
provider, pair, tf,
|
||||
start=fetch_start, end=end,
|
||||
use_cache=use_cache,
|
||||
)
|
||||
if df.empty:
|
||||
raise RuntimeError(f"No data for {pair} {tf.value}")
|
||||
dfs_full[tf] = df
|
||||
|
||||
# ── Pre-compute indicators ──
|
||||
t0 = time.perf_counter()
|
||||
caches: Dict[str, IndicatorCache] = {
|
||||
tf.value: precompute_indicators(dfs_full[tf], platform_cfg.technical, gpu_log=True)
|
||||
for tf in timeframes
|
||||
}
|
||||
logger.info(f"Indicator pre-compute: {time.perf_counter() - t0:.2f}s")
|
||||
|
||||
# ── Build aggregator with GPU engine ──
|
||||
aggregator = SignalAggregator(platform_cfg)
|
||||
aggregator.tech_engine = GPUTechnicalEngine(platform_cfg.technical, caches)
|
||||
|
||||
# ── Walk primary TF ──
|
||||
primary_df = dfs_full[primary_tf]
|
||||
bt_start_ts = pd.Timestamp(start)
|
||||
bt_end_ts = pd.Timestamp(end)
|
||||
|
||||
first_eligible = max(
|
||||
warmup_bars,
|
||||
int(primary_df.index.searchsorted(bt_start_ts)),
|
||||
)
|
||||
last_pos = min(
|
||||
int(primary_df.index.searchsorted(bt_end_ts, side="right")),
|
||||
len(primary_df),
|
||||
)
|
||||
|
||||
primary_dur = pd.Timedelta(seconds=BAR_SECONDS[primary_tf])
|
||||
|
||||
# Counters
|
||||
total_bars_in_window = 0
|
||||
total_bars_sampled = 0
|
||||
outcome_counts = Counter()
|
||||
filter_counts = Counter()
|
||||
# Filters that killed bars that were otherwise directional (non-NEUTRAL score)
|
||||
directional_filter_counts = Counter()
|
||||
# Pre-pipeline rejections (XAU calendar filters applied before analyze_from_dataframes)
|
||||
pre_pipeline = Counter()
|
||||
|
||||
# Track signal directions
|
||||
signal_dir_counts = Counter()
|
||||
quality_counts = Counter()
|
||||
|
||||
# Score tracker
|
||||
tracker = ScoreTracker()
|
||||
|
||||
# Track "would-have-been" trades: bars where the primary score was
|
||||
# directional but got killed by a specific filter
|
||||
filter_kill_breakdown = defaultdict(Counter)
|
||||
|
||||
logger.info(
|
||||
f"Diagnosing {display} {primary_tf.value}: "
|
||||
f"bars {first_eligible}-{last_pos} (every {sample_every}th bar)"
|
||||
)
|
||||
|
||||
t_loop = time.perf_counter()
|
||||
for pos in range(first_eligible, last_pos, sample_every):
|
||||
total_bars_in_window += 1
|
||||
ts = primary_df.index[pos]
|
||||
decision_time = ts + primary_dur
|
||||
|
||||
# ── XAU calendar pre-filter (same as gpu_backtest.py) ──
|
||||
if is_xau:
|
||||
block = _is_xau_blocked_by_calendar(ts, platform_cfg)
|
||||
if block:
|
||||
pre_pipeline[block] += 1
|
||||
continue
|
||||
|
||||
total_bars_sampled += 1
|
||||
|
||||
# ── Build sliced dfs (same logic as gpu_backtest.py) ──
|
||||
slice_dfs: Dict[Timeframe, pd.DataFrame] = {}
|
||||
for tf in timeframes:
|
||||
full = dfs_full[tf]
|
||||
bar_dur = pd.Timedelta(seconds=BAR_SECONDS[tf])
|
||||
cutoff = decision_time - bar_dur
|
||||
cut = int(full.index.searchsorted(cutoff, side="right"))
|
||||
if cut == 0:
|
||||
continue
|
||||
slice_dfs[tf] = full.iloc[:cut]
|
||||
|
||||
if primary_tf not in slice_dfs:
|
||||
continue
|
||||
|
||||
# ── Run the full signal pipeline ──
|
||||
report: SignalReport = aggregator.analyze_from_dataframes(
|
||||
symbol=pair,
|
||||
dfs=slice_dfs,
|
||||
context=None,
|
||||
use_news=False,
|
||||
)
|
||||
|
||||
# Record score/confidence
|
||||
tracker.record(report.final_score, report.final_confidence)
|
||||
|
||||
# Signal direction
|
||||
signal_dir_counts[report.final_signal.name] += 1
|
||||
quality_counts[report.signal_quality] += 1
|
||||
|
||||
# Classify
|
||||
outcome, filters_hit = classify_bar(report, is_xau)
|
||||
outcome_counts[outcome] += 1
|
||||
|
||||
for f in filters_hit:
|
||||
filter_counts[f] += 1
|
||||
|
||||
# If the primary score was directional (>= 15 or <= -15), the bar
|
||||
# had some potential — track what killed it
|
||||
if abs(report.primary_score) >= 15:
|
||||
for f in filters_hit:
|
||||
directional_filter_counts[f] += 1
|
||||
|
||||
# For bars that got quality D/F but had a directional signal,
|
||||
# track which filter warnings appeared
|
||||
if outcome in ("LOW_QUALITY", "QUALITY_OK_NO_SETUP"):
|
||||
for f in filters_hit:
|
||||
filter_kill_breakdown[outcome][f] += 1
|
||||
|
||||
elapsed = time.perf_counter() - t_loop
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════
|
||||
# PRINT RESULTS
|
||||
# ═════════════════════════════════════════════════════════════════
|
||||
|
||||
bar = "=" * 80
|
||||
print(f"\n{bar}")
|
||||
print(f" FILTER REJECTION DIAGNOSTIC — {display}")
|
||||
print(f" {start.date()} to {end.date()} | {primary_tf.value} "
|
||||
f"(sampled every {sample_every}th bar)")
|
||||
print(f"{bar}")
|
||||
|
||||
# Scale counts if sampling
|
||||
scale_note = ""
|
||||
if sample_every > 1:
|
||||
scale_note = f" (sampled {total_bars_sampled} of ~{total_bars_in_window * sample_every} eligible bars)"
|
||||
print(f"\n Total bars in backtest window: {total_bars_in_window}")
|
||||
if pre_pipeline:
|
||||
for reason, count in sorted(pre_pipeline.items()):
|
||||
print(f" Pre-pipeline rejection ({reason}): {count}")
|
||||
print(f" Bars analyzed: {total_bars_sampled}{scale_note}")
|
||||
print(f" Analysis took: {elapsed:.1f}s "
|
||||
f"({total_bars_sampled / max(elapsed, 0.001):.0f} bars/s)")
|
||||
|
||||
# ── Outcome breakdown ──
|
||||
print(f"\n{'-' * 80}")
|
||||
print(" OUTCOME BREAKDOWN")
|
||||
print(f"{'-' * 80}")
|
||||
outcome_order = ["TRADE", "QUALITY_OK_NO_SETUP", "LOW_QUALITY", "NEUTRAL"]
|
||||
outcome_labels = {
|
||||
"TRADE": "Actual trades (quality C+ & setup exists)",
|
||||
"QUALITY_OK_NO_SETUP": "Quality C+ but NO trade setup generated",
|
||||
"LOW_QUALITY": "Directional signal but quality D/F",
|
||||
"NEUTRAL": "NEUTRAL (score didn't reach +/-30 or conf < 55)",
|
||||
}
|
||||
for o in outcome_order:
|
||||
count = outcome_counts.get(o, 0)
|
||||
pct = count / max(total_bars_sampled, 1) * 100
|
||||
label = outcome_labels.get(o, o)
|
||||
print(f" {count:6d} ({pct:5.1f}%) {label}")
|
||||
|
||||
# ── Signal direction breakdown ──
|
||||
print(f"\n{'-' * 80}")
|
||||
print(" SIGNAL DIRECTION BREAKDOWN")
|
||||
print(f"{'-' * 80}")
|
||||
for name in ["STRONG_BUY", "BUY", "NEUTRAL", "SELL", "STRONG_SELL"]:
|
||||
count = signal_dir_counts.get(name, 0)
|
||||
pct = count / max(total_bars_sampled, 1) * 100
|
||||
print(f" {count:6d} ({pct:5.1f}%) {name}")
|
||||
|
||||
# ── Quality grade breakdown ──
|
||||
print(f"\n{'-' * 80}")
|
||||
print(" QUALITY GRADE BREAKDOWN")
|
||||
print(f"{'-' * 80}")
|
||||
for grade in ["A+", "A", "B", "C", "D", "F"]:
|
||||
count = quality_counts.get(grade, 0)
|
||||
pct = count / max(total_bars_sampled, 1) * 100
|
||||
print(f" {count:6d} ({pct:5.1f}%) Grade {grade}")
|
||||
|
||||
# ── Filter hit counts (all bars) ──
|
||||
print(f"\n{'-' * 80}")
|
||||
print(" FILTER HIT COUNTS (all bars, a bar can hit multiple filters)")
|
||||
print(f"{'-' * 80}")
|
||||
if filter_counts:
|
||||
for name, count in filter_counts.most_common():
|
||||
pct = count / max(total_bars_sampled, 1) * 100
|
||||
print(f" {count:6d} ({pct:5.1f}%) {name}")
|
||||
else:
|
||||
print(" (no filters triggered)")
|
||||
|
||||
# ── Filters that killed directional bars (primary_score >= 15) ──
|
||||
print(f"\n{'-' * 80}")
|
||||
print(" FILTERS KILLING DIRECTIONAL BARS (|primary_score| >= 15)")
|
||||
print(f"{'-' * 80}")
|
||||
if directional_filter_counts:
|
||||
for name, count in directional_filter_counts.most_common():
|
||||
print(f" {count:6d} {name}")
|
||||
else:
|
||||
print(" (none)")
|
||||
|
||||
# ── Filter breakdown for LOW_QUALITY bars ──
|
||||
if filter_kill_breakdown.get("LOW_QUALITY"):
|
||||
print(f"\n{'-' * 80}")
|
||||
print(" FILTERS ON LOW_QUALITY BARS (had BUY/SELL but quality D/F)")
|
||||
print(f"{'-' * 80}")
|
||||
for name, count in filter_kill_breakdown["LOW_QUALITY"].most_common():
|
||||
print(f" {count:6d} {name}")
|
||||
|
||||
# ── Filter breakdown for QUALITY_OK_NO_SETUP bars ──
|
||||
if filter_kill_breakdown.get("QUALITY_OK_NO_SETUP"):
|
||||
print(f"\n{'-' * 80}")
|
||||
print(" FILTERS ON QUALITY_OK_NO_SETUP BARS (quality C+ but no trade setup)")
|
||||
print(f"{'-' * 80}")
|
||||
for name, count in filter_kill_breakdown["QUALITY_OK_NO_SETUP"].most_common():
|
||||
print(f" {count:6d} {name}")
|
||||
|
||||
# ── Score distribution ──
|
||||
print(f"\n{'-' * 80}")
|
||||
print(" SCORE DISTRIBUTION (|score|)")
|
||||
print(f"{'-' * 80}")
|
||||
for bin_name in ["0-9", "10-19", "20-29", "30-39", "40-49", "50+"]:
|
||||
count = tracker.score_bins.get(bin_name, 0)
|
||||
pct = count / max(total_bars_sampled, 1) * 100
|
||||
marker = " <-- near miss (almost BUY/SELL)" if bin_name == "20-29" else ""
|
||||
print(f" {count:6d} ({pct:5.1f}%) |score| {bin_name}{marker}")
|
||||
|
||||
# ── Confidence distribution ──
|
||||
print(f"\n{'-' * 80}")
|
||||
print(" CONFIDENCE DISTRIBUTION")
|
||||
print(f"{'-' * 80}")
|
||||
for bin_name in ["0-29", "30-44", "45-54", "55-64", "65-74", "75+"]:
|
||||
count = tracker.conf_bins.get(bin_name, 0)
|
||||
pct = count / max(total_bars_sampled, 1) * 100
|
||||
marker = " <-- near miss (almost passing)" if bin_name == "45-54" else ""
|
||||
print(f" {count:6d} ({pct:5.1f}%) confidence {bin_name}{marker}")
|
||||
|
||||
# ── Near misses ──
|
||||
print(f"\n{'-' * 80}")
|
||||
print(" NEAR MISSES")
|
||||
print(f"{'-' * 80}")
|
||||
print(f" {tracker.near_miss_score:6d} Bars with |score| 20-29 (threshold is 30)")
|
||||
print(f" {tracker.near_miss_conf:6d} Bars with confidence 45-54 (threshold is 55)")
|
||||
|
||||
# ── Summary statistics ──
|
||||
if tracker.scores:
|
||||
scores_arr = np.array(tracker.scores)
|
||||
conf_arr = np.array(tracker.confidences)
|
||||
print(f"\n{'-' * 80}")
|
||||
print(" SCORE/CONFIDENCE STATISTICS")
|
||||
print(f"{'-' * 80}")
|
||||
print(f" Score: mean={np.mean(scores_arr):+.1f} "
|
||||
f"median={np.median(scores_arr):+.1f} "
|
||||
f"std={np.std(scores_arr):.1f} "
|
||||
f"min={np.min(scores_arr):+.1f} "
|
||||
f"max={np.max(scores_arr):+.1f}")
|
||||
print(f" Confidence: mean={np.mean(conf_arr):.1f} "
|
||||
f"median={np.median(conf_arr):.1f} "
|
||||
f"std={np.std(conf_arr):.1f} "
|
||||
f"min={np.min(conf_arr):.1f} "
|
||||
f"max={np.max(conf_arr):.1f}")
|
||||
|
||||
# ── Blocked hours analysis (for XAU) ──
|
||||
if is_xau:
|
||||
blocked = set(platform_cfg.xau_strategy.blocked_hours_utc)
|
||||
all_hours = set(range(24))
|
||||
open_hours = sorted(all_hours - blocked)
|
||||
print(f"\n{'-' * 80}")
|
||||
print(" XAU HOUR/DAY CONFIG")
|
||||
print(f"{'-' * 80}")
|
||||
print(f" Blocked hours: {sorted(blocked)}")
|
||||
print(f" Open hours: {open_hours}")
|
||||
print(f" Allowed days: {platform_cfg.xau_strategy.allowed_weekdays} "
|
||||
f"(0=Mon..4=Fri)")
|
||||
print(f" Open fraction: {len(open_hours)/24:.0%} of hours x "
|
||||
f"{len(platform_cfg.xau_strategy.allowed_weekdays)}/5 weekdays = "
|
||||
f"{len(open_hours)/24 * len(platform_cfg.xau_strategy.allowed_weekdays)/5:.0%} of trading time")
|
||||
|
||||
# ── Interpretation / key takeaways ──
|
||||
print(f"\n{bar}")
|
||||
print(" KEY TAKEAWAYS")
|
||||
print(f"{bar}")
|
||||
|
||||
neutral_pct = outcome_counts.get("NEUTRAL", 0) / max(total_bars_sampled, 1) * 100
|
||||
low_q_pct = outcome_counts.get("LOW_QUALITY", 0) / max(total_bars_sampled, 1) * 100
|
||||
trade_count = outcome_counts.get("TRADE", 0)
|
||||
|
||||
print(f" - {neutral_pct:.1f}% of bars never reach a directional signal (NEUTRAL)")
|
||||
print(f" - {low_q_pct:.1f}% of bars get a BUY/SELL but confidence/quality too low")
|
||||
print(f" - {trade_count} bars produce actual trades")
|
||||
|
||||
if filter_counts:
|
||||
top3 = filter_counts.most_common(3)
|
||||
print(f" - Top rejection filters:")
|
||||
for name, count in top3:
|
||||
print(f" {name}: {count} bars hit")
|
||||
|
||||
if tracker.near_miss_score > 50:
|
||||
print(f" - SIGNIFICANT: {tracker.near_miss_score} bars had |score| 20-29 "
|
||||
f"(lowering threshold from 30 to 25 would capture many)")
|
||||
|
||||
if tracker.near_miss_conf > 50:
|
||||
print(f" - SIGNIFICANT: {tracker.near_miss_conf} bars had confidence 45-54 "
|
||||
f"(lowering threshold from 55 to 50 would capture many)")
|
||||
|
||||
print(f"\n{bar}\n")
|
||||
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════
|
||||
# CLI
|
||||
# ═════════════════════════════════════════════════════════════════════
|
||||
|
||||
def _resolve_pair(s: str) -> str:
|
||||
s_up = s.upper().strip()
|
||||
if s_up in PAIR_ALIASES:
|
||||
return PAIR_ALIASES[s_up]
|
||||
if s_up in PAIR_INFO:
|
||||
return s_up
|
||||
raise SystemExit(f"Unknown pair '{s}'. Valid: {', '.join(sorted(PAIR_ALIASES))}")
|
||||
|
||||
|
||||
def _resolve_tf(s: str) -> Timeframe:
|
||||
for tf in Timeframe:
|
||||
if tf.value == s:
|
||||
return tf
|
||||
raise SystemExit(f"Unknown timeframe '{s}'")
|
||||
|
||||
|
||||
def main(argv: Optional[List[str]] = None) -> int:
|
||||
p = argparse.ArgumentParser(
|
||||
description="Diagnose which filters reject the most bars in the backtest pipeline",
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
||||
)
|
||||
p.add_argument("--pair", required=True, help="XAUUSD | GOLD")
|
||||
p.add_argument("--days", type=int, default=365, help="Days of history")
|
||||
p.add_argument("--start", help="ISO date YYYY-MM-DD (overrides --days)")
|
||||
p.add_argument("--end", help="ISO date YYYY-MM-DD (defaults to today)")
|
||||
p.add_argument("--primary-tf", default="15min", help="Primary TF")
|
||||
p.add_argument("--confirm-tfs", default="1h,4h", help="Comma-separated confirmation TFs")
|
||||
p.add_argument("--warmup-bars", type=int, default=250)
|
||||
p.add_argument("--sample-every", type=int, default=4,
|
||||
help="Analyze every Nth bar (1=all, 4=25%% sample for speed)")
|
||||
p.add_argument("--no-cache", action="store_true", help="Skip parquet cache")
|
||||
p.add_argument("--log-level", default="INFO")
|
||||
args = p.parse_args(argv)
|
||||
|
||||
logging.basicConfig(
|
||||
level=args.log_level.upper(),
|
||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||
)
|
||||
|
||||
pair = _resolve_pair(args.pair)
|
||||
primary_tf = _resolve_tf(args.primary_tf)
|
||||
confirm_tfs = [_resolve_tf(s.strip()) for s in args.confirm_tfs.split(",") if s.strip()]
|
||||
|
||||
end = (
|
||||
datetime.fromisoformat(args.end).replace(tzinfo=timezone.utc)
|
||||
if args.end else datetime.now(timezone.utc)
|
||||
)
|
||||
if args.start:
|
||||
start = datetime.fromisoformat(args.start).replace(tzinfo=timezone.utc)
|
||||
else:
|
||||
start = end - timedelta(days=args.days)
|
||||
|
||||
logger.info(
|
||||
f"Diagnosing {PAIR_INFO[pair]['display']} | "
|
||||
f"Window: {start.date()} -> {end.date()} | "
|
||||
f"Sample: every {args.sample_every}th bar"
|
||||
)
|
||||
|
||||
run_diagnosis(
|
||||
pair=pair,
|
||||
start=start,
|
||||
end=end,
|
||||
sample_every=args.sample_every,
|
||||
use_cache=not args.no_cache,
|
||||
primary_tf=primary_tf,
|
||||
confirm_tfs=confirm_tfs,
|
||||
warmup_bars=args.warmup_bars,
|
||||
)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user