Strip backtesting, rebalance indicator weights toward leading signals
Remove backtest.py, backtest_gpu.py, metrics.py, local CSV data, Dukascopy loader, and all backtest CLI modes from main.py. Rebalance engine weights: leading indicators (divergence, RSI, stochastic, BB) now carry 66% of signal weight vs 34% for lagging (EMA, MACD). Add ws_twelvedata.py for live tick streaming. Drop pyarrow and tqdm deps. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Harwinder
@@ -2,14 +2,10 @@
|
||||
"""
|
||||
ForexBot2 — USD/JPY Signal Intelligence (single-pair)
|
||||
=======================================================
|
||||
Entry point for both live signal analysis and walk-forward backtesting.
|
||||
|
||||
Usage:
|
||||
python main.py # Live signal on H1
|
||||
python main.py --timeframe H4 --detail # Detailed H4 signal
|
||||
python main.py --json # JSON output
|
||||
python main.py --backtest # Walk-forward backtest
|
||||
python main.py --backtest --optimize # Parallel grid search
|
||||
"""
|
||||
|
||||
import argparse
|
||||
@@ -233,54 +229,6 @@ def report_to_dict(report: SignalReport) -> dict:
|
||||
return d
|
||||
|
||||
|
||||
# ── Backtest renderer ──────────────────────────────────────────────
|
||||
|
||||
def print_backtest_report(result, params: dict = None, verbose_journal: bool = False):
|
||||
m = result.metrics
|
||||
print(f" {Colors.BOLD}{Colors.WHITE}Backtest Summary — USD/JPY{Colors.RESET}")
|
||||
if params:
|
||||
params_str = ", ".join(f"{k}={v}" for k, v in params.items())
|
||||
print(f" {Colors.GRAY}Params: {params_str}{Colors.RESET}")
|
||||
print(f" {Colors.GRAY}{'─' * 76}{Colors.RESET}")
|
||||
|
||||
initial = result.initial_balance
|
||||
final = result.final_balance
|
||||
ret_pct = m.get("total_return_pct", 0)
|
||||
ret_color = Colors.GREEN if ret_pct >= 0 else Colors.RED
|
||||
|
||||
print(f" Trades: {Colors.BOLD}{m.get('total_trades', 0)}{Colors.RESET}"
|
||||
f" ({m.get('wins', 0)}W / {m.get('losses', 0)}L)")
|
||||
print(f" Win Rate: {m.get('win_rate', 0):.1f}%")
|
||||
print(f" Profit Factor: {m.get('profit_factor', 0):.2f}")
|
||||
pf_inf = m.get('profit_factor', 0) == float('inf')
|
||||
if pf_inf:
|
||||
print(f" (no losing trades — infinite PF)")
|
||||
print(f" Expectancy: {m.get('expectancy_pips', 0):+.2f} pips/trade"
|
||||
f" ({m.get('expectancy_per_trade', 0):+.2f} per trade)")
|
||||
print(f" Initial Balance: ${initial:,.2f}")
|
||||
print(f" Final Balance: {ret_color}${final:,.2f}{Colors.RESET}")
|
||||
print(f" Total Return: {ret_color}{ret_pct:+.2f}%{Colors.RESET}")
|
||||
print(f" CAGR: {m.get('cagr', 0):+.2f}%")
|
||||
print(f" Max Drawdown: {Colors.RED}{m.get('max_drawdown_pct', 0):.2f}%{Colors.RESET}")
|
||||
print(f" Sharpe Ratio: {m.get('sharpe', 0):.2f}")
|
||||
print(f" Sortino Ratio: {m.get('sortino', 0):.2f}")
|
||||
print(f" Avg Duration: {m.get('avg_trade_duration_bars', 0):.1f} bars")
|
||||
print(f" Win Streak: {m.get('longest_winning_streak', 0)} | "
|
||||
f"Loss Streak: {m.get('longest_losing_streak', 0)}")
|
||||
print(f" {Colors.GRAY}{'─' * 76}{Colors.RESET}")
|
||||
|
||||
if verbose_journal and result.trades:
|
||||
print(f"\n {Colors.BOLD}Trade Journal (last 10):{Colors.RESET}")
|
||||
print(f" {'#':<4} {'Entry Time':<22} {'Dir':<5} {'Entry':<9} "
|
||||
f"{'Exit':<9} {'Reason':<6} {'Pips':>8} {'P&L%':>7}")
|
||||
for i, t in enumerate(result.trades[-10:], start=max(1, len(result.trades) - 9)):
|
||||
color = Colors.GREEN if t.pnl_currency > 0 else Colors.RED
|
||||
entry_str = str(t.entry_time)[:19]
|
||||
print(f" {i:<4} {entry_str:<22} {t.direction:<5} {t.entry_price:<9.3f} "
|
||||
f"{t.exit_price:<9.3f} {t.exit_reason:<6} "
|
||||
f"{color}{t.pnl_pips:>+8.1f}{Colors.RESET} "
|
||||
f"{color}{t.pnl_pct:>+6.2f}%{Colors.RESET}")
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
# CLI
|
||||
@@ -288,7 +236,7 @@ def print_backtest_report(result, params: dict = None, verbose_journal: bool = F
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="ForexBot2 — USD/JPY Signal Intelligence + Backtester",
|
||||
description="ForexBot2 — USD/JPY Signal Intelligence",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
@@ -296,58 +244,18 @@ Examples:
|
||||
python main.py --detail Live signal with full breakdown
|
||||
python main.py --timeframe H4 --detail H4 timeframe
|
||||
python main.py --json JSON output
|
||||
python main.py --backtest Walk-forward backtest (full history)
|
||||
python main.py --backtest --bars 2000 Backtest on last 2000 bars
|
||||
python main.py --backtest --optimize Parallel grid search across CPU cores
|
||||
""",
|
||||
)
|
||||
parser.add_argument("--timeframe", "-tf", default="H1",
|
||||
choices=["M5", "M15", "M30", "H1", "H4", "D1"],
|
||||
help="Primary analysis timeframe (default: H1)")
|
||||
parser.add_argument("--detail", action="store_true",
|
||||
help="Show full indicator breakdown (live mode)")
|
||||
help="Show full indicator breakdown")
|
||||
parser.add_argument("--json", action="store_true",
|
||||
help="Output as JSON")
|
||||
parser.add_argument("--verbose", "-v", action="store_true",
|
||||
help="Debug logging")
|
||||
|
||||
# Backtest options
|
||||
parser.add_argument("--backtest", action="store_true",
|
||||
help="Run walk-forward backtest instead of live signal")
|
||||
parser.add_argument("--optimize", action="store_true",
|
||||
help="Run parallel parameter grid search (implies --backtest)")
|
||||
parser.add_argument("--bars", type=int, default=5000,
|
||||
help="Max bars of history (default 5000 — TD ceiling)")
|
||||
parser.add_argument("--start", default=None,
|
||||
help="Backtest window start (YYYY-MM-DD)")
|
||||
parser.add_argument("--end", default=None,
|
||||
help="Backtest window end (YYYY-MM-DD)")
|
||||
parser.add_argument("--min-grade", default="B",
|
||||
choices=["A+", "A", "B", "C", "D", "F"],
|
||||
help="Minimum grade to open backtest trade (default B)")
|
||||
parser.add_argument("--journal", action="store_true",
|
||||
help="Print last 10 trades from backtest journal")
|
||||
parser.add_argument("--journal-csv", default=None,
|
||||
help="Write full trade journal to CSV path")
|
||||
parser.add_argument("--regime-filter", action="store_true",
|
||||
help="Gate signals on higher-TF ADX trend confirmation")
|
||||
parser.add_argument("--regime-min-adx", type=float, default=30.0,
|
||||
help="Min ADX for regime filter (default 30, requires --regime-filter)")
|
||||
parser.add_argument("--session-filter", action="store_true",
|
||||
help="Only trade during a UTC hour window (default 07-16, London + NY)")
|
||||
parser.add_argument("--session-start", type=int, default=7,
|
||||
help="Session start hour UTC, inclusive (default 7)")
|
||||
parser.add_argument("--session-end", type=int, default=16,
|
||||
help="Session end hour UTC, exclusive (default 16)")
|
||||
parser.add_argument("--auto-session-filter", action="store_true",
|
||||
help="Auto-toggle session filter when recent trades show adverse mean pnl%%")
|
||||
parser.add_argument("--auto-session-lookback", type=int, default=10,
|
||||
help="Lookback in closed trades for auto-session detection (default 10)")
|
||||
parser.add_argument("--auto-session-threshold-pct", type=float, default=-0.3,
|
||||
help="Mean pnl%% threshold below which auto-session activates (default -0.3)")
|
||||
parser.add_argument("--ablate", action="store_true",
|
||||
help="Run leave-one-out indicator ablation sweep (parallel)")
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -372,21 +280,6 @@ def build_config(args) -> PlatformConfig:
|
||||
args.timeframe, [Timeframe.M15, Timeframe.H4]
|
||||
)
|
||||
|
||||
# Backtest config
|
||||
config.backtest.start_date = args.start
|
||||
config.backtest.end_date = args.end
|
||||
config.backtest.max_bars = args.bars
|
||||
config.backtest.min_trade_grade = args.min_grade
|
||||
config.backtest.optimize = args.optimize
|
||||
config.backtest.regime_filter = args.regime_filter
|
||||
config.backtest.regime_filter_min_adx = args.regime_min_adx
|
||||
config.backtest.session_filter = args.session_filter
|
||||
config.backtest.session_start_hour = args.session_start
|
||||
config.backtest.session_end_hour = args.session_end
|
||||
config.backtest.auto_session_filter = args.auto_session_filter
|
||||
config.backtest.auto_session_lookback_trades = args.auto_session_lookback
|
||||
config.backtest.auto_session_threshold_pct = args.auto_session_threshold_pct
|
||||
|
||||
return config
|
||||
|
||||
|
||||
@@ -406,148 +299,6 @@ def run_live(args, config: PlatformConfig):
|
||||
print(f"{Colors.CYAN}{'═' * 80}{Colors.RESET}\n")
|
||||
|
||||
|
||||
def run_backtest(args, config: PlatformConfig):
|
||||
from backtest import WalkForwardBacktester
|
||||
from metrics import trades_to_dataframe
|
||||
|
||||
print_header("USD/JPY WALK-FORWARD BACKTEST")
|
||||
print(f"{Colors.GRAY} Loading data and replaying bars...{Colors.RESET}\n")
|
||||
|
||||
backtester = WalkForwardBacktester(config)
|
||||
result = backtester.run()
|
||||
|
||||
if args.json:
|
||||
out = {
|
||||
"metrics": result.metrics,
|
||||
"initial_balance": result.initial_balance,
|
||||
"final_balance": result.final_balance,
|
||||
"trades": [
|
||||
{
|
||||
"entry_time": str(t.entry_time),
|
||||
"exit_time": str(t.exit_time),
|
||||
"direction": t.direction,
|
||||
"entry_price": t.entry_price,
|
||||
"exit_price": t.exit_price,
|
||||
"exit_reason": t.exit_reason,
|
||||
"pnl_pips": t.pnl_pips,
|
||||
"pnl_pct": t.pnl_pct,
|
||||
"grade": t.grade,
|
||||
}
|
||||
for t in result.trades
|
||||
],
|
||||
}
|
||||
print(json.dumps(out, indent=2))
|
||||
else:
|
||||
print_backtest_report(result, verbose_journal=args.journal)
|
||||
|
||||
if args.journal_csv and result.trades:
|
||||
df = trades_to_dataframe(result.trades)
|
||||
df.to_csv(args.journal_csv, index=False)
|
||||
print(f"\n {Colors.GREEN}✓ Trade journal written to {args.journal_csv}{Colors.RESET}")
|
||||
|
||||
|
||||
def run_optimize(args, config: PlatformConfig):
|
||||
from backtest import Optimizer
|
||||
|
||||
print_header("USD/JPY BACKTEST PARAMETER OPTIMIZER")
|
||||
grid = config.backtest.param_grid
|
||||
n_combos = 1
|
||||
for v in grid.values():
|
||||
n_combos *= len(v)
|
||||
print(f"{Colors.GRAY} Grid: {n_combos} combos × {config.backtest.parallel_workers} workers{Colors.RESET}\n")
|
||||
|
||||
optimizer = Optimizer(config)
|
||||
results = optimizer.run_grid(grid)
|
||||
|
||||
print(f"\n {Colors.BOLD}Top 10 Parameter Combinations (by total return):{Colors.RESET}\n")
|
||||
print(f" {'#':<3} {'Return%':>8} {'PF':>6} {'WinR':>6} {'Trades':>7} "
|
||||
f"{'MaxDD%':>7} {'Sharpe':>7} Params")
|
||||
print(f" {Colors.GRAY}{'─' * 90}{Colors.RESET}")
|
||||
|
||||
for i, r in enumerate(results[:10], start=1):
|
||||
m = r.metrics
|
||||
ret = m.get("total_return_pct", 0)
|
||||
ret_color = Colors.GREEN if ret >= 0 else Colors.RED
|
||||
params_str = ", ".join(f"{k}={v}" for k, v in r.params.items())
|
||||
pf = m.get("profit_factor", 0)
|
||||
pf_str = "inf" if pf == float("inf") else f"{pf:.2f}"
|
||||
print(f" {i:<3} {ret_color}{ret:>+7.2f}%{Colors.RESET} {pf_str:>6} "
|
||||
f"{m.get('win_rate', 0):>5.1f}% {m.get('total_trades', 0):>7} "
|
||||
f"{m.get('max_drawdown_pct', 0):>+6.2f}% {m.get('sharpe', 0):>7.2f}"
|
||||
f" {Colors.GRAY}{params_str}{Colors.RESET}")
|
||||
|
||||
print(f"\n {Colors.BOLD}Best combination details:{Colors.RESET}\n")
|
||||
print_backtest_report(results[0], params=results[0].params, verbose_journal=False)
|
||||
|
||||
|
||||
def run_ablation(args, config: PlatformConfig):
|
||||
"""Leave-one-out ablation: baseline + each indicator disabled, in parallel."""
|
||||
from backtest import Optimizer
|
||||
from config import ALL_INDICATORS
|
||||
|
||||
print_header("USD/JPY INDICATOR ABLATION SWEEP")
|
||||
|
||||
grid = {"disable_indicator": [None] + list(ALL_INDICATORS)}
|
||||
n_combos = len(grid["disable_indicator"])
|
||||
print(f"{Colors.GRAY} Grid: {n_combos} runs (1 baseline + {n_combos-1} ablations) "
|
||||
f"× {config.backtest.parallel_workers} workers{Colors.RESET}\n")
|
||||
|
||||
optimizer = Optimizer(config)
|
||||
results = optimizer.run_grid(grid)
|
||||
|
||||
# Find baseline (disable_indicator=None) for delta computation
|
||||
baseline = next(
|
||||
(r for r in results if not r.params.get("disable_indicator")),
|
||||
None,
|
||||
)
|
||||
base_ret = baseline.metrics.get("total_return_pct", 0.0) if baseline else 0.0
|
||||
base_pf = baseline.metrics.get("profit_factor", 0.0) if baseline else 0.0
|
||||
base_sharpe = baseline.metrics.get("sharpe", 0.0) if baseline else 0.0
|
||||
|
||||
# Sort: baseline first, then ablations by impact (largest drop = most useful indicator)
|
||||
ablations = [r for r in results if r is not baseline]
|
||||
ablations.sort(
|
||||
key=lambda r: r.metrics.get("total_return_pct", 0.0) - base_ret
|
||||
)
|
||||
|
||||
print(f" {Colors.BOLD}Indicator Ablation Report{Colors.RESET}")
|
||||
print(f" {Colors.GRAY}{'─' * 88}{Colors.RESET}")
|
||||
print(f" {'Disabled':<14} {'Return%':>9} {'ΔRet%':>8} {'PF':>6} {'ΔPF':>7} "
|
||||
f"{'Sharpe':>7} {'ΔSh':>7} {'Trades':>7}")
|
||||
print(f" {Colors.GRAY}{'─' * 88}{Colors.RESET}")
|
||||
|
||||
def fmt_row(label, m, deltas=True):
|
||||
ret = m.get("total_return_pct", 0.0)
|
||||
pf = m.get("profit_factor", 0.0)
|
||||
sh = m.get("sharpe", 0.0)
|
||||
tr = m.get("total_trades", 0)
|
||||
ret_color = Colors.GREEN if ret >= 0 else Colors.RED
|
||||
pf_str = "inf" if pf == float("inf") else f"{pf:.2f}"
|
||||
if deltas:
|
||||
d_ret = ret - base_ret
|
||||
d_pf = pf - base_pf if pf != float("inf") else 0.0
|
||||
d_sh = sh - base_sharpe
|
||||
d_ret_color = Colors.RED if d_ret < 0 else Colors.GREEN
|
||||
print(f" {label:<14} {ret_color}{ret:>+8.2f}%{Colors.RESET} "
|
||||
f"{d_ret_color}{d_ret:>+7.2f}%{Colors.RESET} "
|
||||
f"{pf_str:>6} {d_pf:>+7.2f} "
|
||||
f"{sh:>7.2f} {d_sh:>+7.2f} {tr:>7}")
|
||||
else:
|
||||
print(f" {Colors.BOLD}{label:<14}{Colors.RESET} "
|
||||
f"{ret_color}{ret:>+8.2f}%{Colors.RESET} {'':>8} "
|
||||
f"{pf_str:>6} {'':>7} {sh:>7.2f} {'':>7} {tr:>7}")
|
||||
|
||||
if baseline:
|
||||
fmt_row("baseline", baseline.metrics, deltas=False)
|
||||
print(f" {Colors.GRAY}{'─' * 88}{Colors.RESET}")
|
||||
for r in ablations:
|
||||
fmt_row(str(r.params.get("disable_indicator")), r.metrics)
|
||||
|
||||
print(f"\n {Colors.GRAY}Indicators ranked by importance — most negative ΔRet means "
|
||||
f"removing it hurt most.{Colors.RESET}")
|
||||
print(f" {Colors.GRAY}Positive ΔRet means the indicator is *hurting* the system.{Colors.RESET}\n")
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
|
||||
@@ -559,15 +310,7 @@ def main():
|
||||
)
|
||||
|
||||
config = build_config(args)
|
||||
|
||||
if args.ablate:
|
||||
run_ablation(args, config)
|
||||
elif args.optimize:
|
||||
run_optimize(args, config)
|
||||
elif args.backtest:
|
||||
run_backtest(args, config)
|
||||
else:
|
||||
run_live(args, config)
|
||||
run_live(args, config)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user