Add walk-forward backtest harness + paginated history seeder

- data_provider: new fetch_historical() that paginates /time_series in 5000-bar chunks (1 credit/chunk) for arbitrary date ranges. Tolerates the API's "no data on specified dates" boundary error gracefully when prior chunks succeeded — that error is just the marker that we walked past the symbol's earliest available bar. - seed_history.py: one-shot CLI that downloads OHLCV for the configured pairs/TFs into history/<PAIR>_<TF>.csv. Skips existing files unless --force. - backtest.py: walk-forward replay through SignalAggregator with strict no-look-ahead semantics. Each bar is only visible to the engine after it has fully closed (`df.index + interval <= as_of`). Trade simulation is first-touch (SL vs TP1) with worst-case (SL first if both touched same bar), spread cost deducted from R, max-hold timeout, no-overlap. Reports per-grade win-rate / avg-R / profit-factor / max-DD, plus optional matplotlib equity + by-grade plots. - requirements: add matplotlib (lazy-imported, only needed for --plot). - .gitignore: history/ and backtest_results/ regenerable, kept untracked. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-06 13:29:48 -04:00
parent 402d49e094
commit 6d235061b9
5 changed files with 971 additions and 0 deletions
@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""
+One-shot historical OHLCV seeder for backtesting.
+
+Downloads bars from Twelve Data for the configured pairs/timeframes,
+paginated to stay within the 5000-bar per-call limit, and writes each
+(symbol, timeframe) to history/<PAIR>_<TF>.csv.
+
+Usage:
+    python seed_history.py --years 1
+    python seed_history.py --years 2 --pairs EURUSD GBPUSD USDJPY
+    python seed_history.py --tfs M15 H1 D1 --years 1 --force
+
+Cost (free tier, 800 credits/day):
+    1 year M15 + H1 + D1 × 7 majors  ≈ 56 credits
+    5 years same                     ≈ 231 credits
+"""
+
+import argparse
+import logging
+import sys
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+try:
+    from dotenv import load_dotenv
+    load_dotenv()
+except ImportError:
+    pass
+
+from config import MAJOR_PAIRS, PlatformConfig, Timeframe
+from data_provider import DataUnavailableError, ForexDataProvider
+
+DEFAULT_TFS = ["M15", "H1", "D1"]
+HISTORY_DIR = Path("history")
+
+_TF_MAP = {
+    "M1":  Timeframe.M1,  "M5":  Timeframe.M5,
+    "M15": Timeframe.M15, "M30": Timeframe.M30,
+    "H1":  Timeframe.H1,  "H4":  Timeframe.H4,
+    "D1":  Timeframe.D1,  "W1":  Timeframe.W1,
+}
+
+
+def csv_path(out_dir: Path, symbol: str, tf: Timeframe) -> Path:
+    return out_dir / f"{symbol.replace('=X', '')}_{tf.value}.csv"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Seed historical OHLCV for backtesting (Twelve Data, free tier).",
+    )
+    parser.add_argument("--years", type=float, default=1.0,
+                        help="Years of history to download (default 1.0)")
+    parser.add_argument("--pairs", nargs="+", default=None,
+                        help="Pair symbols, e.g. EURUSD GBPUSD (default all 7 majors)")
+    parser.add_argument("--tfs", nargs="+", default=DEFAULT_TFS,
+                        help=f"Timeframes (default {' '.join(DEFAULT_TFS)})")
+    parser.add_argument("--out", default=str(HISTORY_DIR),
+                        help="Output directory (default history/)")
+    parser.add_argument("--force", action="store_true",
+                        help="Re-download even if a CSV already exists")
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s [%(levelname)s] %(message)s",
+        datefmt="%H:%M:%S",
+    )
+
+    out_dir = Path(args.out)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    pairs = args.pairs if args.pairs else MAJOR_PAIRS
+    pairs = [p if p.endswith("=X") else f"{p}=X" for p in pairs]
+
+    try:
+        tfs = [_TF_MAP[t.upper()] for t in args.tfs]
+    except KeyError as e:
+        print(f"❌ Unknown timeframe: {e}. Valid: {list(_TF_MAP.keys())}")
+        return 2
+
+    end = datetime.now(timezone.utc)
+    start = end - timedelta(days=int(args.years * 365))
+
+    provider = ForexDataProvider(PlatformConfig())
+
+    total = len(pairs) * len(tfs)
+    print(f"📥 Seeding {total} pair × tf combinations from "
+          f"{start.date()} to {end.date()}  ({args.years} year(s))")
+    print(f"   Pairs: {[p.replace('=X', '') for p in pairs]}")
+    print(f"   TFs:   {[tf.value for tf in tfs]}")
+    print(f"   Output: {out_dir}/\n")
+
+    done = 0
+    failed: list = []
+    for symbol in pairs:
+        for tf in tfs:
+            done += 1
+            out_path = csv_path(out_dir, symbol, tf)
+
+            if out_path.exists() and not args.force:
+                print(f"[{done}/{total}] ⏭  {symbol} {tf.value} — exists, "
+                      f"skipping (--force to overwrite)")
+                continue
+
+            print(f"[{done}/{total}] 📥 {symbol} {tf.value}…", flush=True)
+            try:
+                df = provider.fetch_historical(symbol, tf, start, end)
+                df.to_csv(out_path)
+                print(f"   ✅ {len(df):,} bars  "
+                      f"({df.index.min().date()} → {df.index.max().date()})  "
+                      f"→ {out_path}")
+            except DataUnavailableError as e:
+                print(f"   ❌ {e}")
+                failed.append((symbol, tf.value, str(e)))
+            except Exception as e:
+                print(f"   ❌ unexpected: {e}")
+                failed.append((symbol, tf.value, str(e)))
+
+    print()
+    if failed:
+        print(f"⚠️  {len(failed)} fetch(es) failed:")
+        for sym, tfn, msg in failed:
+            print(f"     {sym} {tfn}: {msg}")
+        return 1
+
+    print("✅ Seeding complete. Run: python backtest.py --primary M15")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())