#!/usr/bin/env python3 """ One-shot historical OHLCV seeder for backtesting. Downloads bars from Twelve Data for the configured pairs/timeframes, paginated to stay within the 5000-bar per-call limit, and writes each (symbol, timeframe) to history/_.csv. Usage: python seed_history.py --years 1 python seed_history.py --years 2 --pairs EURUSD GBPUSD USDJPY python seed_history.py --tfs M15 H1 D1 --years 1 --force Cost (free tier, 800 credits/day): 1 year M15 + H1 + D1 × 7 majors ≈ 56 credits 5 years same ≈ 231 credits """ import argparse import logging import sys from datetime import datetime, timedelta, timezone from pathlib import Path try: from dotenv import load_dotenv load_dotenv() except ImportError: pass from config import MAJOR_PAIRS, PlatformConfig, Timeframe from data_provider import DataUnavailableError, ForexDataProvider DEFAULT_TFS = ["M15", "H1", "D1"] HISTORY_DIR = Path("history") _TF_MAP = { "M1": Timeframe.M1, "M5": Timeframe.M5, "M15": Timeframe.M15, "M30": Timeframe.M30, "H1": Timeframe.H1, "H4": Timeframe.H4, "D1": Timeframe.D1, "W1": Timeframe.W1, } def csv_path(out_dir: Path, symbol: str, tf: Timeframe) -> Path: return out_dir / f"{symbol.replace('=X', '')}_{tf.value}.csv" def main() -> int: parser = argparse.ArgumentParser( description="Seed historical OHLCV for backtesting (Twelve Data, free tier).", ) parser.add_argument("--years", type=float, default=1.0, help="Years of history to download (default 1.0)") parser.add_argument("--pairs", nargs="+", default=None, help="Pair symbols, e.g. EURUSD GBPUSD (default all 7 majors)") parser.add_argument("--tfs", nargs="+", default=DEFAULT_TFS, help=f"Timeframes (default {' '.join(DEFAULT_TFS)})") parser.add_argument("--out", default=str(HISTORY_DIR), help="Output directory (default history/)") parser.add_argument("--force", action="store_true", help="Re-download even if a CSV already exists") args = parser.parse_args() logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", datefmt="%H:%M:%S", ) out_dir = Path(args.out) out_dir.mkdir(parents=True, exist_ok=True) pairs = args.pairs if args.pairs else MAJOR_PAIRS pairs = [p if p.endswith("=X") else f"{p}=X" for p in pairs] try: tfs = [_TF_MAP[t.upper()] for t in args.tfs] except KeyError as e: print(f"❌ Unknown timeframe: {e}. Valid: {list(_TF_MAP.keys())}") return 2 end = datetime.now(timezone.utc) start = end - timedelta(days=int(args.years * 365)) provider = ForexDataProvider(PlatformConfig()) total = len(pairs) * len(tfs) print(f"📥 Seeding {total} pair × tf combinations from " f"{start.date()} to {end.date()} ({args.years} year(s))") print(f" Pairs: {[p.replace('=X', '') for p in pairs]}") print(f" TFs: {[tf.value for tf in tfs]}") print(f" Output: {out_dir}/\n") done = 0 failed: list = [] for symbol in pairs: for tf in tfs: done += 1 out_path = csv_path(out_dir, symbol, tf) if out_path.exists() and not args.force: print(f"[{done}/{total}] ⏭ {symbol} {tf.value} — exists, " f"skipping (--force to overwrite)") continue print(f"[{done}/{total}] 📥 {symbol} {tf.value}…", flush=True) try: df = provider.fetch_historical(symbol, tf, start, end) df.to_csv(out_path) print(f" ✅ {len(df):,} bars " f"({df.index.min().date()} → {df.index.max().date()}) " f"→ {out_path}") except DataUnavailableError as e: print(f" ❌ {e}") failed.append((symbol, tf.value, str(e))) except Exception as e: print(f" ❌ unexpected: {e}") failed.append((symbol, tf.value, str(e))) print() if failed: print(f"⚠️ {len(failed)} fetch(es) failed:") for sym, tfn, msg in failed: print(f" {sym} {tfn}: {msg}") return 1 print("✅ Seeding complete. Run: python backtest.py --primary M15") return 0 if __name__ == "__main__": sys.exit(main())