ForexBot2/seed_history.py

#!/usr/bin/env python3
"""
One-shot historical OHLCV seeder for backtesting.

Downloads bars from Twelve Data for the configured pairs/timeframes,
paginated to stay within the 5000-bar per-call limit, and writes each
(symbol, timeframe) to history/<PAIR>_<TF>.csv.

Usage:
    python seed_history.py --years 1
    python seed_history.py --years 2 --pairs EURUSD GBPUSD USDJPY
    python seed_history.py --tfs M15 H1 D1 --years 1 --force

Cost (free tier, 800 credits/day):
    1 year M15 + H1 + D1 × 7 majors  ≈ 56 credits
    5 years same                     ≈ 231 credits
"""

import argparse
import logging
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path

try:
    from dotenv import load_dotenv
    load_dotenv()
except ImportError:
    pass

from config import MAJOR_PAIRS, PlatformConfig, Timeframe
from data_provider import DataUnavailableError, ForexDataProvider

DEFAULT_TFS = ["M15", "H1", "D1"]
HISTORY_DIR = Path("history")

_TF_MAP = {
    "M1":  Timeframe.M1,  "M5":  Timeframe.M5,
    "M15": Timeframe.M15, "M30": Timeframe.M30,
    "H1":  Timeframe.H1,  "H4":  Timeframe.H4,
    "D1":  Timeframe.D1,  "W1":  Timeframe.W1,
}


def csv_path(out_dir: Path, symbol: str, tf: Timeframe) -> Path:
    return out_dir / f"{symbol.replace('=X', '')}_{tf.value}.csv"


def main() -> int:
    parser = argparse.ArgumentParser(
        description="Seed historical OHLCV for backtesting (Twelve Data, free tier).",
    )
    parser.add_argument("--years", type=float, default=1.0,
                        help="Years of history to download (default 1.0)")
    parser.add_argument("--pairs", nargs="+", default=None,
                        help="Pair symbols, e.g. EURUSD GBPUSD (default all 7 majors)")
    parser.add_argument("--tfs", nargs="+", default=DEFAULT_TFS,
                        help=f"Timeframes (default {' '.join(DEFAULT_TFS)})")
    parser.add_argument("--out", default=str(HISTORY_DIR),
                        help="Output directory (default history/)")
    parser.add_argument("--force", action="store_true",
                        help="Re-download even if a CSV already exists")
    args = parser.parse_args()

    logging.basicConfig(
        level=logging.INFO,
        format="%(asctime)s [%(levelname)s] %(message)s",
        datefmt="%H:%M:%S",
    )

    out_dir = Path(args.out)
    out_dir.mkdir(parents=True, exist_ok=True)

    pairs = args.pairs if args.pairs else MAJOR_PAIRS
    pairs = [p if p.endswith("=X") else f"{p}=X" for p in pairs]

    try:
        tfs = [_TF_MAP[t.upper()] for t in args.tfs]
    except KeyError as e:
        print(f"❌ Unknown timeframe: {e}. Valid: {list(_TF_MAP.keys())}")
        return 2

    end = datetime.now(timezone.utc)
    start = end - timedelta(days=int(args.years * 365))

    provider = ForexDataProvider(PlatformConfig())

    total = len(pairs) * len(tfs)
    print(f"📥 Seeding {total} pair × tf combinations from "
          f"{start.date()} to {end.date()}  ({args.years} year(s))")
    print(f"   Pairs: {[p.replace('=X', '') for p in pairs]}")
    print(f"   TFs:   {[tf.value for tf in tfs]}")
    print(f"   Output: {out_dir}/\n")

    done = 0
    failed: list = []
    for symbol in pairs:
        for tf in tfs:
            done += 1
            out_path = csv_path(out_dir, symbol, tf)

            if out_path.exists() and not args.force:
                print(f"[{done}/{total}] ⏭  {symbol} {tf.value} — exists, "
                      f"skipping (--force to overwrite)")
                continue

            print(f"[{done}/{total}] 📥 {symbol} {tf.value}…", flush=True)
            try:
                df = provider.fetch_historical(symbol, tf, start, end)
                df.to_csv(out_path)
                print(f"   ✅ {len(df):,} bars  "
                      f"({df.index.min().date()} → {df.index.max().date()})  "
                      f"→ {out_path}")
            except DataUnavailableError as e:
                print(f"   ❌ {e}")
                failed.append((symbol, tf.value, str(e)))
            except Exception as e:
                print(f"   ❌ unexpected: {e}")
                failed.append((symbol, tf.value, str(e)))

    print()
    if failed:
        print(f"⚠️  {len(failed)} fetch(es) failed:")
        for sym, tfn, msg in failed:
            print(f"     {sym} {tfn}: {msg}")
        return 1

    print("✅ Seeding complete. Run: python backtest.py --primary M15")
    return 0


if __name__ == "__main__":
    sys.exit(main())