ForexBot2/backtest/research/fetch_oanda_m15.py

"""Fetch XAU/USD M15 candles from the Oanda v20 REST API.

Mechanical data-fetching infrastructure only — no analysis here.
Paginates GET /v3/instruments/XAU_USD/candles from a start cursor
through "now" and writes the result to a parquet file.

Run from the repo root:
    venv/bin/python backtest/research/fetch_oanda_m15.py
"""
import sys
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path

import pandas as pd
import requests

sys.path.insert(0, str(Path(__file__).resolve().parents[2]))

from app_config import oanda_api_token, oanda_base_url  # noqa: E402

INSTRUMENT = "XAU_USD"
GRANULARITY = "M15"
START = datetime(2025, 7, 1, 0, 0, 0, tzinfo=timezone.utc)
OUT_PATH = Path(__file__).parent / "oanda_xau_m15_20250701_now.parquet"
COUNT = 5000
REQUEST_SLEEP_S = 0.5
FRESHNESS_WINDOW = timedelta(minutes=30)


def _rfc3339(dt: datetime) -> str:
    return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")


def fetch_all_candles():
    token = oanda_api_token()
    base_url = oanda_base_url()
    if not token:
        raise RuntimeError(
            "No Oanda API token resolved (env OANDA_API_TOKEN / "
            "config.json oanda.api_token). Aborting.")

    url = f"{base_url}/v3/instruments/{INSTRUMENT}/candles"
    headers = {
        "Authorization": f"Bearer {token}",
        "Accept-Datetime-Format": "RFC3339",
    }

    cursor = START
    include_first = True
    all_rows = []
    n_requests = 0

    while True:
        params = {
            "granularity": GRANULARITY,
            "price": "MBA",
            "count": COUNT,
            "dailyAlignment": 0,
            "alignmentTimezone": "Etc/UTC",
            "from": _rfc3339(cursor),
            "includeFirst": "true" if include_first else "false",
        }

        resp = requests.get(url, headers=headers, params=params, timeout=30)
        n_requests += 1
        if resp.status_code != 200:
            raise RuntimeError(
                f"Oanda API request failed: status={resp.status_code}\n"
                f"URL: {resp.url}\n"
                f"Response body: {resp.text}")

        payload = resp.json()
        candles = payload.get("candles", [])

        for c in candles:
            if not c.get("complete", False):
                continue
            mid = c["mid"]
            bid = c["bid"]
            ask = c["ask"]
            all_rows.append({
                "Datetime": pd.Timestamp(c["time"]).tz_convert("UTC")
                if pd.Timestamp(c["time"]).tzinfo is not None
                else pd.Timestamp(c["time"]).tz_localize("UTC"),
                "Open": float(mid["o"]),
                "High": float(mid["h"]),
                "Low": float(mid["l"]),
                "Close": float(mid["c"]),
                "Volume": int(c["volume"]),
                "bid_o": float(bid["o"]),
                "bid_h": float(bid["h"]),
                "bid_l": float(bid["l"]),
                "bid_c": float(bid["c"]),
                "ask_o": float(ask["o"]),
                "ask_h": float(ask["h"]),
                "ask_l": float(ask["l"]),
                "ask_c": float(ask["c"]),
            })

        print(f"  request {n_requests}: from={params['from']} "
              f"got {len(candles)} candles (complete kept so far: "
              f"{len(all_rows)})", file=sys.stderr)

        if not candles:
            break

        last_time = pd.Timestamp(candles[-1]["time"])
        if last_time.tzinfo is None:
            last_time = last_time.tz_localize("UTC")
        else:
            last_time = last_time.tz_convert("UTC")

        now = datetime.now(timezone.utc)
        # Oanda's "count" includes the "from" boundary bar itself, so with
        # includeFirst=false every full page comes back as count-1, not
        # count (verified empirically: count=N, includeFirst=false always
        # yields N-1 candles when a full page of data exists ahead of the
        # cursor). Compare against the count actually expected for this
        # request's includeFirst setting, not the raw requested count.
        expected_full_page = COUNT if include_first else COUNT - 1
        got_fewer_than_requested = len(candles) < expected_full_page
        near_now = (now - last_time.to_pydatetime()) < FRESHNESS_WINDOW

        if got_fewer_than_requested or near_now:
            break

        cursor = last_time.to_pydatetime()
        include_first = False
        time.sleep(REQUEST_SLEEP_S)

    return all_rows, n_requests


def main():
    print(f"Fetching {INSTRUMENT} {GRANULARITY} candles from "
          f"{_rfc3339(START)} through now...", file=sys.stderr)

    rows, n_requests = fetch_all_candles()

    if not rows:
        raise RuntimeError("No complete candles were fetched.")

    df = pd.DataFrame(rows)
    df = df.set_index("Datetime")
    df = df.sort_index()
    df = df[~df.index.duplicated(keep="last")]

    df.to_parquet(OUT_PATH)

    print(f"\nSaved {len(df)} rows to {OUT_PATH}", file=sys.stderr)
    print(f"Requests made: {n_requests}", file=sys.stderr)
    print(f"First timestamp: {df.index[0]}", file=sys.stderr)
    print(f"Last timestamp: {df.index[-1]}", file=sys.stderr)


if __name__ == "__main__":
    main()