"""
Forex Signal Intelligence Platform - Data Provider
====================================================
Retrieves and validates market data from Twelve Data API.
Falls back to synthetic GBM data if the API is unavailable.
"""

import hashlib
import logging
import os
import time
from collections import deque
from datetime import datetime, timedelta, timezone
from typing import Dict, List, Optional

import numpy as np
import pandas as pd
import requests

from config import PlatformConfig, Timeframe, SYMBOL_DISPLAY

logger = logging.getLogger(__name__)


class DataIntegrityError(Exception):
    """Raised when data fails validation checks."""
    pass


class ForexDataProvider:
    """
    Forex data provider backed by Twelve Data API.

    Twelve Data has native 4h candles, eliminating the H1→H4 resampling
    hack needed for Yahoo Finance. Free tier: 800 credits/day, 8 req/min.
    """

    TD_BASE_URL = "https://api.twelvedata.com"

    TD_INTERVALS = {
        Timeframe.M1:  "1min",
        Timeframe.M5:  "5min",
        Timeframe.M15: "15min",
        Timeframe.M30: "30min",
        Timeframe.H1:  "1h",
        Timeframe.H4:  "4h",
        Timeframe.D1:  "1day",
        Timeframe.W1:  "1week",
    }

    # Approximate bars per day for each timeframe (used to size outputsize)
    BARS_PER_DAY = {
        Timeframe.M1:  1440,
        Timeframe.M5:  288,
        Timeframe.M15: 96,
        Timeframe.M30: 48,
        Timeframe.H1:  24,
        Timeframe.H4:  6,
        Timeframe.D1:  1,
        Timeframe.W1:  0.143,
    }

    # Free plan: 8 API credits per minute (each symbol = 1 credit)
    RATE_LIMIT = 8
    RATE_WINDOW = 60.0

    def __init__(self, config: PlatformConfig):
        self.config = config
        self._api_key = (
            os.environ.get("TWELVE_DATA_API_KEY")
            or config.twelve_data_api_key
        )
        self._cache: Dict[str, pd.DataFrame] = {}
        self._cache_timestamps: Dict[str, float] = {}
        self._request_times: deque = deque()  # timestamps of recent API credits used
        self._session = requests.Session()
        self._session.headers.update({
            "User-Agent": "Mozilla/5.0 (compatible; ForexIntel/1.0)"
        })

    @staticmethod
    def _to_td_symbol(symbol: str) -> str:
        """Convert internal symbol (EURUSD=X) to Twelve Data format (EUR/USD)."""
        clean = symbol.replace("=X", "")
        return f"{clean[:3]}/{clean[3:]}"

    def fetch_ohlcv(
        self,
        symbol: str,
        timeframe: Timeframe,
        lookback_days: Optional[int] = None,
    ) -> pd.DataFrame:
        """
        Fetch OHLCV data for a Forex pair with full validation.

        Returns a DataFrame with columns: Open, High, Low, Close, Volume
        Index is a timezone-aware DatetimeIndex in UTC.
        """
        if lookback_days is None:
            lookback_days = self.config.data_lookback_days

        cache_key = f"{symbol}_{timeframe.value}"
        now = time.time()

        if (
            cache_key in self._cache_timestamps
            and (now - self._cache_timestamps[cache_key]) < self.config.refresh_interval
            and cache_key in self._cache
        ):
            cached = self._cache.get(cache_key)
            if cached is not None and not cached.empty:
                return cached

        df = self._fetch_twelve_data(symbol, timeframe, lookback_days)
        df = self._validate_and_clean(df, symbol, timeframe)

        self._cache[cache_key] = df
        self._cache_timestamps[cache_key] = now
        return df

    def prefetch_batch(
        self,
        symbols: List[str],
        timeframes: List[Timeframe],
        lookback_days: Optional[int] = None,
    ) -> None:
        """
        Batch-fetch all symbols for each timeframe in a single API call per TF.

        Twelve Data accepts comma-separated symbols, so 7 pairs × 3 timeframes
        = 3 requests instead of 21 — well within the free-tier 8 req/min limit.
        Results are stored in the local cache; subsequent fetch_ohlcv calls
        for the same symbol/TF will be served from cache.
        """
        if lookback_days is None:
            lookback_days = self.config.data_lookback_days

        for tf in timeframes:
            try:
                self._batch_fetch_timeframe(symbols, tf, lookback_days)
                time.sleep(1.0)  # brief pause between TF requests
            except Exception as e:
                logger.warning(f"Batch prefetch failed for {tf.value}: {e}")

    def fetch_multiple(
        self,
        symbols: List[str],
        timeframe: Timeframe,
        lookback_days: Optional[int] = None,
    ) -> Dict[str, pd.DataFrame]:
        """Fetch data for multiple symbols, returning dict of DataFrames."""
        results = {}
        for symbol in symbols:
            try:
                df = self.fetch_ohlcv(symbol, timeframe, lookback_days)
                results[symbol] = df
            except Exception as e:
                logger.warning(f"Failed to fetch {symbol}: {e}")
        return results

    def get_latest_price(self, symbol: str) -> Optional[float]:
        """Return the most recent close price for a symbol."""
        try:
            df = self.fetch_ohlcv(symbol, Timeframe.M5, lookback_days=1)
            if df is not None and not df.empty:
                return float(df["Close"].iloc[-1])
        except Exception:
            pass
        return None

    def get_spread_estimate(self, symbol: str) -> float:
        """Estimate typical spread from recent M5 high-low ranges."""
        try:
            df = self.fetch_ohlcv(symbol, Timeframe.M5, lookback_days=1)
            if df is not None and not df.empty:
                return float((df["High"] - df["Low"]).median())
        except Exception:
            pass

        fallback_spreads = {
            "EURUSD=X": 0.00010, "GBPUSD=X": 0.00015,
            "USDJPY=X": 0.010,   "USDCHF=X": 0.00015,
            "AUDUSD=X": 0.00015, "USDCAD=X": 0.00018,
            "NZDUSD=X": 0.00020,
        }
        return fallback_spreads.get(symbol, 0.00020)

    def get_pair_info(self, symbol: str) -> Dict:
        """Return display name and pip value for a pair."""
        display = SYMBOL_DISPLAY.get(symbol, symbol)
        is_jpy = "JPY" in symbol
        return {
            "symbol": symbol,
            "display": display,
            "pip_value": 0.01 if is_jpy else 0.0001,
            "is_jpy_pair": is_jpy,
        }

    # ── Private Methods ────────────────────────────────────────────

    def _throttle(self, credits: int = 1) -> None:
        """
        Block until `credits` API credits are available.

        Twelve Data free tier: 8 credits/minute. Each symbol in any request
        (single or batch) consumes 1 credit. This method enforces the limit
        regardless of whether requests are batched or individual.
        """
        while True:
            now = time.time()
            # Evict credits older than the rate window
            while self._request_times and now - self._request_times[0] >= self.RATE_WINDOW:
                self._request_times.popleft()

            available = self.RATE_LIMIT - len(self._request_times)
            if available >= credits:
                for _ in range(credits):
                    self._request_times.append(time.time())
                return

            # Wait until the oldest credit expires
            wait = (self._request_times[0] + self.RATE_WINDOW) - now + 0.2
            logger.info(
                f"Rate limit: {len(self._request_times)}/{self.RATE_LIMIT} credits used "
                f"— waiting {wait:.1f}s for next window..."
            )
            time.sleep(wait)

    def _batch_fetch_timeframe(
        self, symbols: List[str], timeframe: Timeframe, lookback_days: int
    ) -> None:
        """
        Single API call fetching all symbols for one timeframe.
        Twelve Data returns a dict keyed by symbol when multiple are requested.
        Parsed DataFrames are validated and stored in the cache.
        """
        if not self._api_key:
            return

        td_symbols = [self._to_td_symbol(s) for s in symbols]
        interval = self.TD_INTERVALS[timeframe]
        outputsize = min(
            int(lookback_days * self.BARS_PER_DAY[timeframe]) + 20, 5000
        )
        outputsize = max(outputsize, 300)

        # Each symbol in a batch request costs 1 credit
        self._throttle(credits=len(symbols))

        resp = self._session.get(
            f"{self.TD_BASE_URL}/time_series",
            params={
                "symbol": ",".join(td_symbols),
                "interval": interval,
                "outputsize": outputsize,
                "apikey": self._api_key,
                "timezone": "UTC",
                "format": "json",
            },
            timeout=30,
        )
        resp.raise_for_status()
        data = resp.json()

        # If the whole batch request returned a top-level error, raise it
        if isinstance(data, dict) and data.get("status") == "error":
            raise DataIntegrityError(
                f"Batch API error ({timeframe.value}): {data.get('message', 'unknown')}"
            )

        # Batch response: {"EUR/USD": {...}, "GBP/USD": {...}, ...}
        now = time.time()
        for symbol, td_sym in zip(symbols, td_symbols):
            try:
                sym_data = data.get(td_sym, {})
                if sym_data.get("status") == "error":
                    logger.warning(
                        f"Batch fetch error for {td_sym}: {sym_data.get('message')}"
                    )
                    continue

                values = sym_data.get("values")
                if not values:
                    logger.warning(f"No values in batch response for {td_sym}")
                    continue

                records = [
                    {
                        "Datetime": pd.Timestamp(v["datetime"], tz="UTC"),
                        "Open":   float(v["open"]),
                        "High":   float(v["high"]),
                        "Low":    float(v["low"]),
                        "Close":  float(v["close"]),
                        "Volume": float(v.get("volume") or 0),
                    }
                    for v in reversed(values)
                ]
                df = pd.DataFrame(records).set_index("Datetime")
                df.index.name = "Datetime"
                df = self._validate_and_clean(df, symbol, timeframe)

                cache_key = f"{symbol}_{timeframe.value}"
                self._cache[cache_key] = df
                self._cache_timestamps[cache_key] = now
                logger.debug(f"Batch cached {len(df)} bars for {td_sym} @ {interval}")

            except Exception as e:
                logger.warning(f"Failed to parse batch data for {td_sym}: {e}")

    def _fetch_twelve_data(
        self, symbol: str, timeframe: Timeframe, lookback_days: int
    ) -> pd.DataFrame:
        """
        Fetch OHLCV from Twelve Data REST API.
        Falls back to synthetic GBM data if API is unavailable or key is missing.
        """
        if not self._api_key:
            logger.warning("No Twelve Data API key — using synthetic data. "
                           "Set TWELVE_DATA_API_KEY env var or config.twelve_data_api_key.")
            return self._generate_synthetic_data(symbol, timeframe, lookback_days)

        td_symbol = self._to_td_symbol(symbol)
        interval = self.TD_INTERVALS[timeframe]
        outputsize = min(
            int(lookback_days * self.BARS_PER_DAY[timeframe]) + 20,
            5000,
        )
        outputsize = max(outputsize, 300)

        self._throttle(credits=1)

        try:
            resp = self._session.get(
                f"{self.TD_BASE_URL}/time_series",
                params={
                    "symbol": td_symbol,
                    "interval": interval,
                    "outputsize": outputsize,
                    "apikey": self._api_key,
                    "timezone": "UTC",
                    "format": "json",
                },
                timeout=15,
            )
            resp.raise_for_status()
            data = resp.json()

            if data.get("status") == "error":
                raise DataIntegrityError(
                    f"Twelve Data API error for {td_symbol}: {data.get('message', 'unknown')}"
                )

            values = data.get("values")
            if not values:
                raise DataIntegrityError(f"Empty response from Twelve Data for {td_symbol}")

            # Values arrive newest-first; reverse to chronological order
            records = [
                {
                    "Datetime": pd.Timestamp(v["datetime"], tz="UTC"),
                    "Open":   float(v["open"]),
                    "High":   float(v["high"]),
                    "Low":    float(v["low"]),
                    "Close":  float(v["close"]),
                    "Volume": float(v.get("volume") or 0),
                }
                for v in reversed(values)
            ]

            df = pd.DataFrame(records).set_index("Datetime")
            df.index.name = "Datetime"
            logger.debug(f"Fetched {len(df)} bars for {td_symbol} @ {interval}")
            return df

        except DataIntegrityError:
            raise
        except Exception as e:
            logger.info(f"Twelve Data fetch failed for {symbol} ({e}), using synthetic data.")
            return self._generate_synthetic_data(symbol, timeframe, lookback_days)

    def _generate_synthetic_data(
        self, symbol: str, timeframe: Timeframe, lookback_days: int
    ) -> pd.DataFrame:
        """
        Generate realistic synthetic Forex OHLCV using GBM with mean-reversion.
        Used as a fallback when live data is unavailable.
        """
        pair_params = {
            "EURUSD=X": {"price": 1.1350, "vol": 0.07, "drift": -0.001},
            "GBPUSD=X": {"price": 1.3200, "vol": 0.08, "drift":  0.002},
            "USDJPY=X": {"price": 144.50, "vol": 0.09, "drift":  0.003},
            "USDCHF=X": {"price": 0.8850, "vol": 0.07, "drift": -0.001},
            "AUDUSD=X": {"price": 0.6550, "vol": 0.10, "drift": -0.002},
            "USDCAD=X": {"price": 1.3650, "vol": 0.07, "drift":  0.001},
            "NZDUSD=X": {"price": 0.6100, "vol": 0.10, "drift": -0.001},
            "EURGBP=X": {"price": 0.8590, "vol": 0.06, "drift":  0.000},
            "EURJPY=X": {"price": 163.90, "vol": 0.09, "drift":  0.002},
            "GBPJPY=X": {"price": 190.70, "vol": 0.11, "drift":  0.003},
            "AUDJPY=X": {"price": 94.60,  "vol": 0.11, "drift":  0.001},
            "EURCHF=X": {"price": 0.9380, "vol": 0.05, "drift":  0.000},
            "CADJPY=X": {"price": 105.80, "vol": 0.09, "drift":  0.001},
        }

        params = pair_params.get(symbol, {"price": 1.0000, "vol": 0.08, "drift": 0.000})
        base_price = params["price"]
        annual_vol = params["vol"]
        annual_drift = params["drift"]

        tf_minutes = {
            Timeframe.M1: 1, Timeframe.M5: 5, Timeframe.M15: 15,
            Timeframe.M30: 30, Timeframe.H1: 60, Timeframe.H4: 240,
            Timeframe.D1: 1440, Timeframe.W1: 10080,
        }
        bar_minutes = tf_minutes.get(timeframe, 60)
        bars_per_year = (365.25 * 24 * 60) / bar_minutes
        dt = 1.0 / bars_per_year

        n_bars = min(int(lookback_days * 24 * 60 / bar_minutes), 2000)
        n_bars = max(n_bars, 300)

        seed_base = int(hashlib.md5(symbol.encode()).hexdigest()[:8], 16)
        time_component = int(time.time()) // 3600
        np.random.seed((seed_base + time_component) % (2**31))

        bar_vol = annual_vol * np.sqrt(dt)
        bar_drift = annual_drift * dt

        regime_len = np.random.randint(20, 80)
        trend_strength = np.random.normal(0, 0.3)

        price = base_price
        prices = [price]

        for i in range(1, n_bars):
            if i % regime_len == 0:
                regime_len = np.random.randint(20, 80)
                trend_strength = np.random.normal(0, 0.3)

            mr_force = -0.001 * (price - base_price) / base_price
            ret = bar_drift + trend_strength * dt + mr_force + bar_vol * np.random.randn()

            if np.random.random() < 0.02:
                ret *= np.random.uniform(2.0, 4.0)

            price *= (1 + ret)
            prices.append(price)

        prices = np.array(prices)
        opens = np.roll(prices, 1)
        opens[0] = prices[0]

        typical_range = bar_vol * base_price * 1.5
        highs = np.maximum(prices, opens) + np.abs(np.random.normal(0, typical_range * 0.6, n_bars))
        lows  = np.minimum(prices, opens) - np.abs(np.random.normal(0, typical_range * 0.6, n_bars))
        highs = np.maximum(highs, np.maximum(opens, prices))
        lows  = np.minimum(lows,  np.minimum(opens, prices))

        volume = np.random.lognormal(mean=10, sigma=0.5, size=n_bars).astype(int)

        freq_map = {
            Timeframe.M1: "1min",  Timeframe.M5: "5min",
            Timeframe.M15: "15min", Timeframe.M30: "30min",
            Timeframe.H1: "1h",    Timeframe.H4: "4h",
            Timeframe.D1: "D",     Timeframe.W1: "W",
        }
        freq = freq_map.get(timeframe, "1h")
        end_time = pd.Timestamp.now(tz="UTC").floor(f"{bar_minutes}min")
        index = pd.date_range(end=end_time, periods=n_bars, freq=freq, tz="UTC")

        df = pd.DataFrame({
            "Open": opens, "High": highs, "Low": lows,
            "Close": prices, "Volume": volume,
        }, index=index)
        df.index.name = "Datetime"

        logger.info(
            f"Generated {n_bars} synthetic {timeframe.value} bars for "
            f"{SYMBOL_DISPLAY.get(symbol, symbol)} @ {prices[-1]:.5f}"
        )
        return df

    def _validate_and_clean(
        self, df: pd.DataFrame, symbol: str, timeframe: Timeframe
    ) -> pd.DataFrame:
        """
        Validate and clean OHLCV data.

        Steps: drop all-NaN rows → ffill small gaps → remove future timestamps
        → fix OHLC inconsistencies → remove >10-sigma outliers → sort & deduplicate.
        """
        if df is None or df.empty:
            raise DataIntegrityError(f"Empty dataset for {symbol}")

        original_len = len(df)

        df = df.dropna(subset=["Open", "High", "Low", "Close"], how="all")
        df = df.ffill(limit=3)
        df = df.dropna(subset=["Close"])

        now = pd.Timestamp.utcnow()
        future_mask = df.index > now + pd.Timedelta(minutes=5)
        if future_mask.any():
            logger.warning(f"{symbol}: Dropping {future_mask.sum()} future timestamps")
            df = df[~future_mask]

        invalid_ohlc = (
            (df["High"] < df["Low"]) |
            (df["High"] < df["Open"]) |
            (df["High"] < df["Close"]) |
            (df["Low"] > df["Open"]) |
            (df["Low"] > df["Close"])
        )
        if invalid_ohlc.any():
            logger.warning(f"{symbol}: Fixing {invalid_ohlc.sum()} OHLC-inconsistent bars")
            for idx in df[invalid_ohlc].index:
                vals = [df.loc[idx, "Open"], df.loc[idx, "Close"]]
                df.loc[idx, "High"] = max(vals + [df.loc[idx, "High"]])
                df.loc[idx, "Low"]  = min(vals + [df.loc[idx, "Low"]])

        returns = df["Close"].pct_change()
        std_ret = returns.std()
        if std_ret > 0:
            outlier_mask = (returns - returns.mean()).abs() > 10 * std_ret
            if outlier_mask.any():
                logger.warning(f"{symbol}: Removing {outlier_mask.sum()} extreme outlier bars")
                df = df[~outlier_mask]

        min_required = max(
            self.config.technical.ema_trend,
            self.config.technical.ichimoku_senkou_b,
            200,
        ) + 10

        if len(df) < min_required:
            logger.warning(
                f"{symbol} {timeframe.value}: Only {len(df)} bars "
                f"(need {min_required}). Some indicators will be partial."
            )

        df = df.sort_index()
        df = df[~df.index.duplicated(keep="last")]

        cleaned = original_len - len(df)
        if cleaned > 0:
            logger.info(f"{symbol}: Cleaned {cleaned}/{original_len} bars")

        return df