Refocus on USD/JPY and add walk-forward backtester with regime filter
- Strip multi-pair plumbing; drop synthetic-data fallback - Add BacktestConfig + walk-forward replay through the live signal pipeline - Add metrics module (PF, Sharpe, Sortino, max DD, trade journal) - Add parallel grid optimizer with disk parquet cache - Add H4 ADX regime gate (--regime-filter, --regime-min-adx) — cuts chop losses ~84% in 2023 H1 while preserving trend edge Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Harwinder
+130
-327
@@ -1,23 +1,22 @@
|
||||
"""
|
||||
Forex Signal Intelligence Platform - Data Provider
|
||||
====================================================
|
||||
Retrieves and validates market data from Twelve Data API.
|
||||
Falls back to synthetic GBM data if the API is unavailable.
|
||||
ForexBot2 — Data Provider (Twelve Data, USD/JPY only)
|
||||
=======================================================
|
||||
Fetches and validates OHLCV from Twelve Data. No synthetic fallback —
|
||||
if the API fails the caller learns about it. Adds a parquet on-disk
|
||||
cache so the backtest worker pool reuses one download instead of
|
||||
hammering Twelve Data 27 times during a parameter sweep.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from collections import deque
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Dict, List, Optional
|
||||
from typing import Dict, Optional
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
from config import PlatformConfig, Timeframe, SYMBOL_DISPLAY
|
||||
from config import PlatformConfig, Timeframe, USDJPY, DISPLAY, PIP_VALUE
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -28,12 +27,7 @@ class DataIntegrityError(Exception):
|
||||
|
||||
|
||||
class ForexDataProvider:
|
||||
"""
|
||||
Forex data provider backed by Twelve Data API.
|
||||
|
||||
Twelve Data has native 4h candles, eliminating the H1→H4 resampling
|
||||
hack needed for Yahoo Finance. Free tier: 800 credits/day, 8 req/min.
|
||||
"""
|
||||
"""Twelve Data REST client + on-disk parquet cache."""
|
||||
|
||||
TD_BASE_URL = "https://api.twelvedata.com"
|
||||
|
||||
@@ -48,7 +42,6 @@ class ForexDataProvider:
|
||||
Timeframe.W1: "1week",
|
||||
}
|
||||
|
||||
# Approximate bars per day for each timeframe (used to size outputsize)
|
||||
BARS_PER_DAY = {
|
||||
Timeframe.M1: 1440,
|
||||
Timeframe.M5: 288,
|
||||
@@ -60,7 +53,7 @@ class ForexDataProvider:
|
||||
Timeframe.W1: 0.143,
|
||||
}
|
||||
|
||||
# Free plan: 8 API credits per minute (each symbol = 1 credit)
|
||||
# Free plan: 8 credits / minute
|
||||
RATE_LIMIT = 8
|
||||
RATE_WINDOW = 60.0
|
||||
|
||||
@@ -72,7 +65,7 @@ class ForexDataProvider:
|
||||
)
|
||||
self._cache: Dict[str, pd.DataFrame] = {}
|
||||
self._cache_timestamps: Dict[str, float] = {}
|
||||
self._request_times: deque = deque() # timestamps of recent API credits used
|
||||
self._request_times: deque = deque()
|
||||
self._session = requests.Session()
|
||||
self._session.headers.update({
|
||||
"User-Agent": "Mozilla/5.0 (compatible; ForexIntel/1.0)"
|
||||
@@ -80,22 +73,18 @@ class ForexDataProvider:
|
||||
|
||||
@staticmethod
|
||||
def _to_td_symbol(symbol: str) -> str:
|
||||
"""Convert internal symbol (EURUSD=X) to Twelve Data format (EUR/USD)."""
|
||||
clean = symbol.replace("=X", "")
|
||||
return f"{clean[:3]}/{clean[3:]}"
|
||||
|
||||
# ── Public API ─────────────────────────────────────────────────
|
||||
|
||||
def fetch_ohlcv(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: Timeframe,
|
||||
symbol: str = USDJPY,
|
||||
timeframe: Timeframe = Timeframe.H1,
|
||||
lookback_days: Optional[int] = None,
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Fetch OHLCV data for a Forex pair with full validation.
|
||||
|
||||
Returns a DataFrame with columns: Open, High, Low, Close, Volume
|
||||
Index is a timezone-aware DatetimeIndex in UTC.
|
||||
"""
|
||||
"""Fetch OHLCV with in-memory caching for live mode."""
|
||||
if lookback_days is None:
|
||||
lookback_days = self.config.data_lookback_days
|
||||
|
||||
@@ -111,55 +100,63 @@ class ForexDataProvider:
|
||||
if cached is not None and not cached.empty:
|
||||
return cached
|
||||
|
||||
df = self._fetch_twelve_data(symbol, timeframe, lookback_days)
|
||||
outputsize = max(
|
||||
300,
|
||||
min(int(lookback_days * self.BARS_PER_DAY[timeframe]) + 20, 5000),
|
||||
)
|
||||
df = self._fetch_twelve_data(symbol, timeframe, outputsize)
|
||||
df = self._validate_and_clean(df, symbol, timeframe)
|
||||
|
||||
self._cache[cache_key] = df
|
||||
self._cache_timestamps[cache_key] = now
|
||||
return df
|
||||
|
||||
def prefetch_batch(
|
||||
def fetch_full_history(
|
||||
self,
|
||||
symbols: List[str],
|
||||
timeframes: List[Timeframe],
|
||||
lookback_days: Optional[int] = None,
|
||||
) -> None:
|
||||
symbol: str = USDJPY,
|
||||
timeframe: Timeframe = Timeframe.H1,
|
||||
max_bars: int = 5000,
|
||||
use_disk_cache: bool = True,
|
||||
start_date: Optional[str] = None,
|
||||
end_date: Optional[str] = None,
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Batch-fetch all symbols for each timeframe in a single API call per TF.
|
||||
|
||||
Twelve Data accepts comma-separated symbols, so 7 pairs × 3 timeframes
|
||||
= 3 requests instead of 21 — well within the free-tier 8 req/min limit.
|
||||
Results are stored in the local cache; subsequent fetch_ohlcv calls
|
||||
for the same symbol/TF will be served from cache.
|
||||
Pull the largest window Twelve Data allows in one call.
|
||||
On-disk parquet cache prevents re-download across worker subprocesses
|
||||
during an optimizer sweep.
|
||||
"""
|
||||
if lookback_days is None:
|
||||
lookback_days = self.config.data_lookback_days
|
||||
cache_path = self._disk_cache_path(symbol, timeframe, start_date, end_date)
|
||||
|
||||
for tf in timeframes:
|
||||
if use_disk_cache and os.path.exists(cache_path):
|
||||
try:
|
||||
self._batch_fetch_timeframe(symbols, tf, lookback_days)
|
||||
time.sleep(1.0) # brief pause between TF requests
|
||||
df = pd.read_parquet(cache_path)
|
||||
if df.index.tz is None:
|
||||
df.index = df.index.tz_localize("UTC")
|
||||
logger.info(
|
||||
f"Loaded {len(df)} bars from disk cache: {cache_path}"
|
||||
)
|
||||
return df
|
||||
except Exception as e:
|
||||
logger.warning(f"Batch prefetch failed for {tf.value}: {e}")
|
||||
logger.warning(f"Disk cache read failed ({e}); refetching.")
|
||||
|
||||
def fetch_multiple(
|
||||
self,
|
||||
symbols: List[str],
|
||||
timeframe: Timeframe,
|
||||
lookback_days: Optional[int] = None,
|
||||
) -> Dict[str, pd.DataFrame]:
|
||||
"""Fetch data for multiple symbols, returning dict of DataFrames."""
|
||||
results = {}
|
||||
for symbol in symbols:
|
||||
outputsize = min(max_bars, 5000)
|
||||
df = self._fetch_twelve_data(
|
||||
symbol, timeframe, outputsize,
|
||||
start_date=start_date, end_date=end_date,
|
||||
)
|
||||
df = self._validate_and_clean(df, symbol, timeframe)
|
||||
|
||||
if use_disk_cache:
|
||||
os.makedirs(os.path.dirname(cache_path), exist_ok=True)
|
||||
try:
|
||||
df = self.fetch_ohlcv(symbol, timeframe, lookback_days)
|
||||
results[symbol] = df
|
||||
df.to_parquet(cache_path)
|
||||
logger.info(f"Cached {len(df)} bars → {cache_path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to fetch {symbol}: {e}")
|
||||
return results
|
||||
logger.warning(f"Disk cache write failed: {e}")
|
||||
|
||||
def get_latest_price(self, symbol: str) -> Optional[float]:
|
||||
"""Return the most recent close price for a symbol."""
|
||||
return df
|
||||
|
||||
def get_latest_price(self, symbol: str = USDJPY) -> Optional[float]:
|
||||
try:
|
||||
df = self.fetch_ohlcv(symbol, Timeframe.M5, lookback_days=1)
|
||||
if df is not None and not df.empty:
|
||||
@@ -168,47 +165,47 @@ class ForexDataProvider:
|
||||
pass
|
||||
return None
|
||||
|
||||
def get_spread_estimate(self, symbol: str) -> float:
|
||||
"""Estimate typical spread from recent M5 high-low ranges."""
|
||||
def get_spread_estimate(self, symbol: str = USDJPY) -> float:
|
||||
"""Estimate spread from recent M5 H-L median; fallback to 1 pip on JPY."""
|
||||
try:
|
||||
df = self.fetch_ohlcv(symbol, Timeframe.M5, lookback_days=1)
|
||||
if df is not None and not df.empty:
|
||||
return float((df["High"] - df["Low"]).median())
|
||||
except Exception:
|
||||
pass
|
||||
# Fallback: 1 pip = 0.01 for JPY pairs
|
||||
return PIP_VALUE
|
||||
|
||||
fallback_spreads = {
|
||||
"EURUSD=X": 0.00010, "GBPUSD=X": 0.00015,
|
||||
"USDJPY=X": 0.010, "USDCHF=X": 0.00015,
|
||||
"AUDUSD=X": 0.00015, "USDCAD=X": 0.00018,
|
||||
"NZDUSD=X": 0.00020,
|
||||
}
|
||||
return fallback_spreads.get(symbol, 0.00020)
|
||||
|
||||
def get_pair_info(self, symbol: str) -> Dict:
|
||||
"""Return display name and pip value for a pair."""
|
||||
display = SYMBOL_DISPLAY.get(symbol, symbol)
|
||||
is_jpy = "JPY" in symbol
|
||||
def get_pair_info(self, symbol: str = USDJPY) -> Dict:
|
||||
return {
|
||||
"symbol": symbol,
|
||||
"display": display,
|
||||
"pip_value": 0.01 if is_jpy else 0.0001,
|
||||
"is_jpy_pair": is_jpy,
|
||||
"display": DISPLAY,
|
||||
"pip_value": PIP_VALUE,
|
||||
"is_jpy_pair": True,
|
||||
}
|
||||
|
||||
# ── Private Methods ────────────────────────────────────────────
|
||||
# ── Private ────────────────────────────────────────────────────
|
||||
|
||||
def _disk_cache_path(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: Timeframe,
|
||||
start_date: Optional[str] = None,
|
||||
end_date: Optional[str] = None,
|
||||
) -> str:
|
||||
clean = symbol.replace("=X", "").lower()
|
||||
suffix = ""
|
||||
if start_date or end_date:
|
||||
suffix = f"_{start_date or 'min'}_{end_date or 'max'}"
|
||||
return os.path.join(
|
||||
self.config.backtest.cache_dir,
|
||||
f"{clean}_{timeframe.value}{suffix}.parquet",
|
||||
)
|
||||
|
||||
def _throttle(self, credits: int = 1) -> None:
|
||||
"""
|
||||
Block until `credits` API credits are available.
|
||||
|
||||
Twelve Data free tier: 8 credits/minute. Each symbol in any request
|
||||
(single or batch) consumes 1 credit. This method enforces the limit
|
||||
regardless of whether requests are batched or individual.
|
||||
"""
|
||||
"""Free tier: 8 credits / minute. Block until budget allows."""
|
||||
while True:
|
||||
now = time.time()
|
||||
# Evict credits older than the rate window
|
||||
while self._request_times and now - self._request_times[0] >= self.RATE_WINDOW:
|
||||
self._request_times.popleft()
|
||||
|
||||
@@ -218,276 +215,82 @@ class ForexDataProvider:
|
||||
self._request_times.append(time.time())
|
||||
return
|
||||
|
||||
# Wait until the oldest credit expires
|
||||
wait = (self._request_times[0] + self.RATE_WINDOW) - now + 0.2
|
||||
logger.info(
|
||||
f"Rate limit: {len(self._request_times)}/{self.RATE_LIMIT} credits used "
|
||||
f"— waiting {wait:.1f}s for next window..."
|
||||
f"— waiting {wait:.1f}s..."
|
||||
)
|
||||
time.sleep(wait)
|
||||
|
||||
def _batch_fetch_timeframe(
|
||||
self, symbols: List[str], timeframe: Timeframe, lookback_days: int
|
||||
) -> None:
|
||||
"""
|
||||
Single API call fetching all symbols for one timeframe.
|
||||
Twelve Data returns a dict keyed by symbol when multiple are requested.
|
||||
Parsed DataFrames are validated and stored in the cache.
|
||||
"""
|
||||
def _fetch_twelve_data(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: Timeframe,
|
||||
outputsize: int,
|
||||
start_date: Optional[str] = None,
|
||||
end_date: Optional[str] = None,
|
||||
) -> pd.DataFrame:
|
||||
if not self._api_key:
|
||||
return
|
||||
raise DataIntegrityError(
|
||||
"No Twelve Data API key. Set TWELVE_DATA_API_KEY env var "
|
||||
"or config.twelve_data_api_key."
|
||||
)
|
||||
|
||||
td_symbols = [self._to_td_symbol(s) for s in symbols]
|
||||
td_symbol = self._to_td_symbol(symbol)
|
||||
interval = self.TD_INTERVALS[timeframe]
|
||||
outputsize = min(
|
||||
int(lookback_days * self.BARS_PER_DAY[timeframe]) + 20, 5000
|
||||
)
|
||||
outputsize = max(outputsize, 300)
|
||||
|
||||
# Each symbol in a batch request costs 1 credit
|
||||
self._throttle(credits=len(symbols))
|
||||
self._throttle(credits=1)
|
||||
|
||||
params = {
|
||||
"symbol": td_symbol,
|
||||
"interval": interval,
|
||||
"outputsize": outputsize,
|
||||
"apikey": self._api_key,
|
||||
"timezone": "UTC",
|
||||
"format": "json",
|
||||
}
|
||||
if start_date:
|
||||
params["start_date"] = start_date
|
||||
if end_date:
|
||||
params["end_date"] = end_date
|
||||
|
||||
resp = self._session.get(
|
||||
f"{self.TD_BASE_URL}/time_series",
|
||||
params={
|
||||
"symbol": ",".join(td_symbols),
|
||||
"interval": interval,
|
||||
"outputsize": outputsize,
|
||||
"apikey": self._api_key,
|
||||
"timezone": "UTC",
|
||||
"format": "json",
|
||||
},
|
||||
params=params,
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
# If the whole batch request returned a top-level error, raise it
|
||||
if isinstance(data, dict) and data.get("status") == "error":
|
||||
if data.get("status") == "error":
|
||||
raise DataIntegrityError(
|
||||
f"Batch API error ({timeframe.value}): {data.get('message', 'unknown')}"
|
||||
f"Twelve Data error for {td_symbol}: {data.get('message', 'unknown')}"
|
||||
)
|
||||
|
||||
# Batch response: {"EUR/USD": {...}, "GBP/USD": {...}, ...}
|
||||
now = time.time()
|
||||
for symbol, td_sym in zip(symbols, td_symbols):
|
||||
try:
|
||||
sym_data = data.get(td_sym, {})
|
||||
if sym_data.get("status") == "error":
|
||||
logger.warning(
|
||||
f"Batch fetch error for {td_sym}: {sym_data.get('message')}"
|
||||
)
|
||||
continue
|
||||
values = data.get("values")
|
||||
if not values:
|
||||
raise DataIntegrityError(f"Empty response from Twelve Data for {td_symbol}")
|
||||
|
||||
values = sym_data.get("values")
|
||||
if not values:
|
||||
logger.warning(f"No values in batch response for {td_sym}")
|
||||
continue
|
||||
records = [
|
||||
{
|
||||
"Datetime": pd.Timestamp(v["datetime"], tz="UTC"),
|
||||
"Open": float(v["open"]),
|
||||
"High": float(v["high"]),
|
||||
"Low": float(v["low"]),
|
||||
"Close": float(v["close"]),
|
||||
"Volume": float(v.get("volume") or 0),
|
||||
}
|
||||
for v in reversed(values)
|
||||
]
|
||||
|
||||
records = [
|
||||
{
|
||||
"Datetime": pd.Timestamp(v["datetime"], tz="UTC"),
|
||||
"Open": float(v["open"]),
|
||||
"High": float(v["high"]),
|
||||
"Low": float(v["low"]),
|
||||
"Close": float(v["close"]),
|
||||
"Volume": float(v.get("volume") or 0),
|
||||
}
|
||||
for v in reversed(values)
|
||||
]
|
||||
df = pd.DataFrame(records).set_index("Datetime")
|
||||
df.index.name = "Datetime"
|
||||
df = self._validate_and_clean(df, symbol, timeframe)
|
||||
|
||||
cache_key = f"{symbol}_{timeframe.value}"
|
||||
self._cache[cache_key] = df
|
||||
self._cache_timestamps[cache_key] = now
|
||||
logger.debug(f"Batch cached {len(df)} bars for {td_sym} @ {interval}")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to parse batch data for {td_sym}: {e}")
|
||||
|
||||
def _fetch_twelve_data(
|
||||
self, symbol: str, timeframe: Timeframe, lookback_days: int
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Fetch OHLCV from Twelve Data REST API.
|
||||
Falls back to synthetic GBM data if API is unavailable or key is missing.
|
||||
"""
|
||||
if not self._api_key:
|
||||
logger.warning("No Twelve Data API key — using synthetic data. "
|
||||
"Set TWELVE_DATA_API_KEY env var or config.twelve_data_api_key.")
|
||||
return self._generate_synthetic_data(symbol, timeframe, lookback_days)
|
||||
|
||||
td_symbol = self._to_td_symbol(symbol)
|
||||
interval = self.TD_INTERVALS[timeframe]
|
||||
outputsize = min(
|
||||
int(lookback_days * self.BARS_PER_DAY[timeframe]) + 20,
|
||||
5000,
|
||||
)
|
||||
outputsize = max(outputsize, 300)
|
||||
|
||||
self._throttle(credits=1)
|
||||
|
||||
try:
|
||||
resp = self._session.get(
|
||||
f"{self.TD_BASE_URL}/time_series",
|
||||
params={
|
||||
"symbol": td_symbol,
|
||||
"interval": interval,
|
||||
"outputsize": outputsize,
|
||||
"apikey": self._api_key,
|
||||
"timezone": "UTC",
|
||||
"format": "json",
|
||||
},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
if data.get("status") == "error":
|
||||
raise DataIntegrityError(
|
||||
f"Twelve Data API error for {td_symbol}: {data.get('message', 'unknown')}"
|
||||
)
|
||||
|
||||
values = data.get("values")
|
||||
if not values:
|
||||
raise DataIntegrityError(f"Empty response from Twelve Data for {td_symbol}")
|
||||
|
||||
# Values arrive newest-first; reverse to chronological order
|
||||
records = [
|
||||
{
|
||||
"Datetime": pd.Timestamp(v["datetime"], tz="UTC"),
|
||||
"Open": float(v["open"]),
|
||||
"High": float(v["high"]),
|
||||
"Low": float(v["low"]),
|
||||
"Close": float(v["close"]),
|
||||
"Volume": float(v.get("volume") or 0),
|
||||
}
|
||||
for v in reversed(values)
|
||||
]
|
||||
|
||||
df = pd.DataFrame(records).set_index("Datetime")
|
||||
df.index.name = "Datetime"
|
||||
logger.debug(f"Fetched {len(df)} bars for {td_symbol} @ {interval}")
|
||||
return df
|
||||
|
||||
except DataIntegrityError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.info(f"Twelve Data fetch failed for {symbol} ({e}), using synthetic data.")
|
||||
return self._generate_synthetic_data(symbol, timeframe, lookback_days)
|
||||
|
||||
def _generate_synthetic_data(
|
||||
self, symbol: str, timeframe: Timeframe, lookback_days: int
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Generate realistic synthetic Forex OHLCV using GBM with mean-reversion.
|
||||
Used as a fallback when live data is unavailable.
|
||||
"""
|
||||
pair_params = {
|
||||
"EURUSD=X": {"price": 1.1350, "vol": 0.07, "drift": -0.001},
|
||||
"GBPUSD=X": {"price": 1.3200, "vol": 0.08, "drift": 0.002},
|
||||
"USDJPY=X": {"price": 144.50, "vol": 0.09, "drift": 0.003},
|
||||
"USDCHF=X": {"price": 0.8850, "vol": 0.07, "drift": -0.001},
|
||||
"AUDUSD=X": {"price": 0.6550, "vol": 0.10, "drift": -0.002},
|
||||
"USDCAD=X": {"price": 1.3650, "vol": 0.07, "drift": 0.001},
|
||||
"NZDUSD=X": {"price": 0.6100, "vol": 0.10, "drift": -0.001},
|
||||
"EURGBP=X": {"price": 0.8590, "vol": 0.06, "drift": 0.000},
|
||||
"EURJPY=X": {"price": 163.90, "vol": 0.09, "drift": 0.002},
|
||||
"GBPJPY=X": {"price": 190.70, "vol": 0.11, "drift": 0.003},
|
||||
"AUDJPY=X": {"price": 94.60, "vol": 0.11, "drift": 0.001},
|
||||
"EURCHF=X": {"price": 0.9380, "vol": 0.05, "drift": 0.000},
|
||||
"CADJPY=X": {"price": 105.80, "vol": 0.09, "drift": 0.001},
|
||||
}
|
||||
|
||||
params = pair_params.get(symbol, {"price": 1.0000, "vol": 0.08, "drift": 0.000})
|
||||
base_price = params["price"]
|
||||
annual_vol = params["vol"]
|
||||
annual_drift = params["drift"]
|
||||
|
||||
tf_minutes = {
|
||||
Timeframe.M1: 1, Timeframe.M5: 5, Timeframe.M15: 15,
|
||||
Timeframe.M30: 30, Timeframe.H1: 60, Timeframe.H4: 240,
|
||||
Timeframe.D1: 1440, Timeframe.W1: 10080,
|
||||
}
|
||||
bar_minutes = tf_minutes.get(timeframe, 60)
|
||||
bars_per_year = (365.25 * 24 * 60) / bar_minutes
|
||||
dt = 1.0 / bars_per_year
|
||||
|
||||
n_bars = min(int(lookback_days * 24 * 60 / bar_minutes), 2000)
|
||||
n_bars = max(n_bars, 300)
|
||||
|
||||
seed_base = int(hashlib.md5(symbol.encode()).hexdigest()[:8], 16)
|
||||
time_component = int(time.time()) // 3600
|
||||
np.random.seed((seed_base + time_component) % (2**31))
|
||||
|
||||
bar_vol = annual_vol * np.sqrt(dt)
|
||||
bar_drift = annual_drift * dt
|
||||
|
||||
regime_len = np.random.randint(20, 80)
|
||||
trend_strength = np.random.normal(0, 0.3)
|
||||
|
||||
price = base_price
|
||||
prices = [price]
|
||||
|
||||
for i in range(1, n_bars):
|
||||
if i % regime_len == 0:
|
||||
regime_len = np.random.randint(20, 80)
|
||||
trend_strength = np.random.normal(0, 0.3)
|
||||
|
||||
mr_force = -0.001 * (price - base_price) / base_price
|
||||
ret = bar_drift + trend_strength * dt + mr_force + bar_vol * np.random.randn()
|
||||
|
||||
if np.random.random() < 0.02:
|
||||
ret *= np.random.uniform(2.0, 4.0)
|
||||
|
||||
price *= (1 + ret)
|
||||
prices.append(price)
|
||||
|
||||
prices = np.array(prices)
|
||||
opens = np.roll(prices, 1)
|
||||
opens[0] = prices[0]
|
||||
|
||||
typical_range = bar_vol * base_price * 1.5
|
||||
highs = np.maximum(prices, opens) + np.abs(np.random.normal(0, typical_range * 0.6, n_bars))
|
||||
lows = np.minimum(prices, opens) - np.abs(np.random.normal(0, typical_range * 0.6, n_bars))
|
||||
highs = np.maximum(highs, np.maximum(opens, prices))
|
||||
lows = np.minimum(lows, np.minimum(opens, prices))
|
||||
|
||||
volume = np.random.lognormal(mean=10, sigma=0.5, size=n_bars).astype(int)
|
||||
|
||||
freq_map = {
|
||||
Timeframe.M1: "1min", Timeframe.M5: "5min",
|
||||
Timeframe.M15: "15min", Timeframe.M30: "30min",
|
||||
Timeframe.H1: "1h", Timeframe.H4: "4h",
|
||||
Timeframe.D1: "D", Timeframe.W1: "W",
|
||||
}
|
||||
freq = freq_map.get(timeframe, "1h")
|
||||
end_time = pd.Timestamp.now(tz="UTC").floor(f"{bar_minutes}min")
|
||||
index = pd.date_range(end=end_time, periods=n_bars, freq=freq, tz="UTC")
|
||||
|
||||
df = pd.DataFrame({
|
||||
"Open": opens, "High": highs, "Low": lows,
|
||||
"Close": prices, "Volume": volume,
|
||||
}, index=index)
|
||||
df = pd.DataFrame(records).set_index("Datetime")
|
||||
df.index.name = "Datetime"
|
||||
|
||||
logger.info(
|
||||
f"Generated {n_bars} synthetic {timeframe.value} bars for "
|
||||
f"{SYMBOL_DISPLAY.get(symbol, symbol)} @ {prices[-1]:.5f}"
|
||||
)
|
||||
logger.debug(f"Fetched {len(df)} bars for {td_symbol} @ {interval}")
|
||||
return df
|
||||
|
||||
def _validate_and_clean(
|
||||
self, df: pd.DataFrame, symbol: str, timeframe: Timeframe
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Validate and clean OHLCV data.
|
||||
|
||||
Steps: drop all-NaN rows → ffill small gaps → remove future timestamps
|
||||
→ fix OHLC inconsistencies → remove >10-sigma outliers → sort & deduplicate.
|
||||
"""
|
||||
if df is None or df.empty:
|
||||
raise DataIntegrityError(f"Empty dataset for {symbol}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user