harry/ForexBot2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Strip backtesting, rebalance indicator weights toward leading signals

Browse Source 
Remove backtest.py, backtest_gpu.py, metrics.py, local CSV data,
Dukascopy loader, and all backtest CLI modes from main.py. Rebalance
engine weights: leading indicators (divergence, RSI, stochastic, BB)
now carry 66% of signal weight vs 34% for lagging (EMA, MACD). Add
ws_twelvedata.py for live tick streaming. Drop pyarrow and tqdm deps.
This commit is contained in:
Harwinder
2026-05-10 23:40:15 -04:00
parent 2e09237671
commit 7ccd8c4b04
16 changed files with 82 additions and 234029 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data_provider.py
-133
View File
@@ -2,9 +2,6 @@
ForexBot2 — Data Provider (Twelve Data SDK, USD/JPY only)
==========================================================
Fetches and validates OHLCV via the official `twelvedata` Python SDK.
No synthetic fallback — if the API fails the caller learns about it.
Adds a parquet on-disk cache so the backtest worker pool reuses one
download instead of hammering Twelve Data during a parameter sweep.
"""
import logging
@@ -29,14 +26,6 @@ class DataIntegrityError(Exception):
class ForexDataProvider:
"""Twelve Data SDK client + on-disk parquet cache."""
# Local MetaTrader CSV exports — tab-delimited, 7 columns (includes Spread)
LOCAL_CSV_MAP = {
Timeframe.M15: "USDJPY_M15.csv",
Timeframe.H1: "USDJPY_H1.csv",
Timeframe.H4: "USDJPY_H4.csv",
Timeframe.D1: "USDJPY_D1.csv",
}
TD_INTERVALS = {
Timeframe.M1: "1min",
Timeframe.M5: "5min",
@@ -48,18 +37,6 @@ class ForexDataProvider:
Timeframe.W1: "1week",
}
BARS_PER_DAY = {
Timeframe.M1: 1440,
Timeframe.M5: 288,
Timeframe.M15: 96,
Timeframe.M30: 48,
Timeframe.H1: 24,
Timeframe.H4: 6,
Timeframe.D1: 1,
Timeframe.W1: 0.143,
}
# Free plan: 8 credits / minute
RATE_LIMIT = 8
RATE_WINDOW = 60.0
@@ -116,100 +93,6 @@ class ForexDataProvider:
self._cache_timestamps[cache_key] = now
return df
def fetch_full_history(
self,
symbol: str = USDJPY,
timeframe: Timeframe = Timeframe.H1,
max_bars: int = 5000,
use_disk_cache: bool = True,
start_date: Optional[str] = None,
end_date: Optional[str] = None,
) -> pd.DataFrame:
"""
Pull the largest window Twelve Data allows in one call.
On-disk parquet cache prevents re-download across worker subprocesses
during an optimizer sweep.
"""
cache_path = self._disk_cache_path(symbol, timeframe, start_date, end_date)
if use_disk_cache and os.path.exists(cache_path):
try:
df = pd.read_parquet(cache_path)
if df.index.tz is None:
df.index = df.index.tz_localize("UTC")
logger.info(
f"Loaded {len(df)} bars from disk cache: {cache_path}"
)
return df
except Exception as e:
logger.warning(f"Disk cache read failed ({e}); refetching.")
outputsize = min(max_bars, 5000)
df = self._fetch_twelve_data(
symbol, timeframe, outputsize,
start_date=start_date, end_date=end_date,
)
df = self._validate_and_clean(df, symbol, timeframe)
if use_disk_cache:
os.makedirs(os.path.dirname(cache_path), exist_ok=True)
try:
df.to_parquet(cache_path)
logger.info(f"Cached {len(df)} bars → {cache_path}")
except Exception as e:
logger.warning(f"Disk cache write failed: {e}")
return df
def load_local_csv(
self,
timeframe: Timeframe,
data_dir: str = "market_data",
start_date: Optional[str] = None,
end_date: Optional[str] = None,
) -> Optional[pd.DataFrame]:
"""
Load OHLCV from a local MetaTrader CSV export.
Format: tab-separated, columns Time/Open/High/Low/Close/Volume/Spread.
Returns None silently if the file doesn't exist for this timeframe.
"""
fname = self.LOCAL_CSV_MAP.get(timeframe)
if fname is None:
logger.info(f"No local CSV mapping for {timeframe.value} — skipping")
return None
path = os.path.join(data_dir, fname)
if not os.path.exists(path):
logger.warning(f"Local CSV not found: {path}")
return None
df = pd.read_csv(
path, sep="\t",
names=["Time", "Open", "High", "Low", "Close", "Volume", "Spread"],
header=0,
parse_dates=["Time"],
)
df = df.set_index("Time")
df = df.drop(columns=["Spread"], errors="ignore")
for col in ("Open", "High", "Low", "Close", "Volume"):
if col in df.columns:
df[col] = pd.to_numeric(df[col], errors="coerce")
if "Volume" not in df.columns:
df["Volume"] = 0.0
if df.index.tz is None:
df.index = df.index.tz_localize("UTC")
df.index.name = "Datetime"
if start_date:
df = df[df.index >= pd.Timestamp(start_date, tz="UTC")]
if end_date:
df = df[df.index <= pd.Timestamp(end_date, tz="UTC")]
df = self._validate_and_clean(df, USDJPY, timeframe)
logger.info(f"Loaded {len(df):,} bars from local CSV: {path}")
return df
def get_latest_price(self, symbol: str = USDJPY) -> Optional[float]:
try:
df = self.fetch_ohlcv(symbol, Timeframe.M5, lookback_days=1)
@@ -240,22 +123,6 @@ class ForexDataProvider:
# ── Private ────────────────────────────────────────────────────
def _disk_cache_path(
self,
symbol: str,
timeframe: Timeframe,
start_date: Optional[str] = None,
end_date: Optional[str] = None,
) -> str:
clean = symbol.replace("=X", "").lower()
suffix = ""
if start_date or end_date:
suffix = f"_{start_date or 'min'}_{end_date or 'max'}"
return os.path.join(
self.config.backtest.cache_dir,
f"{clean}_{timeframe.value}{suffix}.parquet",
)
def _throttle(self, credits: int = 1) -> None:
"""Free tier: 8 credits / minute. Block until budget allows."""
while True: