Add walk-forward backtest harness + paginated history seeder
- data_provider: new fetch_historical() that paginates /time_series in 5000-bar chunks (1 credit/chunk) for arbitrary date ranges. Tolerates the API's "no data on specified dates" boundary error gracefully when prior chunks succeeded — that error is just the marker that we walked past the symbol's earliest available bar. - seed_history.py: one-shot CLI that downloads OHLCV for the configured pairs/TFs into history/<PAIR>_<TF>.csv. Skips existing files unless --force. - backtest.py: walk-forward replay through SignalAggregator with strict no-look-ahead semantics. Each bar is only visible to the engine after it has fully closed (`df.index + interval <= as_of`). Trade simulation is first-touch (SL vs TP1) with worst-case (SL first if both touched same bar), spread cost deducted from R, max-hold timeout, no-overlap. Reports per-grade win-rate / avg-R / profit-factor / max-DD, plus optional matplotlib equity + by-grade plots. - requirements: add matplotlib (lazy-imported, only needed for --plot). - .gitignore: history/ and backtest_results/ regenerable, kept untracked. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -456,6 +456,125 @@ class ForexDataProvider:
|
||||
f"Failed to fetch {td_symbol}: {e}"
|
||||
) from e
|
||||
|
||||
def fetch_historical(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: Timeframe,
|
||||
start: datetime,
|
||||
end: datetime,
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Paginated historical OHLCV fetch for backtesting seed.
|
||||
|
||||
Walks backward from `end` to `start` in 5000-bar chunks (the free-tier
|
||||
per-call max), 1 credit per chunk. Honours the shared rate limiter so
|
||||
it cooperates with any concurrent live scans.
|
||||
|
||||
Returns a single concatenated DataFrame indexed by UTC timestamp.
|
||||
"""
|
||||
if not self._api_key:
|
||||
raise DataUnavailableError("No Twelve Data API key set.")
|
||||
|
||||
if start.tzinfo is None:
|
||||
start = start.replace(tzinfo=timezone.utc)
|
||||
if end.tzinfo is None:
|
||||
end = end.replace(tzinfo=timezone.utc)
|
||||
|
||||
bars_per_day = self.BARS_PER_DAY[timeframe]
|
||||
# Stay well clear of the 5000-bar cap — leave a small margin for
|
||||
# weekend gaps that don't decrement bars on calendar-day arithmetic.
|
||||
chunk_days = max(1, int(4500 / bars_per_day))
|
||||
|
||||
td_symbol = self._to_td_symbol(symbol)
|
||||
interval = self.TD_INTERVALS[timeframe]
|
||||
|
||||
chunks: List[pd.DataFrame] = []
|
||||
cursor_end = end
|
||||
|
||||
while cursor_end > start:
|
||||
cursor_start = max(start, cursor_end - timedelta(days=chunk_days))
|
||||
|
||||
self._throttle(credits=1)
|
||||
|
||||
resp = self._session.get(
|
||||
f"{self.TD_BASE_URL}/time_series",
|
||||
params={
|
||||
"symbol": td_symbol,
|
||||
"interval": interval,
|
||||
"start_date": cursor_start.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"end_date": cursor_end.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"outputsize": 5000,
|
||||
"timezone": "UTC",
|
||||
"format": "json",
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
if isinstance(data, dict) and data.get("status") == "error":
|
||||
msg = data.get("message", "unknown")
|
||||
# If we already have chunks, an error on a deeper-history call
|
||||
# almost always means "you've walked past the available start."
|
||||
# Treat as graceful end-of-history instead of fatal.
|
||||
if chunks:
|
||||
logger.info(
|
||||
f" {td_symbol} {interval}: end of available history "
|
||||
f"reached ({msg})"
|
||||
)
|
||||
break
|
||||
raise DataUnavailableError(
|
||||
f"Twelve Data error for {td_symbol}: {msg}"
|
||||
)
|
||||
|
||||
values = data.get("values") or []
|
||||
if not values:
|
||||
logger.info(
|
||||
f" {td_symbol} {interval}: empty range "
|
||||
f"{cursor_start.date()}..{cursor_end.date()}, stopping"
|
||||
)
|
||||
break
|
||||
|
||||
records = [
|
||||
{
|
||||
"Datetime": pd.Timestamp(v["datetime"], tz="UTC"),
|
||||
"Open": float(v["open"]),
|
||||
"High": float(v["high"]),
|
||||
"Low": float(v["low"]),
|
||||
"Close": float(v["close"]),
|
||||
"Volume": float(v.get("volume") or 0),
|
||||
}
|
||||
for v in values if v.get("datetime")
|
||||
]
|
||||
df_chunk = pd.DataFrame(records).set_index("Datetime").sort_index()
|
||||
df_chunk.index.name = "Datetime"
|
||||
chunks.append(df_chunk)
|
||||
|
||||
earliest = df_chunk.index.min()
|
||||
logger.info(
|
||||
f" {td_symbol} {interval}: chunk "
|
||||
f"{earliest.date()} → {df_chunk.index.max().date()} "
|
||||
f"({len(df_chunk):,} bars)"
|
||||
)
|
||||
|
||||
# Stop when:
|
||||
# 1. The current chunk's start window already hit the requested
|
||||
# `start` boundary — nothing earlier was asked for.
|
||||
# 2. The chunk is unusually thin (< 100 bars), suggesting we've
|
||||
# reached the edge of the symbol's available history.
|
||||
if cursor_start <= start or len(df_chunk) < 100:
|
||||
break
|
||||
|
||||
cursor_end = earliest - timedelta(seconds=1)
|
||||
|
||||
if not chunks:
|
||||
raise DataUnavailableError(f"No historical data returned for {td_symbol}")
|
||||
|
||||
df = pd.concat(chunks).sort_index()
|
||||
df = df[~df.index.duplicated(keep="last")]
|
||||
df = df[(df.index >= start) & (df.index <= end)]
|
||||
return df
|
||||
|
||||
def _validate_and_clean(
|
||||
self, df: pd.DataFrame, symbol: str, timeframe: Timeframe
|
||||
) -> pd.DataFrame:
|
||||
|
||||
Reference in New Issue
Block a user