Add walk-forward backtest harness + paginated history seeder

- data_provider: new fetch_historical() that paginates /time_series in
  5000-bar chunks (1 credit/chunk) for arbitrary date ranges. Tolerates
  the API's "no data on specified dates" boundary error gracefully when
  prior chunks succeeded — that error is just the marker that we walked
  past the symbol's earliest available bar.
- seed_history.py: one-shot CLI that downloads OHLCV for the configured
  pairs/TFs into history/<PAIR>_<TF>.csv. Skips existing files unless
  --force.
- backtest.py: walk-forward replay through SignalAggregator with strict
  no-look-ahead semantics. Each bar is only visible to the engine after
  it has fully closed (`df.index + interval <= as_of`). Trade simulation
  is first-touch (SL vs TP1) with worst-case (SL first if both touched
  same bar), spread cost deducted from R, max-hold timeout, no-overlap.
  Reports per-grade win-rate / avg-R / profit-factor / max-DD, plus
  optional matplotlib equity + by-grade plots.
- requirements: add matplotlib (lazy-imported, only needed for --plot).
- .gitignore: history/ and backtest_results/ regenerable, kept untracked.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Harwinder
2026-05-06 13:29:48 -04:00
parent 402d49e094
commit 6d235061b9
5 changed files with 971 additions and 0 deletions
+119
View File
@@ -456,6 +456,125 @@ class ForexDataProvider:
f"Failed to fetch {td_symbol}: {e}"
) from e
def fetch_historical(
self,
symbol: str,
timeframe: Timeframe,
start: datetime,
end: datetime,
) -> pd.DataFrame:
"""
Paginated historical OHLCV fetch for backtesting seed.
Walks backward from `end` to `start` in 5000-bar chunks (the free-tier
per-call max), 1 credit per chunk. Honours the shared rate limiter so
it cooperates with any concurrent live scans.
Returns a single concatenated DataFrame indexed by UTC timestamp.
"""
if not self._api_key:
raise DataUnavailableError("No Twelve Data API key set.")
if start.tzinfo is None:
start = start.replace(tzinfo=timezone.utc)
if end.tzinfo is None:
end = end.replace(tzinfo=timezone.utc)
bars_per_day = self.BARS_PER_DAY[timeframe]
# Stay well clear of the 5000-bar cap — leave a small margin for
# weekend gaps that don't decrement bars on calendar-day arithmetic.
chunk_days = max(1, int(4500 / bars_per_day))
td_symbol = self._to_td_symbol(symbol)
interval = self.TD_INTERVALS[timeframe]
chunks: List[pd.DataFrame] = []
cursor_end = end
while cursor_end > start:
cursor_start = max(start, cursor_end - timedelta(days=chunk_days))
self._throttle(credits=1)
resp = self._session.get(
f"{self.TD_BASE_URL}/time_series",
params={
"symbol": td_symbol,
"interval": interval,
"start_date": cursor_start.strftime("%Y-%m-%d %H:%M:%S"),
"end_date": cursor_end.strftime("%Y-%m-%d %H:%M:%S"),
"outputsize": 5000,
"timezone": "UTC",
"format": "json",
},
timeout=30,
)
resp.raise_for_status()
data = resp.json()
if isinstance(data, dict) and data.get("status") == "error":
msg = data.get("message", "unknown")
# If we already have chunks, an error on a deeper-history call
# almost always means "you've walked past the available start."
# Treat as graceful end-of-history instead of fatal.
if chunks:
logger.info(
f" {td_symbol} {interval}: end of available history "
f"reached ({msg})"
)
break
raise DataUnavailableError(
f"Twelve Data error for {td_symbol}: {msg}"
)
values = data.get("values") or []
if not values:
logger.info(
f" {td_symbol} {interval}: empty range "
f"{cursor_start.date()}..{cursor_end.date()}, stopping"
)
break
records = [
{
"Datetime": pd.Timestamp(v["datetime"], tz="UTC"),
"Open": float(v["open"]),
"High": float(v["high"]),
"Low": float(v["low"]),
"Close": float(v["close"]),
"Volume": float(v.get("volume") or 0),
}
for v in values if v.get("datetime")
]
df_chunk = pd.DataFrame(records).set_index("Datetime").sort_index()
df_chunk.index.name = "Datetime"
chunks.append(df_chunk)
earliest = df_chunk.index.min()
logger.info(
f" {td_symbol} {interval}: chunk "
f"{earliest.date()} → {df_chunk.index.max().date()} "
f"({len(df_chunk):,} bars)"
)
# Stop when:
# 1. The current chunk's start window already hit the requested
# `start` boundary — nothing earlier was asked for.
# 2. The chunk is unusually thin (< 100 bars), suggesting we've
# reached the edge of the symbol's available history.
if cursor_start <= start or len(df_chunk) < 100:
break
cursor_end = earliest - timedelta(seconds=1)
if not chunks:
raise DataUnavailableError(f"No historical data returned for {td_symbol}")
df = pd.concat(chunks).sort_index()
df = df[~df.index.duplicated(keep="last")]
df = df[(df.index >= start) & (df.index <= end)]
return df
def _validate_and_clean(
self, df: pd.DataFrame, symbol: str, timeframe: Timeframe
) -> pd.DataFrame: