backtest: real CuPy GPU backend (kernels are bit-identical CPU twins); books byte-identical on both backends (2026-09-18)
CuPy from pip had never compiled a kernel on the dev box (no CUDA_PATH for the splayed CUDA-12 wheel layout); the probe swallowed the error and every book to date was built on the NumPy path under a "GPU backtester" name. - backtest/gpu_kernels.py (new): lazily probed, never-silent BACKEND state (CUDA_PATH pointed at the nvidia-cuda-runtime-cu12 header wheel; the probe compiles a kernel and keeps the failure reason), --backend auto|gpu|cpu and FB2_BACKEND, and three CUDA kernels compiled with --fmad=false: ewm_batch (the _gpu_ewm_alpha recurrence, one thread per series), roll_mean_var (pandas 3.0.2 roll_mean/roll_var replica, Kahan + Welford + unstable recompute), resolve_trades (trailing and static exits, one thread per trade). - gpu_backtest.py: precompute_indicators and simulate_trades dispatch to the GPU twins or to the untouched NumPy paths; caches are host NumPy and device memory is released before the replay workers fork; the Backend: line names the device or the fallback reason. - test_all.py Module 36 (7 tests): a visible CUDA device must never fall back silently; kernels vs their CPU references on synthetic edge cases; backend selection; import guards (cupy banned from the live roots). Suite 536. - CLAUDE.md, instructions_for_backtest.txt, research note with the evidence. Verification: all five reference MD5s reproduced on the GPU and with --backend cpu (ten runs, GPU cache fallbacks 0); precompute GPU == CPU on all 42 cached windows (2.34 M bars); suite green with GPU, with CUDA_VISIBLE_DEVICES="" and with FB2_BACKEND=cpu; --workers 1 and --record pairs equal. The per-bar replay stays the live Python signal code on CPU workers by design; wall time is roughly unchanged (precompute ~3x faster).
This commit is contained in:
Harwinder Sekhon
+300
-1
@@ -10998,7 +10998,7 @@ def _():
|
||||
def _():
|
||||
"""The corpus is trained on offline in venv/; anything the live signal
|
||||
path can reach must stay numpy-only (requirements.txt is the contract)."""
|
||||
banned = {"sklearn", "scipy", "lightgbm", "torch", "transformers"}
|
||||
banned = {"sklearn", "scipy", "lightgbm", "torch", "transformers", "cupy"}
|
||||
roots = [Path(".")] + [Path(p) for p in ("providers", "pair_signals")]
|
||||
offenders = []
|
||||
for root in roots:
|
||||
@@ -13490,6 +13490,305 @@ def _():
|
||||
finally:
|
||||
os.remove(path)
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# MODULE 36: GPU backend parity (backtest/gpu_kernels.py, 2026-09-18)
|
||||
# The CUDA kernels must be bit-for-bit twins of the NumPy/pandas code;
|
||||
# a fallback to the CPU must never be silent (it was, for months).
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
print("\n═══ Module 36: GPU backend parity ═══")
|
||||
|
||||
import importlib
|
||||
import importlib.util as _ilu36
|
||||
from backtest import gpu_kernels as _gk36
|
||||
from backtest import gpu_backtest as _gb36
|
||||
|
||||
_GPU36 = _gk36.BACKEND.available
|
||||
if not _GPU36:
|
||||
print(f" (CUDA unavailable here: {_gk36.BACKEND.describe()} — GPU-path "
|
||||
f"assertions below print a notice instead of running)")
|
||||
|
||||
|
||||
def _same36(a, b) -> bool:
|
||||
"""Equality that treats NaN == NaN (floats, Timestamps, strings, None)."""
|
||||
if isinstance(a, float) and isinstance(b, float) and np.isnan(a) and np.isnan(b):
|
||||
return True
|
||||
return a == b
|
||||
|
||||
|
||||
@test("GPU backend: a visible CUDA device must never fall back silently")
|
||||
def _():
|
||||
if _ilu36.find_spec("cupy") is None:
|
||||
print(" (cupy not installed: the NumPy path is the only path on this box)")
|
||||
return
|
||||
try:
|
||||
_cupy36 = importlib.import_module("cupy") # not a literal import: the AST guard scans this file
|
||||
n_dev = int(_cupy36.cuda.runtime.getDeviceCount())
|
||||
except Exception as e: # noqa: BLE001
|
||||
print(f" (cupy present but no usable CUDA device: {type(e).__name__})")
|
||||
return
|
||||
if n_dev < 1:
|
||||
print(" (no CUDA device visible)")
|
||||
return
|
||||
if _gk36.BACKEND.mode == "cpu":
|
||||
print(" (backend forced to cpu by FB2_BACKEND)")
|
||||
return
|
||||
assert _gk36.BACKEND.available, (
|
||||
f"cupy sees {n_dev} device(s) but the backend fell back: {_gk36.BACKEND.reason}")
|
||||
assert _gk36.BACKEND.describe().startswith("CuPy GPU ("), _gk36.BACKEND.describe()
|
||||
assert _gk36.BACKEND.device_name and _gk36.BACKEND.cupy_version
|
||||
|
||||
|
||||
@test("GPU backend: ewm_batch is bit-identical to _gpu_ewm_alpha (NaN head, interior NaN, flat run, short)")
|
||||
def _():
|
||||
if not _GPU36:
|
||||
print(" (no CUDA: kernel not exercised)")
|
||||
return
|
||||
rng = np.random.default_rng(36)
|
||||
n = 700
|
||||
gold = 3300.0 + np.cumsum(rng.normal(0, 3.0, n))
|
||||
nan_head = gold.copy(); nan_head[:15] = np.nan
|
||||
interior = gold.copy(); interior[100:107] = np.nan; interior[400] = np.nan
|
||||
flat = gold.copy(); flat[200:260] = flat[199]
|
||||
short = gold[:9]
|
||||
gainlike = np.where(rng.normal(size=n) > 0, np.abs(rng.normal(size=n)), 0.0)
|
||||
gainlike[0] = 0.0
|
||||
series, alphas, minps, refs = [], [], [], []
|
||||
for x in (gold, nan_head, interior, flat, short, gainlike):
|
||||
for alpha, mp in ((2.0 / 10, 0), (2.0 / 201, 0), (1.0 / 14, 14), (1.0 / 10, 10), (0.5, 3)):
|
||||
series.append(x); alphas.append(alpha); minps.append(mp)
|
||||
refs.append(_gb36._gpu_ewm_alpha(x, alpha, mp))
|
||||
outs = _gk36.ewm_batch(series, alphas, minps)
|
||||
assert len(outs) == len(refs) == 30
|
||||
for i, (o, r) in enumerate(zip(outs, refs)):
|
||||
assert o.dtype == np.float64 and o.shape == r.shape, i
|
||||
assert np.array_equal(o, r, equal_nan=True), (
|
||||
f"case {i}: {int(np.sum(~((o == r) | (np.isnan(o) & np.isnan(r)))))} bits differ")
|
||||
# and the CPU twin itself still equals pandas on a clean series (live-engine tie)
|
||||
assert np.array_equal(
|
||||
pd.Series(gold).ewm(span=200, adjust=False).mean().to_numpy(),
|
||||
_gk36.ewm_batch([gold], [2.0 / 201], [0])[0], equal_nan=True)
|
||||
assert _gk36.ewm_batch([], [], []) == []
|
||||
|
||||
|
||||
@test("GPU backend: roll_mean_var is bit-identical to pandas rolling mean/var/std (NaN prefix, mid NaN, flat window, spike)")
|
||||
def _():
|
||||
if not _GPU36:
|
||||
print(" (no CUDA: kernel not exercised)")
|
||||
return
|
||||
rng = np.random.default_rng(37)
|
||||
n = 900
|
||||
gold = 3300.0 + np.cumsum(rng.normal(0, 3.0, n))
|
||||
nan_prefix = gold.copy(); nan_prefix[:19] = np.nan
|
||||
mid_nan = gold.copy(); mid_nan[300:305] = np.nan # nobs dips below min_periods, recovers
|
||||
flat = gold.copy(); flat[100:120] = 3311.25; flat[120] = 3311.75 # exactly one window of equal values
|
||||
spike = gold.copy(); spike[500] = 1e6 # no recompute trigger
|
||||
cancel = gold.copy(); cancel[600] = 1e12 # trips the InvCondTol recompute
|
||||
tiny = gold[:7] # n < window: all NaN
|
||||
centred = gold - gold.mean() # sign changes exercise neg_ct
|
||||
series = [gold, nan_prefix, mid_nan, flat, spike, cancel, tiny, centred, -gold]
|
||||
means, vars_, stds = _gk36.roll_mean_var(series, 20, ddof=0)
|
||||
for i, x in enumerate(series):
|
||||
s = pd.Series(x)
|
||||
pm = s.rolling(20).mean().to_numpy()
|
||||
pv = s.rolling(20).var(ddof=0).to_numpy()
|
||||
ps = s.rolling(20).std(ddof=0).to_numpy()
|
||||
assert np.array_equal(means[i], pm, equal_nan=True), f"mean case {i}"
|
||||
assert np.array_equal(vars_[i], pv, equal_nan=True), f"var case {i}"
|
||||
assert np.array_equal(stds[i], ps, equal_nan=True), f"std case {i}"
|
||||
assert np.isnan(means[6]).all() and np.isnan(vars_[6]).all()
|
||||
# ddof=1 too (pandas' default), on the plain series
|
||||
(m1,), (v1,), _ = _gk36.roll_mean_var([gold], 20, ddof=1)
|
||||
assert np.array_equal(v1, pd.Series(gold).rolling(20).var(ddof=1).to_numpy(), equal_nan=True)
|
||||
|
||||
|
||||
@test("GPU backend: precompute_indicators GPU == CPU on every IndicatorCache field; arrays are host NumPy")
|
||||
def _():
|
||||
df = make_xau_ohlcv(600, seed=36)
|
||||
cols = df.columns.get_indexer(["Open", "High", "Low", "Close"])
|
||||
df.iloc[250:262, cols] = float(df["Close"].iloc[249]) # flat stretch: zero TR / DM divisors
|
||||
tech = TechnicalConfig()
|
||||
with _gk36.BACKEND.cpu_only():
|
||||
c = _gb36.precompute_indicators(df, tech)
|
||||
_gb36._assert_host_cache(c)
|
||||
if not _GPU36:
|
||||
print(" (no CUDA: CPU twin only)")
|
||||
return
|
||||
g = _gb36.precompute_indicators(df, tech)
|
||||
_gb36._assert_host_cache(g)
|
||||
checked = 0
|
||||
for name in vars(g):
|
||||
if name in ("df", "index_pos"):
|
||||
continue
|
||||
a, b = getattr(g, name), getattr(c, name)
|
||||
if a is None or b is None:
|
||||
assert a is None and b is None, name
|
||||
continue
|
||||
if isinstance(a, pd.Series):
|
||||
assert a.index.equals(b.index), name
|
||||
a, b = a.to_numpy(), b.to_numpy()
|
||||
assert type(a) is np.ndarray and a.dtype == np.float64, name
|
||||
assert np.array_equal(a, b, equal_nan=True), (
|
||||
f"{name}: {int(np.sum(~((a == b) | (np.isnan(a) & np.isnan(b)))))} values differ")
|
||||
checked += 1
|
||||
assert checked >= 24, checked
|
||||
assert g.index_pos == c.index_pos
|
||||
# the short-frame branch (no EMA200) is identical too
|
||||
short = make_xau_ohlcv(150, seed=5)
|
||||
gs = _gb36.precompute_indicators(short, tech)
|
||||
with _gk36.BACKEND.cpu_only():
|
||||
cs = _gb36.precompute_indicators(short, tech)
|
||||
assert gs.ema_trend is None and cs.ema_trend is None
|
||||
assert np.array_equal(gs.adx, cs.adx, equal_nan=True)
|
||||
|
||||
|
||||
def _sim_fixture36():
|
||||
rng = np.random.default_rng(38)
|
||||
n = 1500
|
||||
idx = pd.date_range("2025-01-06", periods=n, freq="15min", tz="UTC")
|
||||
close = 3000 + np.cumsum(rng.normal(0, 2.0, n))
|
||||
op = close + rng.normal(0, 0.5, n)
|
||||
hi = np.maximum(op, close) + np.abs(rng.normal(0, 1.5, n))
|
||||
lo = np.minimum(op, close) - np.abs(rng.normal(0, 1.5, n))
|
||||
for j in (500,): # a NaN bar
|
||||
op[j] = hi[j] = lo[j] = close[j] = np.nan
|
||||
op[900] = close[899] - 25.0; lo[900] = min(lo[900], op[900]) # gap through a BUY stop
|
||||
op[950] = close[949] + 25.0; hi[950] = max(hi[950], op[950]) # gap through a SELL stop / beyond a BUY target
|
||||
close[n - 1] = np.nan # NaN close on a final window bar
|
||||
sim = pd.DataFrame({"Open": op, "High": hi, "Low": lo, "Close": close}, index=idx)
|
||||
|
||||
def mk(i, buy, arm=0.0, dist=0.0, risk=6.0, tp_dist=9.0):
|
||||
e = float(op[i]) if not np.isnan(op[i]) else float(close[i - 1])
|
||||
sl = e - risk if buy else e + risk
|
||||
tp = e + tp_dist if buy else e - tp_dist
|
||||
return _gb36.Trade(pair=XAUUSD, entry_time=idx[i], entry_price=e, sl=sl, tp1=tp,
|
||||
tp2=tp + 3, direction="BUY" if buy else "SELL", pip_value=0.1,
|
||||
risk_pips=60, reward_pips=90, rr_planned=1.5, quality="A",
|
||||
final_score=40, final_confidence=60, market_regime="trending_up",
|
||||
trail_arm_r=arm, trail_dist_r=dist)
|
||||
|
||||
trades = [mk(int(i), bool(rng.integers(0, 2)), *((0.7, 0.85) if k % 5 == 0 else (0.0, 0.0)))
|
||||
for k, i in enumerate(rng.integers(1, n - 1, 300))]
|
||||
trades += [mk(n - 1, True), mk(n - 1, False)] # window of one NaN-close bar
|
||||
trades += [mk(n - 3, True, risk=500.0, tp_dist=500.0),
|
||||
mk(n - 3, False, risk=500.0, tp_dist=500.0)] # TIMEOUT onto the NaN close -> NO_DATA
|
||||
trades += [mk(n - 400, True, risk=500.0, tp_dist=500.0),
|
||||
mk(n - 400, False, risk=500.0, tp_dist=500.0)] # a clean TIMEOUT at max_bars
|
||||
trades += [mk(500, True), mk(500, False)] # NaN bar first (NO_DATA)
|
||||
trades += [mk(499, True), mk(499, False)] # NaN bar second
|
||||
trades += [mk(899, True), mk(949, False), mk(949, True)] # gap fills
|
||||
trades += [mk(200, True, risk=0.0), mk(200, False, risk=0.0)] # zero risk
|
||||
return sim, trades
|
||||
|
||||
|
||||
@test("GPU backend: simulate_trades GPU == CPU (trailing and static; NaN windows, gaps, zero risk, per-trade trail)")
|
||||
def _():
|
||||
if not _GPU36:
|
||||
print(" (no CUDA: kernel not exercised)")
|
||||
return
|
||||
sim, trades = _sim_fixture36()
|
||||
fields = ("exit_reason", "exit_price", "exit_time", "bars_held", "r_multiple", "mfe_r", "mae_r")
|
||||
for arm in (0.6, 0.0):
|
||||
a = copy.deepcopy(trades)
|
||||
b = copy.deepcopy(trades)
|
||||
_gb36.simulate_trades(a, sim, max_bars=300, spread=0.65, trail_arm_r=arm, trail_dist_r=0.75)
|
||||
with _gk36.BACKEND.cpu_only():
|
||||
_gb36.simulate_trades(b, sim, max_bars=300, spread=0.65, trail_arm_r=arm, trail_dist_r=0.75)
|
||||
for i, (x, y) in enumerate(zip(a, b)):
|
||||
for f in fields:
|
||||
assert _same36(getattr(x, f), getattr(y, f)), (
|
||||
f"arm={arm} trade {i} {f}: gpu={getattr(x, f)!r} cpu={getattr(y, f)!r}")
|
||||
reasons = {t.exit_reason for t in a}
|
||||
assert "NO_DATA" in reasons and "TIMEOUT" in reasons, reasons
|
||||
assert ("TRAIL" in reasons) == (arm > 0)
|
||||
assert ("TP1" in reasons) == (arm == 0)
|
||||
# the NaN-close TIMEOUT never leaks a NaN R into the book
|
||||
assert all(np.isfinite(t.r_multiple) for t in a)
|
||||
# empty and no-data-after-entry cases return without touching the kernel
|
||||
_gb36.simulate_trades([], sim, max_bars=300)
|
||||
|
||||
|
||||
@test("GPU backend: --backend parses, FB2_BACKEND is honoured, cpu_only() routes to the NumPy twins, gpu fails loudly")
|
||||
def _():
|
||||
assert _gb36._parse_args(["--pair", "XAUUSD", "--backend", "cpu"]).backend == "cpu"
|
||||
assert _gb36._parse_args(["--pair", "XAUUSD"]).backend is None
|
||||
try:
|
||||
_gb36._parse_args(["--pair", "XAUUSD", "--backend", "tpu"])
|
||||
assert False, "invalid backend must be rejected"
|
||||
except SystemExit:
|
||||
pass
|
||||
with patch.dict(os.environ, {_gk36.ENV_BACKEND: "cpu"}):
|
||||
b = _gk36.Backend()
|
||||
assert b.mode == "cpu" and b.available is False
|
||||
assert b.describe().startswith("NumPy CPU fallback (forced")
|
||||
with patch.dict(os.environ, {_gk36.ENV_BACKEND: "nonsense"}):
|
||||
assert _gk36.Backend().mode == "auto"
|
||||
# cpu_only(): the GPU twin must not be called
|
||||
with _gk36.BACKEND.cpu_only():
|
||||
assert _gk36.BACKEND.available is False
|
||||
with patch.object(_gb36, "_precompute_indicators_gpu",
|
||||
side_effect=AssertionError("GPU twin called under cpu_only")):
|
||||
_gb36.precompute_indicators(make_xau_ohlcv(120, seed=1), TechnicalConfig())
|
||||
with patch.object(_gb36, "_resolve_trades_gpu",
|
||||
side_effect=AssertionError("GPU resolver called under cpu_only")):
|
||||
sim, trades = _sim_fixture36()
|
||||
_gb36.simulate_trades(trades[:5], sim, max_bars=50, trail_arm_r=0.6)
|
||||
assert _gk36.BACKEND.describe().startswith(("CuPy GPU (", "NumPy CPU fallback ("))
|
||||
# --backend gpu without CUDA raises instead of falling back
|
||||
b3 = _gk36.Backend()
|
||||
b3._probed, b3._ok, b3.reason = True, False, "synthetic probe failure"
|
||||
try:
|
||||
b3.set_mode("gpu")
|
||||
assert False, "must raise"
|
||||
except RuntimeError as e:
|
||||
assert "synthetic probe failure" in str(e)
|
||||
assert b3.describe() == "NumPy CPU fallback (synthetic probe failure)"
|
||||
try:
|
||||
b3.set_mode("tpu")
|
||||
assert False
|
||||
except ValueError:
|
||||
pass
|
||||
# release_device_memory is a no-op without CUDA
|
||||
b3.release_device_memory()
|
||||
|
||||
|
||||
@test("GPU backend: gpu_kernels imports none of the live writers; cupy never enters the live roots or the backtester")
|
||||
def _():
|
||||
banned = {"signal_log", "news_engine", "providers", "telegram_bot", "news_archive",
|
||||
"agent_panel", "violent_watch", "gpu_backtest", "engine", "signal_aggregator",
|
||||
"replay_record", "sklearn", "scipy", "lightgbm", "torch"}
|
||||
kpath = Path("backtest/gpu_kernels.py")
|
||||
seen = set()
|
||||
for node in ast.walk(ast.parse(kpath.read_text(encoding="utf-8"), str(kpath))):
|
||||
if isinstance(node, ast.Import):
|
||||
seen.update(a.name.split(".")[0] for a in node.names)
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
seen.add((node.module or "").split(".")[0])
|
||||
assert not (seen & banned), seen & banned
|
||||
assert "numpy" in seen and "cupy" in seen # cupy only inside the probe / kernels
|
||||
ksrc = kpath.read_text(encoding="utf-8")
|
||||
assert 'KERNEL_OPTIONS = ("--fmad=false",)' in ksrc # no FMA contraction, ever
|
||||
assert "options=KERNEL_OPTIONS" in ksrc
|
||||
# the backtester never imports cupy itself; it reads backend state through gpu_kernels
|
||||
bsrc = Path("backtest/gpu_backtest.py").read_text(encoding="utf-8")
|
||||
assert "import cupy" not in bsrc and "_GPU_AVAILABLE" not in bsrc
|
||||
assert "from backtest import gpu_kernels" in bsrc
|
||||
assert "BACKEND.release_device_memory()" in bsrc # before the fork
|
||||
assert bsrc.index("BACKEND.release_device_memory()") < bsrc.index('mp.get_context("fork")')
|
||||
# live roots stay free of cupy (Module 32's guard bans it too)
|
||||
offenders = []
|
||||
for root in (Path("."), Path("providers"), Path("pair_signals")):
|
||||
for path in sorted(root.glob("*.py")):
|
||||
if "venv" in path.parts:
|
||||
continue
|
||||
for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"), str(path))):
|
||||
names = ([a.name for a in node.names] if isinstance(node, ast.Import)
|
||||
else [node.module or ""] if isinstance(node, ast.ImportFrom) else [])
|
||||
offenders += [f"{path}:{node.lineno}" for nm in names if nm.split(".")[0] == "cupy"]
|
||||
assert not offenders, offenders
|
||||
assert "cupy" not in open("requirements.txt").read()
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# SUMMARY
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
Reference in New Issue
Block a user