Compare commits
4 Commits
59f69400a8
...
761f33888f
| Author | SHA1 | Date | |
|---|---|---|---|
 Harwinder Sekhon
|
761f33888f | ||
|
Harwinder Sekhon
|
e08133e2b5 | ||
|
Harwinder Sekhon
|
97f6a0e206 | ||
|
Harwinder Sekhon
|
cf2ad73343 |
@@ -37,3 +37,5 @@ backtest/__pycache__/
|
||||
logs/
|
||||
backtest/research/commit_ladder_2026-08/
|
||||
backtest/research/llm_news_eval_2026-08/
|
||||
|
||||
backtest/research/sizing_2026-09/
|
||||
|
||||
@@ -1152,6 +1152,101 @@ def export_csv(events: Sequence[SignalEvent], path: str) -> None:
|
||||
print(f"\nWrote {len(recs)} events to {path}")
|
||||
|
||||
|
||||
# ── Scan outcomes (model-free forward returns; no bar data needed) ────
|
||||
|
||||
|
||||
def scan_outcome_rows(db_path: str) -> List[dict]:
|
||||
"""One labelled scan per (timeframe, bar_ts) — the LOWEST signal id.
|
||||
|
||||
Several surfaces log the same bar, so the raw table is duplicated;
|
||||
SQLite takes the bare columns from the row min() selected."""
|
||||
con = open_readonly(db_path)
|
||||
try:
|
||||
if not con.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table'"
|
||||
" AND name='scan_outcomes'").fetchone():
|
||||
return []
|
||||
return [
|
||||
{"id": r[0], "timeframe": r[1], "signal": r[3] or "",
|
||||
"quality": r[4] or "-", "watch_level": r[5],
|
||||
"fwd_4h_atr": r[6], "burst_1h": r[7], "complete": r[8]}
|
||||
for r in con.execute(
|
||||
"SELECT MIN(o.signal_id), o.timeframe, o.bar_ts, s.signal,"
|
||||
" s.quality, s.watch_level, o.fwd_4h_atr, o.burst_1h,"
|
||||
" o.complete FROM scan_outcomes o"
|
||||
" JOIN signals s ON s.id = o.signal_id"
|
||||
" GROUP BY o.timeframe, o.bar_ts")
|
||||
]
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def _scan_stats(rows: Sequence[dict], directional: bool) -> Optional[dict]:
|
||||
"""n / mean / median forward move and the hit rate for one bucket.
|
||||
|
||||
'Hit' = the 4 h move agreed with the signal; for the NEUTRAL baseline
|
||||
it degrades to the share of UP moves."""
|
||||
vals = sorted(r["fwd_4h_atr"] for r in rows if r["fwd_4h_atr"] is not None)
|
||||
if not vals:
|
||||
return None
|
||||
hits = 0
|
||||
for row in rows:
|
||||
move = row["fwd_4h_atr"]
|
||||
if move is None:
|
||||
continue
|
||||
sign = -1 if (directional and "SELL" in row["signal"]) else 1
|
||||
hits += int(sign * move > 0)
|
||||
mid = len(vals) // 2
|
||||
return {"n": len(vals), "mean": sum(vals) / len(vals),
|
||||
"median": vals[mid] if len(vals) % 2 else
|
||||
(vals[mid - 1] + vals[mid]) / 2,
|
||||
"hit_rate": hits / len(vals)}
|
||||
|
||||
|
||||
def print_scan_outcomes(rows: Sequence[dict]) -> None:
|
||||
"""Forward-move report: no replay, no model — just what happened."""
|
||||
if not rows:
|
||||
print("No scan_outcomes rows yet "
|
||||
"(python scripts/label_scan_outcomes.py)")
|
||||
return
|
||||
complete = [r for r in rows if r["complete"]]
|
||||
print(f"\n{'═' * 66}\n SCAN OUTCOMES — 4 h forward move in ATRs "
|
||||
f"({len(complete)} complete of {len(rows)} labelled bars)")
|
||||
print(f"{'═' * 66}")
|
||||
directional = [r for r in complete if r["signal"] not in ("", "NEUTRAL")]
|
||||
neutral = [r for r in complete if r["signal"] == "NEUTRAL"]
|
||||
for title, groups, keyfn, is_dir in (
|
||||
("by timeframe x signal", directional,
|
||||
lambda r: (r["timeframe"], r["signal"]), True),
|
||||
("by timeframe x grade", directional,
|
||||
lambda r: (r["timeframe"], r["quality"]), True),
|
||||
("NEUTRAL baseline", neutral,
|
||||
lambda r: (r["timeframe"], "NEUTRAL"), False)):
|
||||
print(f"\n── {title} " + "─" * (52 - len(title)))
|
||||
buckets: Dict[Tuple[str, str], List[dict]] = {}
|
||||
for row in groups:
|
||||
buckets.setdefault(keyfn(row), []).append(row)
|
||||
for key in sorted(buckets):
|
||||
stat = _scan_stats(buckets[key], is_dir)
|
||||
if stat is None:
|
||||
continue
|
||||
print(f" {key[0]:6} {key[1]:12} n={stat['n']:5d} "
|
||||
f"mean {stat['mean']:+.3f} med {stat['median']:+.3f} "
|
||||
f"{'hit' if is_dir else 'up '} "
|
||||
f"{100 * stat['hit_rate']:.0f}%")
|
||||
print("\n── burst (>=3x ATR bar) within 1 h, by watch level " + "─" * 15)
|
||||
levels: Dict[str, List[int]] = {}
|
||||
for row in complete:
|
||||
if row["burst_1h"] is None:
|
||||
continue
|
||||
levels.setdefault(row["watch_level"] or "NULL", []).append(
|
||||
int(row["burst_1h"]))
|
||||
for level in sorted(levels):
|
||||
hits = levels[level]
|
||||
print(f" {level:6} n={len(hits):5d} burst "
|
||||
f"{100 * sum(hits) / len(hits):.1f}%")
|
||||
|
||||
|
||||
# ── main ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -1172,8 +1267,15 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
|
||||
ap.add_argument("--news", action="store_true",
|
||||
help="news-sentiment lead/lag + outcome buckets "
|
||||
"(requires --stored-candles)")
|
||||
ap.add_argument("--scan-outcomes", action="store_true",
|
||||
help="labelled forward moves per scan (no bar data "
|
||||
"needed); see scripts/label_scan_outcomes.py")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
if args.scan_outcomes:
|
||||
print_scan_outcomes(scan_outcome_rows(args.db))
|
||||
return 0
|
||||
|
||||
if args.news and not args.stored_candles:
|
||||
print("--news requires --stored-candles")
|
||||
return 2
|
||||
|
||||
+45
-1
@@ -87,7 +87,7 @@ _SCANNABLE_TFS = ("M5", "M15", "H1", "H4")
|
||||
|
||||
_KNOWN_TOP = {"telegram", "data_providers", "scanning", "pairs", "log_level",
|
||||
"oanda", "deepseek", "agent_panel", "market_outlook",
|
||||
"signal_log"}
|
||||
"violent_watch", "signal_log"}
|
||||
_KNOWN_OANDA = {
|
||||
"account_id", "api_token", "base_url",
|
||||
}
|
||||
@@ -101,6 +101,8 @@ _KNOWN_AGENT_PANEL = {
|
||||
"verdict_source",
|
||||
}
|
||||
_KNOWN_MARKET_OUTLOOK = {"enabled", "telegram", "max_per_day", "candles_bars"}
|
||||
_KNOWN_VIOLENT_WATCH = {"enabled", "alert_line", "headsup",
|
||||
"max_headsup_per_day", "headsup_min_level"}
|
||||
_KNOWN_SIGNAL_LOG = {"enabled", "path", "log_blocked_scans"}
|
||||
_KNOWN_TELEGRAM = {"token", "chat_id", "channel_id", "alert_min_quality"}
|
||||
_KNOWN_SCANNING = {"pairs", "timeframes"}
|
||||
@@ -113,6 +115,16 @@ _MARKET_OUTLOOK_DEFAULTS = {
|
||||
"candles_bars": 20,
|
||||
}
|
||||
|
||||
# Violent-move watch (display-only heads-up) — dark by default.
|
||||
_VIOLENT_WATCH_DEFAULTS = {
|
||||
"enabled": False,
|
||||
"alert_line": True,
|
||||
"headsup": True,
|
||||
"max_headsup_per_day": 12,
|
||||
"headsup_min_level": "hot",
|
||||
}
|
||||
_VALID_WATCH_LEVELS = ("watch", "hot")
|
||||
|
||||
|
||||
class AppConfigError(Exception):
|
||||
"""config.json is present but invalid — fail fast at boot."""
|
||||
@@ -136,6 +148,8 @@ class AppConfig:
|
||||
agent_panel_settings: Dict[str, Any] = field(default_factory=dict)
|
||||
market_outlook_settings: Dict[str, Any] = field(
|
||||
default_factory=lambda: dict(_MARKET_OUTLOOK_DEFAULTS))
|
||||
violent_watch_settings: Dict[str, Any] = field(
|
||||
default_factory=lambda: dict(_VIOLENT_WATCH_DEFAULTS))
|
||||
signal_log_settings: Dict[str, Any] = field(default_factory=dict)
|
||||
log_level: str = "INFO"
|
||||
|
||||
@@ -337,6 +351,25 @@ def load_app_config(path: str = CONFIG_PATH) -> AppConfig:
|
||||
f"config.json: market_outlook.{key} must be a positive integer")
|
||||
app.market_outlook_settings.update(outlook)
|
||||
|
||||
watch = raw.get("violent_watch", {})
|
||||
_check_keys(watch, _KNOWN_VIOLENT_WATCH, '"violent_watch"')
|
||||
for flag in ("enabled", "alert_line", "headsup"):
|
||||
if flag in watch and not isinstance(watch[flag], bool):
|
||||
raise AppConfigError(
|
||||
f"config.json: violent_watch.{flag} must be true or false")
|
||||
if "max_headsup_per_day" in watch:
|
||||
cap = watch["max_headsup_per_day"]
|
||||
if isinstance(cap, bool) or not isinstance(cap, int) or cap < 0:
|
||||
raise AppConfigError(
|
||||
"config.json: violent_watch.max_headsup_per_day must be a "
|
||||
"non-negative integer")
|
||||
if ("headsup_min_level" in watch
|
||||
and watch["headsup_min_level"] not in _VALID_WATCH_LEVELS):
|
||||
raise AppConfigError(
|
||||
f'config.json: violent_watch.headsup_min_level '
|
||||
f'"{watch["headsup_min_level"]}" not in {_VALID_WATCH_LEVELS}')
|
||||
app.violent_watch_settings.update(watch)
|
||||
|
||||
slog = raw.get("signal_log", {})
|
||||
_check_keys(slog, _KNOWN_SIGNAL_LOG, '"signal_log"')
|
||||
if "enabled" in slog and not isinstance(slog["enabled"], bool):
|
||||
@@ -462,6 +495,17 @@ def market_outlook_enabled(app: Optional[AppConfig] = None) -> bool:
|
||||
return bool(app.market_outlook_settings.get("enabled", False))
|
||||
|
||||
|
||||
# ── Violent-move watch resolution helper ─────────────────────────────
|
||||
|
||||
def violent_watch_enabled(app: Optional[AppConfig] = None) -> bool:
|
||||
"""Whether the display-only violent-move watch is on (env wins)."""
|
||||
env = os.environ.get("VIOLENT_WATCH_ENABLED")
|
||||
if env:
|
||||
return env.strip().lower() in ("1", "true", "yes", "on")
|
||||
app = app or get_app_config()
|
||||
return bool(app.violent_watch_settings.get("enabled", False))
|
||||
|
||||
|
||||
# ── Signal-log resolution helpers (env → config.json → default) ──────
|
||||
|
||||
def signal_log_enabled(app: Optional[AppConfig] = None) -> bool:
|
||||
|
||||
@@ -62,6 +62,7 @@ from config import ( # noqa: E402
|
||||
from providers import BaseDataProvider, make_data_provider # noqa: E402
|
||||
from engine import IndicatorSignal, TechnicalAnalysis, TechnicalEngine # noqa: E402
|
||||
from signal_aggregator import SignalAggregator, SignalReport, TradeSetup # noqa: E402
|
||||
import position_sizing # noqa: E402
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────
|
||||
# GPU shim — CuPy if available AND CUDA runtime works, else NumPy.
|
||||
@@ -1074,6 +1075,13 @@ class Trade:
|
||||
indicators_neutral: int = 0
|
||||
indicators_total: int = 0 # total scored indicators
|
||||
|
||||
# RESEARCH (--export-features only): decision-time feature vector
|
||||
# (position_sizing.FEATURE_ORDER) and the stake multiplier the setup
|
||||
# carried. Empty / 1.0 on every normal run, so the default book is
|
||||
# untouched.
|
||||
features: Dict[str, float] = field(default_factory=dict)
|
||||
stake_multiplier: float = 1.0
|
||||
|
||||
# Filled on exit:
|
||||
exit_time: Optional[pd.Timestamp] = None
|
||||
exit_price: Optional[float] = None
|
||||
@@ -1454,6 +1462,11 @@ class BacktestConfig:
|
||||
# afterwards in a fast ordered pass. Results are identical to the
|
||||
# sequential path by construction.
|
||||
workers: int = 0
|
||||
# RESEARCH: also compute position_sizing.build_features() at each
|
||||
# entry and append them to --export-csv. Off by default — the extra
|
||||
# columns would break comparison against the pinned 63-column books,
|
||||
# and the feature build costs time per candidate.
|
||||
export_features: bool = False
|
||||
|
||||
|
||||
def _is_xau_blocked_by_calendar(ts: pd.Timestamp, cfg: PlatformConfig) -> Optional[str]:
|
||||
@@ -1610,6 +1623,17 @@ def _replay_span(span: Tuple[int, int]):
|
||||
adj_tp1 = entry_price - tp1_distance
|
||||
adj_tp2 = entry_price - tp2_distance
|
||||
|
||||
# RESEARCH: decision-time features (same frames the aggregator
|
||||
# just saw, so live == backtest by construction). Deliberately
|
||||
# NOT wrapped in try/except — build_features already swallows its
|
||||
# own failures, so anything escaping here is a real bug.
|
||||
feats: Dict[str, float] = {}
|
||||
if bt_cfg.export_features:
|
||||
feats = position_sizing.build_features(
|
||||
report, ts_setup, slice_dfs, bt_cfg.primary_tf,
|
||||
list(bt_cfg.confirmation_tfs),
|
||||
)
|
||||
|
||||
candidates.append((pos, Trade(
|
||||
pair=bt_cfg.pair,
|
||||
entry_time=entry_time,
|
||||
@@ -1635,6 +1659,10 @@ def _replay_span(span: Tuple[int, int]):
|
||||
indicators_opposing=oppose,
|
||||
indicators_neutral=neutral,
|
||||
indicators_total=total,
|
||||
features=feats,
|
||||
stake_multiplier=float(ts_setup.stake_multiplier
|
||||
if ts_setup.stake_multiplier is not None
|
||||
else 1.0),
|
||||
trail_arm_r=(platform_cfg.signals.pb_trail_arm_r
|
||||
if tag == "XAU_PB" else 0.0),
|
||||
trail_dist_r=(platform_cfg.signals.pb_trail_dist_r
|
||||
@@ -2157,11 +2185,17 @@ def print_trade_reasoning(trades: List[Trade], limit: int = 5) -> None:
|
||||
print(f" MFE / MAE: {t.mfe_r:.2f}R / {t.mae_r:.2f}R")
|
||||
|
||||
|
||||
def trades_to_dataframe(trades: List[Trade]) -> pd.DataFrame:
|
||||
def trades_to_dataframe(trades: List[Trade],
|
||||
include_features: bool = False) -> pd.DataFrame:
|
||||
"""Export trades for downstream analysis (e.g. parquet/CSV).
|
||||
|
||||
Includes per-instructions: entry reasoning, indicator-by-indicator
|
||||
signal/value/agreement, and overall agreement count.
|
||||
|
||||
`include_features` (research, --export-features) appends
|
||||
stake_multiplier, r_sized and the position_sizing.FEATURE_ORDER
|
||||
columns AFTER the existing ones. Default off so the column set stays
|
||||
byte-identical to the pinned books.
|
||||
"""
|
||||
rows = []
|
||||
for t in trades:
|
||||
@@ -2200,8 +2234,22 @@ def trades_to_dataframe(trades: List[Trade]) -> pd.DataFrame:
|
||||
row[f"ind_{name}_value"] = b["value"]
|
||||
row[f"ind_{name}_conf"] = b["confidence"]
|
||||
row[f"ind_{name}_agrees"] = b["agrees"]
|
||||
if include_features:
|
||||
row["stake_multiplier"] = t.stake_multiplier
|
||||
row["r_sized"] = t.r_multiple * t.stake_multiplier
|
||||
for name in position_sizing.FEATURE_ORDER:
|
||||
row[name] = t.features.get(name, float("nan"))
|
||||
rows.append(row)
|
||||
return pd.DataFrame(rows)
|
||||
df = pd.DataFrame(rows)
|
||||
if include_features and len(df.columns):
|
||||
# A later trade can introduce an indicator column the first trade
|
||||
# lacked (pandas orders columns by first appearance), which would
|
||||
# land AFTER the extras. Force the extras to the tail so the default
|
||||
# export is always a strict prefix of the feature export.
|
||||
extras = ["stake_multiplier", "r_sized", *position_sizing.FEATURE_ORDER]
|
||||
base_cols = [c for c in df.columns if c not in extras]
|
||||
df = df[base_cols + [c for c in extras if c in df.columns]]
|
||||
return df
|
||||
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════
|
||||
@@ -2263,6 +2311,11 @@ def _parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace:
|
||||
help="Round-trip cost in price units per trade (default: pair's "
|
||||
"default_spread from PAIR_INFO — XAU $0.30; 0 disables)")
|
||||
p.add_argument("--export-csv", help="Optional path to dump per-trade CSV (incl. indicator breakdown)")
|
||||
p.add_argument("--export-features", action="store_true",
|
||||
help="Append the 65 decision-time f_* feature columns "
|
||||
"(+ stake_multiplier, r_sized) to --export-csv; "
|
||||
"research flag, default off so pinned 63-column "
|
||||
"books stay comparable")
|
||||
p.add_argument("--detail", action="store_true",
|
||||
help="Print full per-trade reasoning (indicators, signals, agreement, why entered)")
|
||||
p.add_argument("--workers", type=int, default=0,
|
||||
@@ -2416,7 +2469,14 @@ def main(argv: Optional[List[str]] = None) -> int:
|
||||
spread=spread,
|
||||
workers=args.workers,
|
||||
fixed_rr_tp=args.fixed_rr_tp or 0.0,
|
||||
export_features=args.export_features,
|
||||
)
|
||||
if args.export_features:
|
||||
logger.info(
|
||||
f"--export-features ON: appending stake_multiplier, r_sized and "
|
||||
f"{len(position_sizing.FEATURE_ORDER)} f_* feature columns to the "
|
||||
f"CSV export (research flag; book itself is unchanged)"
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Backend: {'CuPy GPU' if _GPU_AVAILABLE else 'NumPy CPU fallback'} | "
|
||||
@@ -2430,7 +2490,7 @@ def main(argv: Optional[List[str]] = None) -> int:
|
||||
print_trade_reasoning(trades, limit=args.reasoning_samples)
|
||||
|
||||
if args.export_csv:
|
||||
df = trades_to_dataframe(trades)
|
||||
df = trades_to_dataframe(trades, include_features=args.export_features)
|
||||
df.to_csv(args.export_csv, index=False)
|
||||
logger.info(f"Exported {len(df)} trades → {args.export_csv}")
|
||||
|
||||
|
||||
@@ -57,3 +57,13 @@ Win rate over rolling 20-trade windows (check for edge decay)
|
||||
Critical extra
|
||||
"Do not trade" log: setups flagged but skipped, and what happened after
|
||||
|
||||
|
||||
--- 2026-09-06: decision-time feature export ---
|
||||
- `--export-features` appends stake_multiplier, r_sized and the 65 `f_*` decision-time
|
||||
features (position_sizing.FEATURE_ORDER) AFTER the 63 pinned columns; default export is
|
||||
byte-identical to the pinned books. Use backtest/research/book_report.py for weekday /
|
||||
hour x weekday / session / streak / rolling-WR / feature-quintile (both halves) tables,
|
||||
backtest/research/burst_map.py for the burst-by-hour map, and
|
||||
backtest/research/sizing_model.py for the walk-forward + shuffle-control harness.
|
||||
- Any sizing report must show the FLAT book and the SIZED book side by side and the
|
||||
label-shuffle null; the 2026-09-06 campaign verdict (nothing ships) is the reference.
|
||||
|
||||
@@ -0,0 +1,581 @@
|
||||
"""Full descriptive report of a backtest trade book (research only).
|
||||
|
||||
Usage:
|
||||
python backtest/research/book_report.py --book BOOK.csv --bar-minutes N \
|
||||
[--split YYYY-MM-DD|median] [--parquet-4h PATH] [--parquet-1d PATH] \
|
||||
[--out DIR]
|
||||
|
||||
Prints (and writes to DIR/report.txt) headline metrics for the whole book
|
||||
and for each half, group tables (weekday / decision hour / session / quality
|
||||
/ regime / vol_regime / direction / strategy / exit_reason / indicators_agree),
|
||||
win-loss streaks, a rolling win-rate summary, and a quantile-bucket screen of
|
||||
every numeric decision-time feature the book carries (`f_*` / `x_*` columns
|
||||
plus score, confidence, rr_planned, indicators_agree/oppose and `ind_*_value`).
|
||||
|
||||
The screen is deliberately half-split: a feature only "passes" when the
|
||||
top-bucket-minus-bottom-bucket mean-R gap has the SAME SIGN in both halves of
|
||||
the book and both extreme buckets are populated in both halves. Bin edges are
|
||||
fit on the WHOLE book (so the halves share edges and stay comparable) — that
|
||||
leaks a little cross-half information into the edges themselves, which is why
|
||||
the sign test, not the size of the gap, is the thing to trust.
|
||||
|
||||
Everything here is measurement, not simulation: it never re-runs the signal
|
||||
engine, it only describes a book that gpu_backtest.py already produced.
|
||||
Analysis tooling only — never imported by the live code.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import math
|
||||
import os
|
||||
from typing import Iterable
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
SESSIONS = (
|
||||
("asian", 3, 6),
|
||||
("london", 7, 12),
|
||||
("overlap", 13, 16),
|
||||
("ny", 17, 22),
|
||||
)
|
||||
|
||||
# Numeric non-`f_`/`x_` columns that are known at DECISION time and are
|
||||
# therefore legal screen candidates. bars_held / r_multiple / mfe_r / mae_r
|
||||
# are post-entry outcomes and must never appear here.
|
||||
BASE_SCREEN_COLS = ("score", "confidence", "rr_planned",
|
||||
"indicators_agree", "indicators_oppose")
|
||||
|
||||
|
||||
def session_of(hour: int) -> str:
|
||||
for name, lo, hi in SESSIONS:
|
||||
if lo <= hour <= hi:
|
||||
return name
|
||||
return "off"
|
||||
|
||||
|
||||
def feature_columns(df: pd.DataFrame) -> list[str]:
|
||||
"""Decision-time feature columns carried by this book, generically."""
|
||||
extra = [c for c in df.columns if c.startswith(("f_", "x_"))]
|
||||
base = [c for c in BASE_SCREEN_COLS if c in df.columns]
|
||||
ind = [c for c in df.columns
|
||||
if c.startswith("ind_") and c.endswith("_value")]
|
||||
out, seen = [], set()
|
||||
for c in extra + base + ind:
|
||||
if c not in seen:
|
||||
seen.add(c)
|
||||
out.append(c)
|
||||
return out
|
||||
|
||||
|
||||
# ───────────────────────────── loading ──────────────────────────────
|
||||
|
||||
def load_book(csv: str, bar_minutes: int = 5,
|
||||
split: str = "median") -> pd.DataFrame:
|
||||
"""Read a gpu_backtest trade book and add the decision-time columns.
|
||||
|
||||
`decision_time` = entry_time − one bar: entries fill at the NEXT bar's
|
||||
open, so the bar the signal actually decided on closed one bar earlier.
|
||||
That is the clock blocked_hours_utc and every session filter evaluate.
|
||||
`half` is 1 before the split instant and 2 at/after it.
|
||||
"""
|
||||
df = pd.read_csv(csv)
|
||||
for col in ("entry_time", "exit_time"):
|
||||
if col in df.columns:
|
||||
df[col] = pd.to_datetime(df[col], utc=True, format="mixed")
|
||||
df["decision_time"] = df["entry_time"] - pd.Timedelta(minutes=bar_minutes)
|
||||
df["decision_hour"] = df["decision_time"].dt.hour
|
||||
df["weekday"] = df["decision_time"].dt.weekday
|
||||
df["session"] = df["decision_hour"].map(session_of)
|
||||
df = df.sort_values("entry_time", kind="mergesort").reset_index(drop=True)
|
||||
df["half"] = _halves(df["entry_time"], split)
|
||||
return df
|
||||
|
||||
|
||||
def split_instant(entry_time: pd.Series, split: str) -> pd.Timestamp:
|
||||
if split == "median":
|
||||
return entry_time.median()
|
||||
return pd.Timestamp(split, tz="UTC")
|
||||
|
||||
|
||||
def _halves(entry_time: pd.Series, split: str) -> pd.Series:
|
||||
cut = split_instant(entry_time, split)
|
||||
return np.where(entry_time < cut, 1, 2)
|
||||
|
||||
|
||||
# ───────────────────────────── metrics ──────────────────────────────
|
||||
|
||||
def base_metrics(r: pd.Series, df: pd.DataFrame | None = None) -> dict:
|
||||
"""Headline stats for a series of R multiples.
|
||||
|
||||
max_dd uses the gpu_backtest.compute_metrics convention: a 0.0 is
|
||||
prepended to the cumulative-R curve so an opening losing streak counts
|
||||
as drawdown from flat, not from an already-negative first trade.
|
||||
pf/expectancy split on r > 0 / r < 0 (exact zeros are neither).
|
||||
"""
|
||||
r = pd.Series(r, dtype="float64").dropna()
|
||||
n = int(len(r))
|
||||
if n == 0:
|
||||
return {"n": 0, "total_r": 0.0, "mean_r": 0.0, "median_r": 0.0,
|
||||
"wr": 0.0, "pf": 0.0, "max_dd": 0.0, "sharpe": 0.0,
|
||||
"expectancy": 0.0, "expectancy_alt": 0.0, "recovery": 0.0,
|
||||
"avg_bars": float("nan"), "avg_mfe": float("nan"),
|
||||
"avg_mae": float("nan")}
|
||||
wins, losses = r[r > 0], r[r < 0]
|
||||
gross_win = float(wins.sum())
|
||||
gross_loss = float(-losses.sum())
|
||||
cum = np.concatenate(([0.0], np.cumsum(r.to_numpy())))
|
||||
max_dd = float((np.maximum.accumulate(cum) - cum).max())
|
||||
std = float(r.std(ddof=0))
|
||||
mean_r = float(r.mean())
|
||||
wr = len(wins) / n
|
||||
lr = len(losses) / n
|
||||
avg_win = float(wins.mean()) if len(wins) else 0.0
|
||||
avg_loss = float(-losses.mean()) if len(losses) else 0.0
|
||||
expectancy_alt = wr * avg_win - lr * avg_loss
|
||||
# Identity check: the textbook expectancy form must equal the plain mean
|
||||
# whenever no trade is exactly flat (and it does even then, since a 0 R
|
||||
# trade contributes 0 to both sides).
|
||||
assert math.isclose(expectancy_alt, mean_r, rel_tol=1e-9, abs_tol=1e-9), (
|
||||
f"expectancy mismatch: {expectancy_alt} vs mean {mean_r}")
|
||||
out = {
|
||||
"n": n,
|
||||
"total_r": float(r.sum()),
|
||||
"mean_r": mean_r,
|
||||
"median_r": float(r.median()),
|
||||
"wr": wr * 100.0,
|
||||
"pf": (gross_win / gross_loss) if gross_loss > 0 else float("inf"),
|
||||
"max_dd": max_dd,
|
||||
"sharpe": (mean_r / std) if std > 0 else 0.0,
|
||||
"expectancy": mean_r,
|
||||
"expectancy_alt": expectancy_alt,
|
||||
"recovery": (float(r.sum()) / max_dd) if max_dd > 0 else float("inf"),
|
||||
"avg_bars": float("nan"),
|
||||
"avg_mfe": float("nan"),
|
||||
"avg_mae": float("nan"),
|
||||
}
|
||||
if df is not None:
|
||||
for key, col in (("avg_bars", "bars_held"), ("avg_mfe", "mfe_r"),
|
||||
("avg_mae", "mae_r")):
|
||||
if col in df.columns:
|
||||
out[key] = float(pd.to_numeric(df[col],
|
||||
errors="coerce").mean())
|
||||
return out
|
||||
|
||||
|
||||
def _pf(r: pd.Series) -> float:
|
||||
"""Profit factor, hour_analysis-compatible (loss side is r <= 0)."""
|
||||
pos = r[r > 0].sum()
|
||||
neg = -r[r <= 0].sum()
|
||||
return float(pos / max(neg, 1e-9))
|
||||
|
||||
|
||||
def group_table(df: pd.DataFrame, by, min_n: int = 10) -> pd.DataFrame:
|
||||
"""Per-group P&L with a half-1 / half-2 sign-stability flag."""
|
||||
keys = [by] if isinstance(by, str) else list(by)
|
||||
rows = []
|
||||
for key, sub in df.groupby(keys, dropna=False, sort=True):
|
||||
r = sub["r_multiple"]
|
||||
h1, h2 = sub[sub["half"] == 1]["r_multiple"], sub[sub["half"] == 2]["r_multiple"]
|
||||
m1 = float(h1.mean()) if len(h1) else float("nan")
|
||||
m2 = float(h2.mean()) if len(h2) else float("nan")
|
||||
stable = bool(len(h1) >= min_n and len(h2) >= min_n
|
||||
and np.sign(m1) != 0 and np.sign(m1) == np.sign(m2))
|
||||
rec = {}
|
||||
# pandas >= 2.2 hands back a 1-tuple even for a single group key.
|
||||
vals = key if isinstance(key, tuple) else (key,)
|
||||
for k, v in zip(keys, vals):
|
||||
rec[k] = v
|
||||
rec.update({
|
||||
"n": int(len(sub)),
|
||||
"total_r": round(float(r.sum()), 2),
|
||||
"mean_r": round(float(r.mean()), 3),
|
||||
"wr": round(float((r > 0).mean() * 100), 2),
|
||||
"pf": round(_pf(r), 2),
|
||||
"n_h1": int(len(h1)),
|
||||
"total_r_h1": round(float(h1.sum()), 2),
|
||||
"mean_r_h1": round(m1, 3) if len(h1) else float("nan"),
|
||||
"n_h2": int(len(h2)),
|
||||
"total_r_h2": round(float(h2.sum()), 2),
|
||||
"mean_r_h2": round(m2, 3) if len(h2) else float("nan"),
|
||||
"sign_stable": stable,
|
||||
})
|
||||
rows.append(rec)
|
||||
out = pd.DataFrame(rows)
|
||||
if not out.empty:
|
||||
out = out.set_index(keys)
|
||||
return out
|
||||
|
||||
|
||||
def streaks(df: pd.DataFrame) -> dict:
|
||||
"""Longest win/loss runs and the longest stretch under water (in trades)."""
|
||||
d = df.sort_values("entry_time", kind="mergesort")
|
||||
r = d["r_multiple"].to_numpy(dtype="float64")
|
||||
best_w = best_l = cur_w = cur_l = 0
|
||||
for x in r:
|
||||
if x > 0:
|
||||
cur_w, cur_l = cur_w + 1, 0
|
||||
elif x < 0:
|
||||
cur_l, cur_w = cur_l + 1, 0
|
||||
else:
|
||||
cur_w = cur_l = 0
|
||||
best_w, best_l = max(best_w, cur_w), max(best_l, cur_l)
|
||||
cum = np.concatenate(([0.0], np.cumsum(r)))
|
||||
peak = np.maximum.accumulate(cum)
|
||||
flat, longest = 0, 0
|
||||
for i in range(1, len(cum)):
|
||||
flat = 0 if cum[i] >= peak[i] else flat + 1
|
||||
longest = max(longest, flat)
|
||||
return {"max_win_streak": best_w, "max_loss_streak": best_l,
|
||||
"longest_flat_trades": longest}
|
||||
|
||||
|
||||
def rolling_wr(df: pd.DataFrame, window: int = 20) -> pd.Series:
|
||||
d = df.sort_values("entry_time", kind="mergesort")
|
||||
s = (d["r_multiple"] > 0).astype(float).rolling(window).mean() * 100
|
||||
s.index = d["entry_time"].to_numpy()
|
||||
return s
|
||||
|
||||
|
||||
def equity_curve(df: pd.DataFrame) -> pd.DataFrame:
|
||||
d = df.sort_values("entry_time", kind="mergesort")
|
||||
cum = d["r_multiple"].cumsum()
|
||||
peak = pd.concat([pd.Series([0.0]), cum]).cummax().iloc[1:]
|
||||
peak.index = cum.index
|
||||
return pd.DataFrame({"entry_time": d["entry_time"].to_numpy(),
|
||||
"cum_r": cum.to_numpy(),
|
||||
"drawdown": (peak - cum).to_numpy()})
|
||||
|
||||
|
||||
# ─────────────────────── quantile bucket screen ─────────────────────
|
||||
|
||||
def quantile_buckets(df: pd.DataFrame, col: str, q: int = 5,
|
||||
min_n: int = 10) -> pd.DataFrame:
|
||||
"""Equal-frequency buckets of `col` with per-half mean R.
|
||||
|
||||
Bin edges are fit on the WHOLE book (both halves share edges).
|
||||
Result carries gap_h1 / gap_h2 / sign_stable in `.attrs` and as
|
||||
repeated columns so the CSV is self-describing.
|
||||
"""
|
||||
v = pd.to_numeric(df.get(col), errors="coerce")
|
||||
d = df.loc[v.notna()].copy()
|
||||
d["_v"] = v.loc[v.notna()]
|
||||
empty = pd.DataFrame(columns=["bin", "lo", "hi", "n", "mean_r", "pf", "wr",
|
||||
"n_h1", "mean_r_h1", "n_h2", "mean_r_h2",
|
||||
"gap_h1", "gap_h2", "sign_stable"])
|
||||
empty.attrs = {"feature": col, "gap_h1": float("nan"),
|
||||
"gap_h2": float("nan"), "sign_stable": False, "n": 0,
|
||||
"mode": "empty"}
|
||||
if d.empty:
|
||||
return empty
|
||||
|
||||
nuniq = int(d["_v"].nunique())
|
||||
if nuniq < 3:
|
||||
mode = "values"
|
||||
d["_bin"] = d["_v"]
|
||||
order = sorted(d["_v"].unique())
|
||||
edges = {b: (float(b), float(b)) for b in order}
|
||||
else:
|
||||
mode = "qcut"
|
||||
binned = pd.qcut(d["_v"], q, duplicates="drop")
|
||||
d["_bin"] = binned
|
||||
order = list(binned.cat.categories)
|
||||
edges = {b: (float(b.left), float(b.right)) for b in order}
|
||||
|
||||
rows = []
|
||||
for b in order:
|
||||
sub = d[d["_bin"] == b]
|
||||
if sub.empty:
|
||||
continue
|
||||
r = sub["r_multiple"]
|
||||
h1 = sub[sub["half"] == 1]["r_multiple"]
|
||||
h2 = sub[sub["half"] == 2]["r_multiple"]
|
||||
lo, hi = edges[b]
|
||||
rows.append({
|
||||
"bin": str(b), "lo": round(lo, 4), "hi": round(hi, 4),
|
||||
"n": int(len(sub)), "mean_r": round(float(r.mean()), 3),
|
||||
"pf": round(_pf(r), 2),
|
||||
"wr": round(float((r > 0).mean() * 100), 2),
|
||||
"n_h1": int(len(h1)),
|
||||
"mean_r_h1": round(float(h1.mean()), 3) if len(h1) else float("nan"),
|
||||
"n_h2": int(len(h2)),
|
||||
"mean_r_h2": round(float(h2.mean()), 3) if len(h2) else float("nan"),
|
||||
})
|
||||
out = pd.DataFrame(rows)
|
||||
if out.empty:
|
||||
return empty
|
||||
|
||||
bot, top = out.iloc[0], out.iloc[-1]
|
||||
gap_h1 = float(top["mean_r_h1"] - bot["mean_r_h1"])
|
||||
gap_h2 = float(top["mean_r_h2"] - bot["mean_r_h2"])
|
||||
populated = min(bot["n_h1"], bot["n_h2"], top["n_h1"], top["n_h2"]) >= min_n
|
||||
signs_ok = (not math.isnan(gap_h1) and not math.isnan(gap_h2)
|
||||
and np.sign(gap_h1) != 0
|
||||
and np.sign(gap_h1) == np.sign(gap_h2))
|
||||
sign_stable = bool(populated and signs_ok and len(out) >= 2)
|
||||
out["gap_h1"] = round(gap_h1, 3) if not math.isnan(gap_h1) else float("nan")
|
||||
out["gap_h2"] = round(gap_h2, 3) if not math.isnan(gap_h2) else float("nan")
|
||||
out["sign_stable"] = sign_stable
|
||||
out.attrs = {"feature": col, "gap_h1": gap_h1, "gap_h2": gap_h2,
|
||||
"sign_stable": sign_stable, "n": int(len(d)), "mode": mode}
|
||||
return out
|
||||
|
||||
|
||||
def feature_screen(df: pd.DataFrame, q: int = 5, min_n: int = 10,
|
||||
columns: Iterable[str] | None = None) -> pd.DataFrame:
|
||||
"""Bucket screen over every decision-time numeric feature in the book."""
|
||||
cols = list(columns) if columns is not None else feature_columns(df)
|
||||
rows = []
|
||||
for c in cols:
|
||||
bk = quantile_buckets(df, c, q=q, min_n=min_n)
|
||||
a = bk.attrs
|
||||
stable = bool(a.get("sign_stable"))
|
||||
g1, g2 = a.get("gap_h1", float("nan")), a.get("gap_h2", float("nan"))
|
||||
min_abs = (min(abs(g1), abs(g2)) if stable else 0.0)
|
||||
rows.append({"feature": c,
|
||||
"gap_h1": round(g1, 3) if g1 == g1 else float("nan"),
|
||||
"gap_h2": round(g2, 3) if g2 == g2 else float("nan"),
|
||||
"min_abs_gap": round(float(min_abs), 3),
|
||||
"sign_stable": stable,
|
||||
"n": int(a.get("n", 0)),
|
||||
"bins": int(len(bk))})
|
||||
out = pd.DataFrame(rows)
|
||||
if out.empty:
|
||||
return out
|
||||
return out.sort_values(["min_abs_gap", "n"],
|
||||
ascending=[False, False]).reset_index(drop=True)
|
||||
|
||||
|
||||
# ────────────────────── higher-timeframe extension ──────────────────
|
||||
|
||||
def wilder_atr(bars: pd.DataFrame, period: int = 14) -> pd.Series:
|
||||
h, l, c = bars["High"], bars["Low"], bars["Close"]
|
||||
tr = pd.concat([h - l, (h - c.shift()).abs(), (l - c.shift()).abs()],
|
||||
axis=1).max(axis=1)
|
||||
return tr.ewm(alpha=1 / period, min_periods=period, adjust=False).mean()
|
||||
|
||||
|
||||
def _ext_series(parquet: str, minutes: int) -> pd.DataFrame:
|
||||
bars = pd.read_parquet(parquet).sort_index()
|
||||
bars = bars[~bars.index.duplicated(keep="last")]
|
||||
ext = ((bars["Close"] - bars["Close"].rolling(20).mean())
|
||||
/ wilder_atr(bars, 14))
|
||||
out = pd.DataFrame({
|
||||
"close_time": bars.index + pd.Timedelta(minutes=minutes),
|
||||
"ext": ext.to_numpy(),
|
||||
}).dropna().reset_index(drop=True)
|
||||
# Books parse to datetime64[us, UTC] (format="mixed"), parquet indexes are
|
||||
# [ns, UTC]; merge_asof refuses mismatched resolutions.
|
||||
out["close_time"] = out["close_time"].astype("datetime64[ns, UTC]")
|
||||
return out
|
||||
|
||||
|
||||
def add_htf_extension(df: pd.DataFrame, parquet_4h: str | None,
|
||||
parquet_1d: str | None) -> pd.DataFrame:
|
||||
"""Add direction-signed H4 / D1 stretch-from-mean in ATR units.
|
||||
|
||||
ext = (close − 20-bar SMA of close) / Wilder ATR(14) on that timeframe,
|
||||
taken from the LAST higher-TF bar whose CLOSE time (bar label + one bar)
|
||||
is <= decision_time — i.e. only information the live scan could have had.
|
||||
Signed +1 for BUY, −1 for SELL, so positive always means "extended in the
|
||||
direction of the trade".
|
||||
"""
|
||||
out = df.copy()
|
||||
sign = np.where(out["direction"].astype(str).str.upper().str.startswith("B"),
|
||||
1.0, -1.0)
|
||||
for col, parquet, minutes in (("x_h4_ext_atr", parquet_4h, 240),
|
||||
("x_d1_ext_atr", parquet_1d, 1440)):
|
||||
if not parquet:
|
||||
continue
|
||||
right = _ext_series(parquet, minutes)
|
||||
left = out[["decision_time"]].reset_index()
|
||||
left["decision_time"] = left["decision_time"].astype(
|
||||
"datetime64[ns, UTC]")
|
||||
left = left.sort_values("decision_time", kind="mergesort")
|
||||
merged = pd.merge_asof(left, right, left_on="decision_time",
|
||||
right_on="close_time", direction="backward")
|
||||
merged["_stale_h"] = ((merged["decision_time"] - merged["close_time"])
|
||||
.dt.total_seconds() / 3600.0)
|
||||
merged = merged.set_index("index").reindex(out.index)
|
||||
out[col] = merged["ext"].to_numpy() * sign
|
||||
out.attrs[f"{col}_max_stale_h"] = float(
|
||||
merged["_stale_h"].max()) if len(merged) else float("nan")
|
||||
return out
|
||||
|
||||
|
||||
# ──────────────────────────── reporting ─────────────────────────────
|
||||
|
||||
def _fmt(v, nd=3):
|
||||
if isinstance(v, float):
|
||||
if math.isinf(v):
|
||||
return "inf"
|
||||
if math.isnan(v):
|
||||
return "n/a"
|
||||
return f"{v:.{nd}f}"
|
||||
return str(v)
|
||||
|
||||
|
||||
def _metrics_block(name: str, m: dict) -> list[str]:
|
||||
return [
|
||||
f" {name:<10} n={m['n']:<5} totalR={_fmt(m['total_r'], 2):>8} "
|
||||
f"meanR={_fmt(m['mean_r'])} medR={_fmt(m['median_r'])} "
|
||||
f"WR={_fmt(m['wr'], 2)}% PF={_fmt(m['pf'], 2)}",
|
||||
f" {'':<10} maxDD={_fmt(m['max_dd'], 2)} recovery={_fmt(m['recovery'], 2)}"
|
||||
f" sharpe={_fmt(m['sharpe'])} expectancy={_fmt(m['expectancy'])}"
|
||||
f" (wr·avgW − lr·avgL = {_fmt(m['expectancy_alt'])})",
|
||||
f" {'':<10} avg bars={_fmt(m['avg_bars'], 1)} "
|
||||
f"avg MFE={_fmt(m['avg_mfe'])}R avg MAE={_fmt(m['avg_mae'])}R",
|
||||
]
|
||||
|
||||
|
||||
def build_report(df: pd.DataFrame, book: str, bar_minutes: int, split: str,
|
||||
out_dir: str, q: int = 5, min_n: int = 10) -> str:
|
||||
lines: list[str] = []
|
||||
add = lines.append
|
||||
cut = split_instant(df["entry_time"], split)
|
||||
add("=" * 78)
|
||||
add(f"BOOK REPORT {os.path.basename(book)}")
|
||||
add("=" * 78)
|
||||
add(f"trades : {len(df)}")
|
||||
add(f"entry window : {df['entry_time'].min()} -> {df['entry_time'].max()}")
|
||||
add(f"bar minutes : {bar_minutes} (decision_time = entry_time − 1 bar)")
|
||||
add(f"split rule : {split} -> cut at {cut} "
|
||||
f"(half 1 = before, half 2 = at/after)")
|
||||
add(f"half sizes : h1={int((df['half'] == 1).sum())} "
|
||||
f"h2={int((df['half'] == 2).sum())}")
|
||||
add("session map : asian 3-6, london 7-12, overlap 13-16, ny 17-22, "
|
||||
"else off (on DECISION hour, UTC)")
|
||||
add("bucket edges : quantile bins are fit on the WHOLE book; the two "
|
||||
"halves share the same edges.")
|
||||
feats = feature_columns(df)
|
||||
add(f"screen features : {len(feats)} "
|
||||
f"({sum(1 for c in feats if c.startswith(('f_', 'x_')))} f_/x_ columns)")
|
||||
for k, v in df.attrs.items():
|
||||
add(f"htf staleness : {k} = {_fmt(v, 1)} h (max age of the matched bar)")
|
||||
|
||||
add("")
|
||||
add("── HEADLINE METRICS ──")
|
||||
lines.extend(_metrics_block("full", base_metrics(df["r_multiple"], df)))
|
||||
for h in (1, 2):
|
||||
sub = df[df["half"] == h]
|
||||
lines.extend(_metrics_block(f"half {h}",
|
||||
base_metrics(sub["r_multiple"], sub)))
|
||||
|
||||
group_specs = [
|
||||
("weekday", "weekday"), ("decision_hour", "decision_hour"),
|
||||
("session", "session"), ("quality", "quality"), ("regime", "regime"),
|
||||
("vol_regime", "vol_regime"), ("direction", "direction"),
|
||||
("strategy", "strategy"), ("exit_reason", "exit_reason"),
|
||||
("indicators_agree", "indicators_agree"),
|
||||
]
|
||||
tables = {}
|
||||
for name, col in group_specs:
|
||||
if col not in df.columns:
|
||||
continue
|
||||
t = group_table(df, col, min_n=min_n)
|
||||
tables[name] = t
|
||||
add("")
|
||||
add(f"── BY {name.upper()} ──")
|
||||
add(t.to_string() if not t.empty else " (empty)")
|
||||
|
||||
# (weekday, decision_hour) pivots
|
||||
add("")
|
||||
add("── weekday x decision_hour : TOTAL R ──")
|
||||
wh = group_table(df, ["weekday", "decision_hour"], min_n=min_n)
|
||||
if not wh.empty:
|
||||
piv_r = wh["total_r"].unstack("decision_hour")
|
||||
piv_n = wh["n"].unstack("decision_hour")
|
||||
add(piv_r.fillna(0).round(2).to_string())
|
||||
add("")
|
||||
add("── weekday x decision_hour : N ──")
|
||||
add(piv_n.fillna(0).astype(int).to_string())
|
||||
tables["weekday_hour"] = wh
|
||||
piv_r.to_csv(os.path.join(out_dir, "groups_weekday_hour_total_r.csv"))
|
||||
piv_n.to_csv(os.path.join(out_dir, "groups_weekday_hour_n.csv"))
|
||||
|
||||
st = streaks(df)
|
||||
add("")
|
||||
add("── STREAKS ──")
|
||||
add(f" max consecutive wins : {st['max_win_streak']}")
|
||||
add(f" max consecutive losses : {st['max_loss_streak']}")
|
||||
add(f" longest flat stretch : {st['longest_flat_trades']} trades "
|
||||
f"between equity highs")
|
||||
|
||||
rw = rolling_wr(df, 20).dropna()
|
||||
add("")
|
||||
add("── ROLLING WIN RATE (20 trades) ──")
|
||||
if len(rw):
|
||||
add(f" min={_fmt(rw.min(), 2)}% median={_fmt(rw.median(), 2)}% "
|
||||
f"max={_fmt(rw.max(), 2)}% n={len(rw)}")
|
||||
add(" last 5: " + ", ".join(
|
||||
f"{ts:%Y-%m-%d %H:%M}={v:.2f}%" for ts, v in rw.tail(5).items()))
|
||||
else:
|
||||
add(" (fewer than 20 trades)")
|
||||
|
||||
scr = feature_screen(df, q=q, min_n=min_n)
|
||||
add("")
|
||||
add("── FEATURE SCREEN (top 25 by min |gap| across halves) ──")
|
||||
add(" gap = mean R(top bucket) − mean R(bottom bucket), per half; "
|
||||
"min_abs_gap is 0 unless the sign is stable.")
|
||||
add(scr.head(25).to_string(index=False) if not scr.empty else " (none)")
|
||||
|
||||
stable = scr[scr["sign_stable"]].head(10) if not scr.empty else scr
|
||||
add("")
|
||||
add(f"── BUCKET TABLES FOR THE TOP {len(stable)} SIGN-STABLE FEATURES ──")
|
||||
if stable.empty:
|
||||
add(" (no feature has a sign-stable top-vs-bottom gap)")
|
||||
for feat in stable["feature"]:
|
||||
bk = quantile_buckets(df, feat, q=q, min_n=min_n)
|
||||
add("")
|
||||
add(f" · {feat} (mode={bk.attrs['mode']}, n={bk.attrs['n']}, "
|
||||
f"gap_h1={_fmt(bk.attrs['gap_h1'])}, "
|
||||
f"gap_h2={_fmt(bk.attrs['gap_h2'])})")
|
||||
add(bk.to_string(index=False))
|
||||
bk.to_csv(os.path.join(out_dir, f"buckets_{feat}.csv"), index=False)
|
||||
|
||||
# CSV artefacts
|
||||
for name, t in tables.items():
|
||||
t.to_csv(os.path.join(out_dir, f"groups_{name}.csv"))
|
||||
scr.to_csv(os.path.join(out_dir, "feature_screen.csv"), index=False)
|
||||
equity_curve(df).to_csv(os.path.join(out_dir, "equity.csv"), index=False)
|
||||
all_buckets = []
|
||||
for c in feats:
|
||||
bk = quantile_buckets(df, c, q=q, min_n=min_n)
|
||||
if bk.empty:
|
||||
continue
|
||||
bk = bk.copy()
|
||||
bk.insert(0, "feature", c)
|
||||
all_buckets.append(bk)
|
||||
if all_buckets:
|
||||
pd.concat(all_buckets, ignore_index=True).to_csv(
|
||||
os.path.join(out_dir, "buckets.csv"), index=False)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
ap.add_argument("--book", required=True)
|
||||
ap.add_argument("--bar-minutes", type=int, required=True)
|
||||
ap.add_argument("--split", default="median",
|
||||
help="'median' (default) or YYYY-MM-DD")
|
||||
ap.add_argument("--parquet-4h", default=None)
|
||||
ap.add_argument("--parquet-1d", default=None)
|
||||
ap.add_argument("--q", type=int, default=5)
|
||||
ap.add_argument("--min-n", type=int, default=10)
|
||||
ap.add_argument("--out", default="backtest/research/book_report_out")
|
||||
args = ap.parse_args()
|
||||
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
df = load_book(args.book, args.bar_minutes, args.split)
|
||||
if args.parquet_4h or args.parquet_1d:
|
||||
df = add_htf_extension(df, args.parquet_4h, args.parquet_1d)
|
||||
text = build_report(df, args.book, args.bar_minutes, args.split,
|
||||
args.out, q=args.q, min_n=args.min_n)
|
||||
print(text)
|
||||
with open(os.path.join(args.out, "report.txt"), "w") as fh:
|
||||
fh.write(text + "\n")
|
||||
print(f"\n[written] {os.path.join(args.out, 'report.txt')}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,121 @@
|
||||
"""What do the system's OWN indicators read right before a violent bar? Analysis tooling only.
|
||||
|
||||
Uses backtest.gpu_backtest.precompute_indicators (the same series the backtester feeds the
|
||||
engine) on 3 years of bars; every state is taken at t-1 (known before the bar). Reports burst
|
||||
rate (range >= 3x prior ATR14) for the NEXT bar and within the NEXT HOUR by indicator state,
|
||||
by regime, by volatility regime, by weekday x hour, day of month, calendar proximity — each in
|
||||
both halves — and whether the state predicts the burst's DIRECTION.
|
||||
|
||||
python backtest/research/burst_indicator_map.py PARQUET BAR_MINUTES [--start --end]
|
||||
"""
|
||||
import argparse, sys
|
||||
from pathlib import Path
|
||||
import numpy as np, pandas as pd
|
||||
ROOT = Path(__file__).resolve().parents[2]; sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "backtest"))
|
||||
from gpu_backtest import precompute_indicators
|
||||
from config import TechnicalConfig
|
||||
|
||||
CAL = ROOT / "backtest" / "cache" / "calendar.parquet"
|
||||
|
||||
|
||||
def wilder_atr(df, period=14):
|
||||
pc = df["Close"].shift(1)
|
||||
tr = pd.concat([df["High"] - df["Low"], (df["High"] - pc).abs(), (df["Low"] - pc).abs()], axis=1).max(axis=1)
|
||||
return tr.ewm(alpha=1 / period, min_periods=period, adjust=False).mean()
|
||||
|
||||
|
||||
def rate_table(d, col, target, half, order=None):
|
||||
base1 = d.loc[half == 1, target].mean(); base2 = d.loc[half == 2, target].mean()
|
||||
g = d.groupby(col, observed=True)
|
||||
out = pd.DataFrame({"n": g.size(),
|
||||
"h1_lift": d[half == 1].groupby(col, observed=True)[target].mean() / base1,
|
||||
"h2_lift": d[half == 2].groupby(col, observed=True)[target].mean() / base2,
|
||||
"rate%": 100 * g[target].mean()})
|
||||
if order is not None: out = out.reindex([o for o in order if o in out.index])
|
||||
out["stable"] = np.where((out.h1_lift > 1.15) & (out.h2_lift > 1.15), "HIGH", np.where((out.h1_lift < 0.85) & (out.h2_lift < 0.85), "low", ""))
|
||||
return out.round(2)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(); ap.add_argument("parquet"); ap.add_argument("bar_minutes", type=int)
|
||||
ap.add_argument("--start", default="2023-09-05"); ap.add_argument("--end", default="2026-09-05")
|
||||
a = ap.parse_args(); bm = a.bar_minutes
|
||||
px = pd.read_parquet(a.parquet)
|
||||
px = px[(px.index >= pd.Timestamp(a.start, tz="UTC") - pd.Timedelta(days=30)) & (px.index < pd.Timestamp(a.end, tz="UTC"))]
|
||||
ic = precompute_indicators(px, TechnicalConfig())
|
||||
d = pd.DataFrame(index=px.index)
|
||||
atr_prev = wilder_atr(px).shift(1)
|
||||
rng = px["High"] - px["Low"]
|
||||
d["burst"] = (rng >= 3.0 * atr_prev).astype(int)
|
||||
H = max(1, 60 // bm)
|
||||
d["next_hour"] = d["burst"][::-1].rolling(H, min_periods=1).max()[::-1].astype(int)
|
||||
d["next_bar"] = d["burst"]
|
||||
d["bdir"] = np.sign(px["Close"] - px["Open"])
|
||||
# states at t-1
|
||||
sh = lambda arr: pd.Series(np.asarray(arr, dtype=float), index=px.index).shift(1)
|
||||
rsi = sh(ic.rsi); adx = sh(ic.adx); adxs = sh(ic.adx_slope); hist = sh(ic.macd_hist); bbp = sh(ic.bb_position)
|
||||
bbw = sh(ic.bb_band_width); bbaw = sh(ic.bb_avg_width); kcsq = sh(ic.kc_squeeze); roc = sh(ic.roc)
|
||||
ef, em, es = sh(ic.ema_fast), sh(ic.ema_medium), sh(ic.ema_slow); et = sh(ic.ema_trend) if ic.ema_trend is not None else None
|
||||
close1 = px["Close"].shift(1); atr1 = sh(ic.atr)
|
||||
kcu, kcl = sh(ic.kc_upper), sh(ic.kc_lower)
|
||||
d["rsi"] = pd.cut(rsi, [0, 30, 40, 50, 60, 70, 100], labels=["<30", "30-40", "40-50", "50-60", "60-70", ">=70"])
|
||||
d["adx"] = pd.cut(adx, [0, 20, 25, 35, 50, 200], labels=["<20", "20-25", "25-35", "35-50", ">=50"])
|
||||
d["adx_slope"] = pd.cut(adxs, [-1e9, -3, 0, 3, 1e9], labels=["falling<-3", "-3..0", "0..3", "rising>3"])
|
||||
hist_prev = hist.shift(1)
|
||||
d["macd"] = np.select([(hist > 0) & (hist > hist_prev), (hist > 0), (hist < 0) & (hist < hist_prev), (hist < 0)],
|
||||
["pos&rising", "pos&fading", "neg&falling", "neg&fading"], default="flat")
|
||||
d["bb_pos"] = pd.cut(bbp, [-1e9, 0, 0.2, 0.8, 1.0, 1e9], labels=["<0 (below)", "0-0.2", "0.2-0.8", "0.8-1", ">1 (above)"])
|
||||
d["bb_squeeze"] = np.where(bbw < 0.75 * bbaw, "squeeze", "normal")
|
||||
d["kc_squeeze"] = np.where(kcsq > 0, "TTM squeeze", "no")
|
||||
d["kc_pos"] = np.select([close1 > kcu, close1 < kcl], ["above KC", "below KC"], default="inside")
|
||||
align = np.sign(close1 - ef) + np.sign(ef - em) + np.sign(em - es) + (np.sign(es - et) if et is not None else 0)
|
||||
d["ema_align"] = pd.cut(align, [-5, -3.5, -0.5, 0.5, 3.5, 5], labels=["fully bear(-4)", "mostly bear", "mixed", "mostly bull", "fully bull(+4)"])
|
||||
ribbon = (pd.concat([ef, em, es], axis=1).max(axis=1) - pd.concat([ef, em, es], axis=1).min(axis=1)) / atr1
|
||||
d["ribbon"] = np.where(ribbon < 1.0, "compressed(<1 ATR)", "expanded")
|
||||
ema_dir = np.where(align > 0, 1, np.where(align < 0, -1, 0))
|
||||
d["regime"] = np.select([(adx >= 25) & (ema_dir > 0), (adx >= 25) & (ema_dir < 0), adx >= 25, adxs > 3.0],
|
||||
["trending_up", "trending_down", "trending(mixed)", "transitioning"], default="ranging")
|
||||
atr_pct = atr1.rolling(50).apply(lambda w: (w[-1] > w).mean() * 100, raw=True)
|
||||
d["vol_regime"] = pd.cut(atr_pct, [-1, 30, 60, 80, 101], labels=["low", "normal", "high", "extreme"])
|
||||
d["roc"] = pd.cut(roc, [-1e9, -1, -0.3, 0.3, 1, 1e9], labels=["<-1%", "-1..-0.3", "-0.3..0.3", "0.3..1", ">1%"])
|
||||
d["ema200_dist"] = pd.cut((close1 - et) / et * 100 if et is not None else pd.Series(np.nan, index=px.index),
|
||||
[-1e9, -2.5, -0.5, 0.5, 2.5, 1e9], labels=["<-2.5%", "-2.5..-0.5", "near(±0.5%)", "0.5..2.5", ">2.5%"])
|
||||
# time
|
||||
ct = px.index + pd.Timedelta(minutes=bm)
|
||||
d["hour"] = ct.hour; d["weekday"] = ct.weekday; d["dom"] = pd.cut(ct.day, [0, 5, 10, 15, 20, 25, 31], labels=["1-5", "6-10", "11-15", "16-20", "21-25", "26-31"])
|
||||
d["first_friday"] = np.where((ct.weekday == 4) & (ct.day <= 7), "first Friday", "other")
|
||||
# calendar proximity (high-impact USD)
|
||||
cal = pd.read_parquet(CAL); cal["ts"] = pd.to_datetime(cal.dateline, unit="s", utc=True)
|
||||
hi = np.sort(cal[(cal.currency == "USD") & (cal.impactName.str.lower() == "high")].ts.values.astype("datetime64[ns]"))
|
||||
t = ct.values.astype("datetime64[ns]")
|
||||
nxt = np.searchsorted(hi, t) # next event at/after bar close
|
||||
mins_to = np.where(nxt < len(hi), (hi[np.minimum(nxt, len(hi) - 1)] - t) / np.timedelta64(1, "m"), 1e9)
|
||||
prv = nxt - 1
|
||||
mins_since = np.where(prv >= 0, (t - hi[np.maximum(prv, 0)]) / np.timedelta64(1, "m"), 1e9)
|
||||
d["calendar"] = np.select([mins_since <= 30, mins_to <= 15, mins_to <= 60, mins_to <= 180], ["0-30m AFTER hi-impact", "next 15m", "15-60m ahead", "1-3h ahead"], default="none within 3h")
|
||||
d = d[d.index >= pd.Timestamp(a.start, tz="UTC")].dropna(subset=["rsi", "adx", "bb_pos", "vol_regime"])
|
||||
half = pd.Series(np.where(np.arange(len(d)) < len(d) // 2, 1, 2), index=d.index)
|
||||
print(f"bars {len(d)} base next_bar {100*d.next_bar.mean():.3f}% next_hour {100*d.next_hour.mean():.2f}% (lift = rate / base; 'HIGH' = >1.15x in BOTH halves, 'low' = <0.85x in both)")
|
||||
groups = ["calendar", "regime", "vol_regime", "rsi", "adx", "adx_slope", "macd", "bb_pos", "bb_squeeze", "kc_squeeze", "kc_pos", "ema_align", "ribbon", "roc", "ema200_dist", "dom", "first_friday"]
|
||||
for target in ("next_bar", "next_hour"):
|
||||
print(f"\n################ TARGET: {target} ################")
|
||||
for g in groups:
|
||||
print(f"-- {g}"); print(rate_table(d, g, target, half).to_string())
|
||||
# weekday x hour top cells (next_bar)
|
||||
piv = d.pivot_table(index="weekday", columns="hour", values="next_bar", aggfunc="mean") * 100
|
||||
base = 100 * d.next_bar.mean()
|
||||
cells = piv.stack().sort_values(ascending=False)
|
||||
print(f"\n################ TOP 15 weekday x close-hour cells (next-bar burst pct, base {base:.2f}) ################")
|
||||
names = {0: "Mon", 1: "Tue", 2: "Wed", 3: "Thu", 4: "Fri", 6: "Sun"}
|
||||
for (wd, h), v in cells.head(15).items():
|
||||
n = int(((d.weekday == wd) & (d.hour == h)).sum())
|
||||
print(f" {names.get(wd, wd)} {h:02d}:xx {v:.2f}% ({v/base:.1f}x, n={n})")
|
||||
# direction: does the state predict the burst direction?
|
||||
b = d[d.burst == 1]
|
||||
print("\n################ BURST DIRECTION vs prior state (share of bursts going UP) ################")
|
||||
for g, up_states in (("ema_align", None), ("rsi", None), ("macd", None), ("regime", None)):
|
||||
print(f"-- {g}: " + ", ".join(f"{k}: {100*(v>0).mean():.0f}% up (n={len(v)})" for k, v in b.groupby(g, observed=True).bdir if len(v) >= 20))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,445 @@
|
||||
"""Where and when gold moves violently — burst map of the raw bars.
|
||||
|
||||
Usage:
|
||||
python backtest/research/burst_map.py --parquet PATH --bar-minutes N \
|
||||
[--k 3,4,5] [--book BOOK.csv] [--within-bars 12] \
|
||||
[--start YYYY-MM-DD --end YYYY-MM-DD] [--out DIR]
|
||||
|
||||
A "burst" is one bar whose range is >= K x the PRIOR bar's Wilder ATR(14) —
|
||||
the same definition burst_profile.py uses (that module computes it inline
|
||||
inside burst_table() and exposes no reusable helper, so the formula is
|
||||
replicated here bar-for-bar: TR = max(H-L, |H-Cprev|, |L-Cprev|), ATR =
|
||||
TR.ewm(alpha=1/14, min_periods=14, adjust=False).mean(), ratio =
|
||||
(H-L) / ATR.shift(1)).
|
||||
|
||||
Reports:
|
||||
* burst frequency by (weekday, UTC bar-CLOSE hour), plus mean |1 h| and
|
||||
|4 h| absolute moves in $ and in ATR units on the same grid;
|
||||
* post-burst drift in the burst's own direction over 6/12/24/48 bars,
|
||||
overall and by session bucket — does a violent bar continue or snap back;
|
||||
* the largest trailing-60-minute close-to-close moves, de-overlapped;
|
||||
* if a trade book is given: how trades that were decided just after a burst,
|
||||
or that were OPEN when one printed, performed against the rest.
|
||||
|
||||
Every ATR-unit number here is normalised by the PRIOR bar's ATR — the same
|
||||
denominator as the burst test, so a "4x burst that ran another 4 ATR" reads
|
||||
directly. All hours are UTC and are the bar's CLOSE hour (label + one bar),
|
||||
matching the clock the live gates evaluate.
|
||||
|
||||
Analysis tooling only — never imported by the live code.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import math
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
try: # package import (backtest.research.burst_map)
|
||||
from backtest.research.book_report import base_metrics, load_book, session_of
|
||||
except ImportError: # run directly as a script from anywhere
|
||||
from book_report import base_metrics, load_book, session_of
|
||||
|
||||
SLOT_UNKNOWN = ""
|
||||
|
||||
|
||||
# ─────────────────────────── bar primitives ─────────────────────────
|
||||
|
||||
def load_bars(parquet: str, start: str | None = None,
|
||||
end: str | None = None) -> pd.DataFrame:
|
||||
"""Read a cached OHLCV parquet, de-duplicated and oldest-first."""
|
||||
df = pd.read_parquet(parquet).sort_index()
|
||||
df = df[~df.index.duplicated(keep="last")]
|
||||
if start:
|
||||
df = df[df.index >= pd.Timestamp(start, tz="UTC")]
|
||||
if end:
|
||||
df = df[df.index <= pd.Timestamp(end, tz="UTC")]
|
||||
return df
|
||||
|
||||
|
||||
def wilder_atr(df: pd.DataFrame, period: int = 14) -> pd.Series:
|
||||
"""Wilder ATR — identical to the inline formula in burst_profile.py."""
|
||||
h, l, c = df["High"], df["Low"], df["Close"]
|
||||
tr = pd.concat([h - l, (h - c.shift()).abs(), (l - c.shift()).abs()],
|
||||
axis=1).max(axis=1)
|
||||
return tr.ewm(alpha=1 / period, min_periods=period, adjust=False).mean()
|
||||
|
||||
|
||||
def burst_ratio(df: pd.DataFrame, period: int = 14) -> pd.Series:
|
||||
"""Bar range expressed in PRIOR-bar ATRs."""
|
||||
return (df["High"] - df["Low"]) / wilder_atr(df, period).shift(1)
|
||||
|
||||
|
||||
def burst_flags(df: pd.DataFrame, k: float, period: int = 14) -> pd.Series:
|
||||
"""True where the bar's range is >= k x the prior bar's Wilder ATR(14)."""
|
||||
return (burst_ratio(df, period) >= k).fillna(False)
|
||||
|
||||
|
||||
def burst_dir(df: pd.DataFrame) -> pd.Series:
|
||||
"""+1 up bar, −1 down bar, 0 doji (sign of Close − Open)."""
|
||||
return np.sign(df["Close"] - df["Open"]).astype(float)
|
||||
|
||||
|
||||
def close_times(df: pd.DataFrame, bar_minutes: int) -> pd.DatetimeIndex:
|
||||
"""Bar CLOSE timestamps (the parquet index labels bar OPEN time)."""
|
||||
return df.index + pd.Timedelta(minutes=bar_minutes)
|
||||
|
||||
|
||||
# ───────────────────── when does gold move? tables ──────────────────
|
||||
|
||||
def hour_weekday_table(df: pd.DataFrame, k: float, bar_minutes: int = 5,
|
||||
ks: tuple[float, ...] | None = None) -> dict:
|
||||
"""Burst frequency and absolute-move size on a (weekday, close-hour) grid.
|
||||
|
||||
Returns a dict of DataFrames (several tables are wanted, one frame each):
|
||||
burst_pct : weekday rows x close-hour cols, % of bars that are >= k x
|
||||
bursts (one frame; per-k frames live in `burst_pct_by_k`)
|
||||
absmove : (weekday, close-hour) rows; bars, mean |1 h move| and
|
||||
|4 h move| in $ and in prior-bar ATR units
|
||||
by_hour : the same stats collapsed to close-hour
|
||||
by_weekday : the same stats collapsed to weekday
|
||||
"""
|
||||
ks = tuple(ks) if ks else (k,)
|
||||
ct = close_times(df, bar_minutes)
|
||||
hour = pd.Series(ct.hour, index=df.index)
|
||||
wd = pd.Series(ct.weekday, index=df.index)
|
||||
atr_prev = wilder_atr(df).shift(1)
|
||||
|
||||
per_hour = max(1, int(round(60 / bar_minutes)))
|
||||
c = df["Close"]
|
||||
mv1 = (c - c.shift(per_hour)).abs()
|
||||
mv4 = (c - c.shift(4 * per_hour)).abs()
|
||||
stats = pd.DataFrame({
|
||||
"weekday": wd, "hour": hour,
|
||||
"abs1h_usd": mv1, "abs4h_usd": mv4,
|
||||
"abs1h_atr": mv1 / atr_prev, "abs4h_atr": mv4 / atr_prev,
|
||||
})
|
||||
for kk in ks:
|
||||
stats[f"burst_{kk:g}x"] = burst_flags(df, kk).astype(float) * 100.0
|
||||
|
||||
agg = {"abs1h_usd": "mean", "abs4h_usd": "mean",
|
||||
"abs1h_atr": "mean", "abs4h_atr": "mean"}
|
||||
agg.update({f"burst_{kk:g}x": "mean" for kk in ks})
|
||||
|
||||
grid = stats.groupby(["weekday", "hour"]).agg(agg)
|
||||
grid.insert(0, "bars", stats.groupby(["weekday", "hour"]).size())
|
||||
by_hour = stats.groupby("hour").agg(agg)
|
||||
by_hour.insert(0, "bars", stats.groupby("hour").size())
|
||||
by_wd = stats.groupby("weekday").agg(agg)
|
||||
by_wd.insert(0, "bars", stats.groupby("weekday").size())
|
||||
|
||||
pct_by_k = {kk: grid[f"burst_{kk:g}x"].unstack("hour") for kk in ks}
|
||||
return {
|
||||
"burst_pct": pct_by_k[k if k in pct_by_k else ks[0]].round(3),
|
||||
"burst_pct_by_k": {kk: v.round(3) for kk, v in pct_by_k.items()},
|
||||
"absmove": grid.round(3),
|
||||
"by_hour": by_hour.round(3),
|
||||
"by_weekday": by_wd.round(3),
|
||||
}
|
||||
|
||||
|
||||
def post_burst_drift(df: pd.DataFrame, k: float,
|
||||
horizons: tuple[int, ...] = (6, 12, 24, 48),
|
||||
bar_minutes: int = 5) -> pd.DataFrame:
|
||||
"""Signed drift IN the burst's direction over the next h bars, in ATRs.
|
||||
|
||||
A positive mean means violent bars continue; negative means they snap
|
||||
back. Split by the session bucket of the burst bar's close hour.
|
||||
"""
|
||||
flags = burst_flags(df, k)
|
||||
d = burst_dir(df)
|
||||
atr_prev = wilder_atr(df).shift(1)
|
||||
ct = close_times(df, bar_minutes)
|
||||
bucket = pd.Series([session_of(h) for h in ct.hour], index=df.index)
|
||||
c = df["Close"]
|
||||
|
||||
rows = []
|
||||
for h in horizons:
|
||||
fwd = (c.shift(-h) - c) * d / atr_prev
|
||||
sub = fwd[flags].dropna()
|
||||
buckets = bucket[flags].reindex(sub.index)
|
||||
rows.append({"horizon_bars": h, "bucket": "ALL", "n": int(len(sub)),
|
||||
"mean_atr": round(float(sub.mean()), 4) if len(sub) else float("nan"),
|
||||
"median_atr": round(float(sub.median()), 4) if len(sub) else float("nan"),
|
||||
"pct_positive": round(float((sub > 0).mean() * 100), 2) if len(sub) else float("nan")})
|
||||
for b, s in sub.groupby(buckets):
|
||||
rows.append({"horizon_bars": h, "bucket": b, "n": int(len(s)),
|
||||
"mean_atr": round(float(s.mean()), 4),
|
||||
"median_atr": round(float(s.median()), 4),
|
||||
"pct_positive": round(float((s > 0).mean() * 100), 2)})
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def slot_label(weekday: int, hour: int, day: int) -> str:
|
||||
"""Time-slot label for a big move.
|
||||
|
||||
NOTE: there is no economic-calendar file in this repo, so these are
|
||||
TIME SLOTS, not confirmed events — "payroll window" means "the clock
|
||||
slot US payrolls normally print in", nothing more.
|
||||
"""
|
||||
if weekday == 4 and day <= 7 and hour in (12, 13):
|
||||
return "first-Friday 12:30 slot (payroll-report window)"
|
||||
if weekday == 2 and hour in (18, 19):
|
||||
return "Wednesday 18:00 slot (central-bank window)"
|
||||
if hour in (12, 13):
|
||||
return "12:30 slot (US data window)"
|
||||
if hour == 14:
|
||||
return "14:00 slot"
|
||||
if hour in (21, 22, 23):
|
||||
return "settlement/after-hours"
|
||||
if hour in (0, 1, 2):
|
||||
return "Asia open"
|
||||
return SLOT_UNKNOWN
|
||||
|
||||
|
||||
def largest_moves(df: pd.DataFrame, n: int = 40, window_minutes: int = 60,
|
||||
bar_minutes: int = 5) -> pd.DataFrame:
|
||||
"""The n biggest trailing-`window_minutes` close-to-close moves.
|
||||
|
||||
Overlapping windows are de-duplicated greedily: the largest move wins and
|
||||
anything whose close time is within +/- `window_minutes` of an already
|
||||
kept move is dropped, so one violent hour contributes one row.
|
||||
"""
|
||||
bars = max(1, int(round(window_minutes / bar_minutes)))
|
||||
c = df["Close"]
|
||||
move = c - c.shift(bars)
|
||||
atr_prev = wilder_atr(df).shift(1)
|
||||
ct = close_times(df, bar_minutes)
|
||||
tbl = pd.DataFrame({
|
||||
"close_time": ct, "move_usd": move.to_numpy(),
|
||||
"move_atr": (move / atr_prev).to_numpy(),
|
||||
}).dropna()
|
||||
tbl["abs_usd"] = tbl["move_usd"].abs()
|
||||
tbl = tbl.sort_values("abs_usd", ascending=False)
|
||||
|
||||
keep, taken = [], []
|
||||
gap = pd.Timedelta(minutes=window_minutes)
|
||||
for _, row in tbl.iterrows():
|
||||
t = row["close_time"]
|
||||
if any(abs(t - u) <= gap for u in taken):
|
||||
continue
|
||||
taken.append(t)
|
||||
keep.append(row)
|
||||
if len(keep) >= n:
|
||||
break
|
||||
out = pd.DataFrame(keep).reset_index(drop=True)
|
||||
if out.empty:
|
||||
return out
|
||||
out["weekday"] = out["close_time"].dt.weekday
|
||||
out["hour"] = out["close_time"].dt.hour
|
||||
out["direction"] = np.where(out["move_usd"] > 0, "UP", "DOWN")
|
||||
out["slot"] = [slot_label(w, h, t.day) for w, h, t
|
||||
in zip(out["weekday"], out["hour"], out["close_time"])]
|
||||
out["move_usd"] = out["move_usd"].round(2)
|
||||
out["move_atr"] = out["move_atr"].round(2)
|
||||
return out[["close_time", "weekday", "hour", "direction", "move_usd",
|
||||
"move_atr", "slot"]]
|
||||
|
||||
|
||||
# ───────────────────────── book <-> burst join ──────────────────────
|
||||
|
||||
def _stats(r: pd.Series) -> dict:
|
||||
m = base_metrics(r)
|
||||
return {"n": m["n"], "total_r": round(m["total_r"], 2),
|
||||
"mean_r": round(m["mean_r"], 3), "wr": round(m["wr"], 2),
|
||||
"pf": round(m["pf"], 2) if math.isfinite(m["pf"]) else float("inf")}
|
||||
|
||||
|
||||
def join_book_to_bursts(book_df: pd.DataFrame, bars: pd.DataFrame, k: float,
|
||||
within_bars: int = 12,
|
||||
bar_minutes: int = 5) -> dict:
|
||||
"""Split a trade book by its relationship to burst bars.
|
||||
|
||||
(a) `after_burst` : decision_time falls within `within_bars` bars after
|
||||
the most recent burst's close (bar distance is counted POSITIONALLY on
|
||||
the bar index, so weekend gaps cannot inflate it).
|
||||
(b) `burst_during` : at least one burst closed inside [entry, exit).
|
||||
(c) `burst_agrees` / `burst_opposes`: of (b), whether the FIRST burst that
|
||||
printed during the trade pushed with or against the trade direction.
|
||||
"""
|
||||
ct = close_times(bars, bar_minutes)
|
||||
ct_arr = ct.to_numpy()
|
||||
flags = burst_flags(bars, k).to_numpy()
|
||||
bdir = burst_dir(bars).to_numpy()
|
||||
burst_pos = np.flatnonzero(flags)
|
||||
|
||||
# Last burst position at or before each bar (−1 where none yet).
|
||||
last_burst = np.where(flags, np.arange(len(flags)), -1)
|
||||
last_burst = np.maximum.accumulate(last_burst)
|
||||
|
||||
dec = book_df["decision_time"].astype("datetime64[ns, UTC]").to_numpy()
|
||||
ent = book_df["entry_time"].astype("datetime64[ns, UTC]").to_numpy()
|
||||
exi = book_df["exit_time"].astype("datetime64[ns, UTC]").to_numpy()
|
||||
|
||||
# Position of the last bar that had CLOSED by decision_time.
|
||||
idx = np.searchsorted(ct_arr, dec, side="right") - 1
|
||||
valid = idx >= 0
|
||||
lb = np.where(valid, last_burst[np.clip(idx, 0, len(flags) - 1)], -1)
|
||||
bars_since = np.where((lb >= 0) & valid, idx - lb, np.iinfo(np.int32).max)
|
||||
after = pd.Series((bars_since >= 0) & (bars_since <= within_bars),
|
||||
index=book_df.index)
|
||||
|
||||
# Bursts printing inside the open trade.
|
||||
bct = ct_arr[burst_pos] if len(burst_pos) else np.array([], dtype=ct_arr.dtype)
|
||||
lo = np.searchsorted(bct, ent, side="left")
|
||||
hi = np.searchsorted(bct, exi, side="left")
|
||||
during = pd.Series(hi > lo, index=book_df.index)
|
||||
trade_sign = np.where(
|
||||
book_df["direction"].astype(str).str.upper().str.startswith("B"),
|
||||
1.0, -1.0)
|
||||
has = hi > lo
|
||||
first_dir = np.zeros(len(book_df), dtype=float)
|
||||
if len(burst_pos) and has.any():
|
||||
# hi > lo guarantees lo[has] indexes a real burst.
|
||||
first_dir[has] = bdir[burst_pos[lo[has]]]
|
||||
agrees = pd.Series(has & (first_dir * trade_sign > 0),
|
||||
index=book_df.index)
|
||||
opposes = pd.Series(has & (first_dir * trade_sign < 0),
|
||||
index=book_df.index)
|
||||
|
||||
r = book_df["r_multiple"]
|
||||
out = {
|
||||
"k": k, "within_bars": within_bars,
|
||||
"bursts_in_window": int(len(burst_pos)),
|
||||
"after_burst": _stats(r[after]),
|
||||
"not_after_burst": _stats(r[~after]),
|
||||
"burst_during": _stats(r[during]),
|
||||
"no_burst_during": _stats(r[~during]),
|
||||
"burst_agrees": _stats(r[agrees]),
|
||||
"burst_opposes": _stats(r[opposes]),
|
||||
"median_bars_since_burst": float(np.median(
|
||||
bars_since[bars_since < np.iinfo(np.int32).max])) if (
|
||||
bars_since < np.iinfo(np.int32).max).any() else float("nan"),
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
# ──────────────────────────── reporting ─────────────────────────────
|
||||
|
||||
def _fmt_stats(name: str, s: dict) -> str:
|
||||
pf = "inf" if math.isinf(s["pf"]) else f"{s['pf']:.2f}"
|
||||
return (f" {name:<18} n={s['n']:<5} totalR={s['total_r']:>8.2f} "
|
||||
f"meanR={s['mean_r']:>7.3f} WR={s['wr']:>6.2f}% PF={pf}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
ap.add_argument("--parquet", required=True)
|
||||
ap.add_argument("--bar-minutes", type=int, required=True)
|
||||
ap.add_argument("--k", default="3,4,5",
|
||||
help="burst thresholds in prior-bar ATRs (default 3,4,5)")
|
||||
ap.add_argument("--book", default=None)
|
||||
ap.add_argument("--book-bar-minutes", type=int, default=None,
|
||||
help="bar minutes of the BOOK (default: --bar-minutes)")
|
||||
ap.add_argument("--within-bars", type=int, default=12)
|
||||
ap.add_argument("--start", default=None)
|
||||
ap.add_argument("--end", default=None)
|
||||
ap.add_argument("--n-moves", type=int, default=40)
|
||||
ap.add_argument("--out", default="backtest/research/burst_map_out")
|
||||
args = ap.parse_args()
|
||||
|
||||
ks = [float(x) for x in args.k.split(",") if x.strip()]
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
bars = load_bars(args.parquet, args.start, args.end)
|
||||
bm = args.bar_minutes
|
||||
primary = ks[0]
|
||||
|
||||
lines: list[str] = []
|
||||
add = lines.append
|
||||
add("=" * 78)
|
||||
add(f"BURST MAP {os.path.basename(args.parquet)}")
|
||||
add("=" * 78)
|
||||
add(f"bars : {len(bars)} ({bars.index[0]} -> {bars.index[-1]}"
|
||||
f", labels = bar OPEN time)")
|
||||
add(f"bar minutes : {bm} hours below are bar CLOSE hours (UTC)")
|
||||
add(f"burst definition : range >= K x PRIOR bar Wilder ATR(14); "
|
||||
f"K = {', '.join(f'{k:g}' for k in ks)}")
|
||||
add("ATR units : always normalised by the PRIOR bar's ATR")
|
||||
for k in ks:
|
||||
f = burst_flags(bars, k)
|
||||
add(f" K={k:g}: {int(f.sum())} burst bars "
|
||||
f"({float(f.mean()) * 100:.3f}% of bars)")
|
||||
|
||||
tabs = hour_weekday_table(bars, primary, bm, ks=tuple(ks))
|
||||
add("")
|
||||
add(f"── BURST % BY (weekday, close-hour), K={primary:g} ──")
|
||||
add(tabs["burst_pct"].fillna(0).round(2).to_string())
|
||||
add("")
|
||||
add("── MEAN ABSOLUTE MOVE BY CLOSE-HOUR (all weekdays) ──")
|
||||
add(tabs["by_hour"].to_string())
|
||||
add("")
|
||||
add("── MEAN ABSOLUTE MOVE BY WEEKDAY ──")
|
||||
add(tabs["by_weekday"].to_string())
|
||||
|
||||
add("")
|
||||
add(f"── POST-BURST DRIFT (in the burst's own direction), K={primary:g} ──")
|
||||
drift = post_burst_drift(bars, primary, bar_minutes=bm)
|
||||
add(drift.to_string(index=False))
|
||||
|
||||
add("")
|
||||
add(f"── LARGEST {args.n_moves} TRAILING-60-MIN MOVES (de-overlapped) ──")
|
||||
add(" slot labels are CLOCK SLOTS — no economic calendar exists in "
|
||||
"this repo, so no event is confirmed.")
|
||||
moves = largest_moves(bars, n=args.n_moves, bar_minutes=bm)
|
||||
add(moves.to_string(index=False))
|
||||
|
||||
join_rows = []
|
||||
if args.book:
|
||||
bbm = args.book_bar_minutes or bm
|
||||
book = load_book(args.book, bbm)
|
||||
add("")
|
||||
add(f"── BOOK JOIN {os.path.basename(args.book)} "
|
||||
f"({len(book)} trades, book bar = {bbm} min) ──")
|
||||
first, last = close_times(bars, bm)[0], close_times(bars, bm)[-1]
|
||||
outside = int(((book["decision_time"] < first)
|
||||
| (book["exit_time"] > last)).sum())
|
||||
if outside:
|
||||
add(f" !! {outside} trade(s) start before / end after the bar "
|
||||
f"window ({first} .. {last}) — their burst context is "
|
||||
f"partial. Widen --start/--end or use a longer parquet.")
|
||||
add(" CAUTION: group (b) is survivorship-shaped — a trade only sees "
|
||||
"a burst if it survived long enough to still be open, so its high "
|
||||
"PF is mostly 'winners last longer', not a burst edge.")
|
||||
for k in ks:
|
||||
j = join_book_to_bursts(book, bars, k, args.within_bars, bm)
|
||||
add(f" K={k:g} ({j['bursts_in_window']} bursts in window; "
|
||||
f"median bars since last burst = "
|
||||
f"{j['median_bars_since_burst']:.0f})")
|
||||
add(f" (a) decided within {args.within_bars} bars of a burst")
|
||||
add(_fmt_stats("after_burst", j["after_burst"]))
|
||||
add(_fmt_stats("not_after_burst", j["not_after_burst"]))
|
||||
add(" (b) a burst printed while the trade was open")
|
||||
add(_fmt_stats("burst_during", j["burst_during"]))
|
||||
add(_fmt_stats("no_burst_during", j["no_burst_during"]))
|
||||
add(" (c) of (b), first burst vs the trade's direction")
|
||||
add(_fmt_stats("burst_agrees", j["burst_agrees"]))
|
||||
add(_fmt_stats("burst_opposes", j["burst_opposes"]))
|
||||
for grp in ("after_burst", "not_after_burst", "burst_during",
|
||||
"no_burst_during", "burst_agrees", "burst_opposes"):
|
||||
join_rows.append({"k": k, "group": grp,
|
||||
"within_bars": args.within_bars,
|
||||
**j[grp]})
|
||||
|
||||
text = "\n".join(lines)
|
||||
print(text)
|
||||
with open(os.path.join(args.out, "burst_map.txt"), "w") as fh:
|
||||
fh.write(text + "\n")
|
||||
pd.concat([v.assign(k=kk) for kk, v in tabs["burst_pct_by_k"].items()]
|
||||
).to_csv(os.path.join(args.out, "hour_weekday_burst_pct.csv"))
|
||||
tabs["absmove"].to_csv(os.path.join(args.out, "hour_weekday_absmove.csv"))
|
||||
tabs["by_hour"].to_csv(os.path.join(args.out, "by_hour.csv"))
|
||||
tabs["by_weekday"].to_csv(os.path.join(args.out, "by_weekday.csv"))
|
||||
pd.concat([post_burst_drift(bars, k, bar_minutes=bm).assign(k=k)
|
||||
for k in ks]).to_csv(
|
||||
os.path.join(args.out, "post_burst_drift.csv"), index=False)
|
||||
moves.to_csv(os.path.join(args.out, "largest_moves.csv"), index=False)
|
||||
if join_rows:
|
||||
pd.DataFrame(join_rows).to_csv(
|
||||
os.path.join(args.out, "book_join.csv"), index=False)
|
||||
print(f"\n[written] {os.path.join(args.out, 'burst_map.txt')}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Are there signs BEFORE a violent bar? Analysis tooling only — never imported by the live code.
|
||||
|
||||
For every bar t, features are computed from bars <= t-1 only. Targets:
|
||||
next_bar : bar t itself is a burst (range >= K x prior Wilder ATR14)
|
||||
next_hour : any burst in bars t .. t+H-1
|
||||
Reports per-quintile burst rates (lift vs base rate) in both halves, the same
|
||||
within/outside the scheduled-data hours, and an out-of-sample logistic model
|
||||
(fit on half 1, scored on half 2): AUC + lift in the top 5% of scores.
|
||||
|
||||
python backtest/research/burst_precursors.py PARQUET BAR_MINUTES [--k 3] [--start ... --end ...]
|
||||
"""
|
||||
import argparse, sys
|
||||
import numpy as np, pandas as pd
|
||||
|
||||
|
||||
def wilder_atr(df, period=14):
|
||||
pc = df["Close"].shift(1)
|
||||
tr = pd.concat([df["High"] - df["Low"], (df["High"] - pc).abs(), (df["Low"] - pc).abs()], axis=1).max(axis=1)
|
||||
return tr.ewm(alpha=1 / period, min_periods=period, adjust=False).mean()
|
||||
|
||||
|
||||
def build(df, bar_minutes, k):
|
||||
d = df.copy()
|
||||
rng = d["High"] - d["Low"]
|
||||
atr_prev = wilder_atr(d).shift(1) # ATR known at t-1
|
||||
d["burst"] = (rng >= k * atr_prev).astype(int)
|
||||
H = max(1, 60 // bar_minutes)
|
||||
# burst within next H bars (t..t+H-1)
|
||||
d["next_hour"] = d["burst"][::-1].rolling(H, min_periods=1).max()[::-1].astype(int)
|
||||
d["next_bar"] = d["burst"]
|
||||
close_t = d.index + pd.Timedelta(minutes=bar_minutes)
|
||||
d["hour"] = close_t.hour
|
||||
r1 = rng.shift(1)
|
||||
d["squeeze6"] = r1.rolling(6).mean() / atr_prev
|
||||
d["squeeze12"] = r1.rolling(12).mean() / atr_prev
|
||||
d["last_bar_range"] = r1 / atr_prev
|
||||
sma = d["Close"].shift(1).rolling(20).mean(); sd = d["Close"].shift(1).rolling(20).std(ddof=0)
|
||||
bbw = (4 * sd / sma)
|
||||
d["bbw_pct500"] = bbw.rolling(500).rank(pct=True)
|
||||
v1 = d["Volume"].shift(1)
|
||||
d["vol_ratio"] = v1.rolling(3).mean() / v1.rolling(50).mean()
|
||||
d["drift12"] = (d["Close"].shift(1) - d["Close"].shift(13)).abs() / atr_prev
|
||||
d["atr_pct500"] = atr_prev.rolling(500).rank(pct=True)
|
||||
b = d["burst"].shift(1).fillna(0).astype(int)
|
||||
since = np.zeros(len(d)); cnt = 200
|
||||
bv = b.values
|
||||
for i in range(len(d)):
|
||||
cnt = 0 if bv[i] == 1 else min(200, cnt + 1)
|
||||
since[i] = cnt
|
||||
d["bars_since_burst"] = since
|
||||
return d.dropna(subset=["squeeze6", "squeeze12", "bbw_pct500", "vol_ratio", "drift12", "atr_pct500"])
|
||||
|
||||
|
||||
FEATS = ["squeeze6", "squeeze12", "last_bar_range", "bbw_pct500", "vol_ratio", "drift12", "atr_pct500", "bars_since_burst"]
|
||||
|
||||
|
||||
def quintile_table(d, feat, target, half):
|
||||
q = pd.qcut(d[feat], 5, labels=False, duplicates="drop")
|
||||
base1 = d.loc[half == 1, target].mean(); base2 = d.loc[half == 2, target].mean()
|
||||
rows = []
|
||||
for b in sorted(q.dropna().unique()):
|
||||
m = q == b
|
||||
lo, hi = d.loc[m, feat].min(), d.loc[m, feat].max()
|
||||
r1 = d.loc[m & (half == 1), target].mean(); r2 = d.loc[m & (half == 2), target].mean()
|
||||
rows.append((int(b), lo, hi, int(m.sum()), 100 * r1, r1 / base1, 100 * r2, r2 / base2))
|
||||
t = pd.DataFrame(rows, columns=["bin", "lo", "hi", "n", "h1_rate%", "h1_lift", "h2_rate%", "h2_lift"])
|
||||
return t.round(3)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(); ap.add_argument("parquet"); ap.add_argument("bar_minutes", type=int)
|
||||
ap.add_argument("--k", type=float, default=3.0); ap.add_argument("--start"); ap.add_argument("--end")
|
||||
a = ap.parse_args()
|
||||
df = pd.read_parquet(a.parquet)
|
||||
if a.start: df = df[df.index >= pd.Timestamp(a.start, tz="UTC")]
|
||||
if a.end: df = df[df.index < pd.Timestamp(a.end, tz="UTC")]
|
||||
d = build(df, a.bar_minutes, a.k)
|
||||
half = pd.Series(np.where(np.arange(len(d)) < len(d) // 2, 1, 2), index=d.index)
|
||||
print(f"bars {len(d)} K={a.k} base rate next_bar {100*d.next_bar.mean():.3f}% next_hour {100*d.next_hour.mean():.2f}%")
|
||||
print(f"half 1 ends {d.index[len(d)//2 - 1]}")
|
||||
# 1. the clock
|
||||
hr = d.groupby("hour")[["next_bar", "next_hour"]].mean() * 100
|
||||
hr["lift_next_bar"] = hr["next_bar"] / (100 * d.next_bar.mean())
|
||||
print("\n== THE CLOCK (bar-close hour UTC): burst rate % and lift ==")
|
||||
print(hr.round(2).T.to_string())
|
||||
sched = d["hour"].isin([1, 12, 13])
|
||||
print(f"\nscheduled hours (01,12,13) hold {100*d.loc[sched,'next_bar'].sum()/d.next_bar.sum():.1f}% of bursts in {100*sched.mean():.1f}% of bars")
|
||||
# 2. per-feature quintiles, both halves
|
||||
for target in ("next_bar", "next_hour"):
|
||||
print(f"\n== TARGET {target}: quintile burst rate (lift vs base) — ALL hours ==")
|
||||
for f in FEATS:
|
||||
t = quintile_table(d, f, target, half)
|
||||
print(f"-- {f}: lifts h1 {list(t.h1_lift)} h2 {list(t.h2_lift)} (bins lo..hi {t.lo.iloc[0]:.2f}..{t.hi.iloc[-1]:.2f})")
|
||||
# 3. same, OUTSIDE the scheduled hours (does anything predict the unscheduled ones?)
|
||||
u = d[~sched]; hu = half[~sched]
|
||||
print(f"\n== next_bar lifts OUTSIDE scheduled hours (base {100*u.next_bar.mean():.3f}%) ==")
|
||||
for f in FEATS:
|
||||
t = quintile_table(u, f, "next_bar", hu)
|
||||
print(f"-- {f}: h1 {list(t.h1_lift)} h2 {list(t.h2_lift)}")
|
||||
# 4. OOS logistic model: fit half 1, score half 2
|
||||
try:
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
from sklearn.metrics import roc_auc_score
|
||||
X = d[FEATS].copy()
|
||||
for h in range(24): X[f"h{h}"] = (d["hour"] == h).astype(float)
|
||||
Xc = d[FEATS].copy() # clock-free variant
|
||||
for name, XX in (("features+clock", X), ("features only", Xc), ("clock only", X[[f"h{h}" for h in range(24)]])):
|
||||
mu, sd = XX[half == 1].mean(), XX[half == 1].std().replace(0, 1)
|
||||
Z = (XX - mu) / sd
|
||||
for target in ("next_bar", "next_hour"):
|
||||
m = LogisticRegression(C=0.1, max_iter=2000, class_weight="balanced").fit(Z[half == 1], d.loc[half == 1, target])
|
||||
p = m.predict_proba(Z[half == 2])[:, 1]; y = d.loc[half == 2, target]
|
||||
auc = roc_auc_score(y, p)
|
||||
top = p >= np.quantile(p, 0.95)
|
||||
print(f"OOS {name:15s} {target:9s}: AUC {auc:.3f} | top-5% of scores: burst rate {100*y[top].mean():.2f}% vs base {100*y.mean():.2f}% (lift {y[top].mean()/y.mean():.2f}x), captures {100*y[top].sum()/y.sum():.1f}% of bursts")
|
||||
except ImportError:
|
||||
print("sklearn missing — OOS model skipped")
|
||||
# 5. direction: does the burst continue the prior 12-bar drift?
|
||||
bt = d[d.burst == 1]
|
||||
prior = np.sign(bt["Close"].shift(1) - bt["Close"].shift(13)) if False else np.sign(d["Close"].shift(1) - d["Close"].shift(13))[d.burst == 1]
|
||||
bdir = np.sign(bt["Close"] - bt["Open"])
|
||||
agree = (prior == bdir)[(prior != 0) & (bdir != 0)]
|
||||
print(f"\nburst direction == prior-hour drift direction: {100*agree.mean():.1f}% of {len(agree)} bursts (50% = no information)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Do scheduled economic events move gold violently, and do they hurt OPEN trades? Analysis tooling only.
|
||||
|
||||
Needs backtest/cache/calendar.parquet (ff_calendar_scraper.py). Uses USD events by impact.
|
||||
python backtest/research/calendar_impact.py PARQUET_BARS BAR_MINUTES BOOK.csv ARM DIST [--start --end]
|
||||
"""
|
||||
import argparse, sys
|
||||
from pathlib import Path
|
||||
import numpy as np, pandas as pd
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from open_trade_bursts import wilder_atr, simulate, r_of, stats, K, COST
|
||||
|
||||
CAL = Path(__file__).resolve().parent.parent / "cache" / "calendar.parquet"
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(); ap.add_argument("parquet"); ap.add_argument("bar_minutes", type=int)
|
||||
ap.add_argument("book"); ap.add_argument("arm", type=float); ap.add_argument("dist", type=float)
|
||||
ap.add_argument("--start", default="2023-09-05"); ap.add_argument("--end", default="2026-09-05")
|
||||
a = ap.parse_args(); bm = a.bar_minutes
|
||||
cal = pd.read_parquet(CAL)
|
||||
cal["ts"] = pd.to_datetime(cal["dateline"], unit="s", utc=True)
|
||||
cal = cal[(cal.ts >= pd.Timestamp(a.start, tz="UTC")) & (cal.ts < pd.Timestamp(a.end, tz="UTC"))]
|
||||
usd = cal[cal.currency == "USD"].copy()
|
||||
usd["impact"] = usd["impactName"].str.lower()
|
||||
print(f"calendar: {len(cal)} events in window, USD {len(usd)}: " + ", ".join(f"{k}={v}" for k, v in usd.impact.value_counts().items()))
|
||||
px = pd.read_parquet(a.parquet); px = px[(px.index >= pd.Timestamp(a.start, tz="UTC")) & (px.index < pd.Timestamp(a.end, tz="UTC"))]
|
||||
atr_prev = wilder_atr(px).shift(1)
|
||||
rng = px["High"] - px["Low"]; burst = (rng >= K * atr_prev).fillna(False)
|
||||
close = px["Close"]; idx = px.index
|
||||
H1 = 60 // bm
|
||||
move60 = (close.shift(-H1) - close).abs() / atr_prev # |move| over the next hour from bar OPEN-ish (label)
|
||||
move15 = (close.shift(-max(1, 15 // bm)) - close).abs() / atr_prev
|
||||
burst60 = burst[::-1].rolling(H1, min_periods=1).max()[::-1].astype(bool)
|
||||
# event -> bar containing it (label <= ts < label + bm)
|
||||
def bar_pos(ts):
|
||||
p = idx.searchsorted(ts, side="right") - 1
|
||||
return p if 0 <= p < len(idx) and (ts - idx[p]) < pd.Timedelta(minutes=bm) else -1
|
||||
usd["pos"] = [bar_pos(t) for t in usd.ts]
|
||||
usd = usd[usd.pos >= 0]
|
||||
usd["bar_range_atr"] = rng.values[usd.pos] / atr_prev.values[usd.pos]
|
||||
usd["burst_in_60m"] = burst60.values[usd.pos]
|
||||
usd["move60_atr"] = move60.values[usd.pos]; usd["move60_usd"] = (close.shift(-H1) - close).abs().values[usd.pos]
|
||||
usd["move15_atr"] = move15.values[usd.pos]
|
||||
# baseline: all bars, and bars in the same close-hours without any USD high/medium event within the hour
|
||||
chour = (idx + pd.Timedelta(minutes=bm)).hour
|
||||
ev_bars = set()
|
||||
for p in usd[usd.impact.isin(["high", "medium"])].pos: ev_bars.update(range(max(0, p - H1), min(len(idx), p + H1)))
|
||||
quiet = ~np.isin(np.arange(len(idx)), list(ev_bars))
|
||||
def base_row(mask, label):
|
||||
return dict(group=label, n=int(mask.sum()), bar_range_atr=float((rng / atr_prev)[mask].mean()), burst_in_60m_pct=100 * float(burst60[mask].mean()),
|
||||
move15_atr=float(move15[mask].mean()), move60_atr=float(move60[mask].mean()), move60_usd=float((close.shift(-H1) - close).abs()[mask].mean()))
|
||||
rows = [base_row(np.ones(len(idx), bool), "ALL bars"), base_row(quiet & np.isin(chour, [12, 13, 14]), "12-14 UTC bars with NO USD event")]
|
||||
for imp in ("high", "medium", "low"):
|
||||
g = usd[usd.impact == imp]
|
||||
rows.append(dict(group=f"USD {imp}-impact events", n=len(g), bar_range_atr=g.bar_range_atr.mean(), burst_in_60m_pct=100 * g.burst_in_60m.mean(),
|
||||
move15_atr=g.move15_atr.mean(), move60_atr=g.move60_atr.mean(), move60_usd=g.move60_usd.mean()))
|
||||
print("\n== HOW MUCH GOLD MOVES AROUND USD EVENTS (event bar = the bar containing the release) ==")
|
||||
print(pd.DataFrame(rows).round(2).to_string(index=False))
|
||||
hi = usd[usd.impact == "high"]
|
||||
top = hi.groupby("name").agg(n=("ts", "size"), bar_range_atr=("bar_range_atr", "mean"), burst60_pct=("burst_in_60m", lambda s: 100 * s.mean()),
|
||||
move60_usd=("move60_usd", "mean"), move60_atr=("move60_atr", "mean")).sort_values("n", ascending=False)
|
||||
print("\n== HIGH-IMPACT USD EVENTS BY NAME (n >= 6) ==")
|
||||
print(top[top.n >= 6].round(2).to_string())
|
||||
# what share of ALL bursts sit within ±30 min of a high/medium USD event?
|
||||
near = np.zeros(len(idx), bool)
|
||||
for p in usd[usd.impact.isin(["high", "medium"])].pos: near[max(0, p - 30 // bm):min(len(idx), p + 30 // bm + 1)] = True
|
||||
nearh = np.zeros(len(idx), bool)
|
||||
for p in hi.pos: nearh[max(0, p - 30 // bm):min(len(idx), p + 30 // bm + 1)] = True
|
||||
print(f"\nshare of ALL violent bars within ±30 min of a USD high-impact event: {100*burst[nearh].sum()/burst.sum():.1f}% (those bars are {100*nearh.mean():.1f}% of time)")
|
||||
print(f"share within ±30 min of a USD high- or medium-impact event: {100*burst[near].sum()/burst.sum():.1f}% (bars {100*near.mean():.1f}% of time)")
|
||||
# open trades through high-impact events
|
||||
book = pd.read_csv(a.book); book["entry_time"] = pd.to_datetime(book.entry_time, utc=True, format="mixed")
|
||||
bars = dict(O=px.Open.values, H=px.High.values, L=px.Low.values, C=px.Close.values, burst=burst.values, chour=chour.values)
|
||||
pos = idx.get_indexer(book.entry_time); ok = pos >= 0; book = book[ok].reset_index(drop=True); pos = pos[ok]
|
||||
ev_pos = np.sort(hi.pos.values)
|
||||
# flat mask: exit at the open of the bar that CONTAINS a high-impact event... use the bar before (exit ~5-15 min before)
|
||||
flat_mask = np.zeros(len(idx), bool); tight_mask = np.zeros(len(idx), bool)
|
||||
for p in ev_pos:
|
||||
if p - 1 >= 0: flat_mask[p - 1] = True
|
||||
tight_mask[max(0, p - 1):min(len(idx), p + H1)] = True
|
||||
res = {}
|
||||
for v, kw in (("base", {}), ("mask_flat", dict(flat_mask=flat_mask)), ("mask_tight", dict(tight_mask=tight_mask))):
|
||||
rs, fl, thru = [], [], []
|
||||
for i, t in book.iterrows():
|
||||
xp, held, reason, flag = simulate(bars, pos[i], t.direction, t.entry, t.sl, a.arm, a.dist, v, **kw)
|
||||
rs.append(r_of(t.direction, t.entry, t.sl, xp)); fl.append(flag)
|
||||
if v == "base":
|
||||
j0, j1 = pos[i], pos[i] + held
|
||||
k = ev_pos[(ev_pos > j0) & (ev_pos <= j1)]
|
||||
thru.append(k[0] if len(k) else -1)
|
||||
res[v] = (np.array(rs), np.array(fl))
|
||||
if v == "base": book["ev_pos"] = thru
|
||||
base = res["base"][0]; half = np.where(np.arange(len(book)) < len(book) // 2, 1, 2)
|
||||
hit = book.ev_pos.values >= 0
|
||||
print(f"\n== OPEN TRADES THAT HELD THROUGH A HIGH-IMPACT USD EVENT ({a.book.split('/')[-1]}) ==")
|
||||
print(f"trades through an event: {hit.sum()} of {len(book)} ({100*hit.mean():.1f}%) | mean R {base[hit].mean():+.3f} vs {base[~hit].mean():+.3f} for the rest | WR {100*(base[hit]>0).mean():.0f}% vs {100*(base[~hit]>0).mean():.0f}%")
|
||||
if hit.sum():
|
||||
d = np.where(book.direction == "BUY", 1.0, -1.0)
|
||||
m15 = np.array([ (close.shift(-max(1,15//bm)).values[p] - close.values[p]) if p >= 0 else np.nan for p in book.ev_pos])
|
||||
with_ = hit & (d * m15 > 0); against = hit & (d * m15 < 0)
|
||||
print(f" event's first 15-min move WITH the trade: n={with_.sum()} mean R {base[with_].mean():+.3f} | AGAINST: n={against.sum()} mean R {base[against].mean():+.3f} (WR {100*(base[against]>0).mean():.0f}%)")
|
||||
print("\n== PROTECTION AROUND HIGH-IMPACT USD EVENTS ==")
|
||||
b = stats(base)
|
||||
for v, (r, f) in res.items():
|
||||
s = stats(r); aff = f == 1
|
||||
d1 = r[half == 1].sum() - base[half == 1].sum(); d2 = r[half == 2].sum() - base[half == 2].sum()
|
||||
print(f"{v:11s} total {s['total']:+8.2f} Δ {s['total']-b['total']:+7.2f} PF {s['pf']:.2f} DD {s['dd']:.2f} affected {aff.sum():4d} (affected: base {base[aff].sum():+.2f} -> {r[aff].sum():+.2f}) | Δ h1 {d1:+.2f} / h2 {d2:+.2f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,150 @@
|
||||
"""What do violent bars do to OPEN trades, and can we protect them? Analysis tooling only.
|
||||
|
||||
Replays every trade of a book on the raw bars with the backtester's exit model (SL-first,
|
||||
gap-through at open, close-only trail ratchet, 300-bar timeout, $0.65 cost) and compares:
|
||||
base : reproduce the book (validation: must match r_multiple for almost every trade)
|
||||
flat_data : exit at the OPEN of the first bar whose close-hour enters {12,13} UTC (US data)
|
||||
flat_asia : same for close-hour {0,1} UTC (Asian open)
|
||||
tight_data : entering {12,13}, hold a protective stop 0.5R below the best price for the window
|
||||
tight_asia : same for {0,1}
|
||||
react_exit : a burst bar (range >= 3x prior ATR14) closes AGAINST the trade -> exit next open
|
||||
react_tight : after an opposing burst, protective stop 0.5R below the best price for the rest
|
||||
Plus a damage table: what happens to a trade when the first burst hits it (with vs against).
|
||||
|
||||
python backtest/research/open_trade_bursts.py BOOK.csv PARQUET BAR_MINUTES ARM DIST
|
||||
"""
|
||||
import sys
|
||||
import numpy as np, pandas as pd
|
||||
|
||||
COST = 0.65; MAX_BARS = 300; K = 3.0; PROT = 0.5
|
||||
|
||||
|
||||
def wilder_atr(df, period=14):
|
||||
pc = df["Close"].shift(1)
|
||||
tr = pd.concat([df["High"] - df["Low"], (df["High"] - pc).abs(), (df["Low"] - pc).abs()], axis=1).max(axis=1)
|
||||
return tr.ewm(alpha=1 / period, min_periods=period, adjust=False).mean()
|
||||
|
||||
|
||||
def simulate(bars, i0, direction, entry, sl, arm, dist, variant, hours_flat=(), hours_tight=(), flat_mask=None, tight_mask=None):
|
||||
"""bars: dict of numpy arrays; i0 = index of the entry bar. Returns (exit_price, bars_held, reason, flag)."""
|
||||
O, H, L, C, burst, chour = bars["O"], bars["H"], bars["L"], bars["C"], bars["burst"], bars["chour"]
|
||||
d = 1.0 if direction == "BUY" else -1.0
|
||||
risk = abs(entry - sl); stop = sl; peak = entry; flag = 0
|
||||
n = len(O); tight = False; tight_until = -1; exit_next = False
|
||||
for j in range(i0, min(n, i0 + MAX_BARS)):
|
||||
if exit_next:
|
||||
return O[j], j - i0, "REACT", flag
|
||||
h = chour[j]
|
||||
in_flat = h in hours_flat; in_tight = h in hours_tight
|
||||
if variant.startswith("flat") and in_flat:
|
||||
return O[j], j - i0, "FLAT", 1
|
||||
if variant == "mask_flat" and flat_mask is not None and flat_mask[j]:
|
||||
return O[j], j - i0, "FLAT", 1
|
||||
if variant.startswith("tight") and in_tight:
|
||||
tight = True; tight_until = j; flag = 1
|
||||
if variant == "mask_tight" and tight_mask is not None and tight_mask[j]:
|
||||
tight = True; tight_until = j; flag = 1
|
||||
# stop check (SL first, gap-through at open)
|
||||
if d > 0:
|
||||
if O[j] <= stop: return O[j], j - i0, "SL", flag
|
||||
if L[j] <= stop: return stop, j - i0, "SL", flag
|
||||
else:
|
||||
if O[j] >= stop: return O[j], j - i0, "SL", flag
|
||||
if H[j] >= stop: return stop, j - i0, "SL", flag
|
||||
# update peak / trail at bar close
|
||||
peak = max(peak, H[j]) if d > 0 else min(peak, L[j])
|
||||
fav = d * (peak - entry)
|
||||
if fav >= arm * risk:
|
||||
stop = max(stop, peak - dist * risk) if d > 0 else min(stop, peak + dist * risk)
|
||||
if tight and (variant == "react_tight" or j <= tight_until or in_tight):
|
||||
stop = max(stop, peak - PROT * risk) if d > 0 else min(stop, peak + PROT * risk)
|
||||
if variant.startswith("react") and burst[j] and d * (C[j] - O[j]) < 0:
|
||||
flag = 1
|
||||
if variant == "react_exit": exit_next = True
|
||||
else: tight = True
|
||||
j = min(n - 1, i0 + MAX_BARS - 1)
|
||||
return C[j], j - i0, "TIMEOUT", flag
|
||||
|
||||
|
||||
def r_of(direction, entry, sl, exit_price):
|
||||
d = 1.0 if direction == "BUY" else -1.0
|
||||
return (d * (exit_price - entry) - COST) / abs(entry - sl)
|
||||
|
||||
|
||||
def stats(r):
|
||||
r = np.asarray(r); eq = np.concatenate([[0.0], np.cumsum(r)]); dd = (np.maximum.accumulate(eq) - eq).max()
|
||||
pf = r[r > 0].sum() / max(-r[r < 0].sum(), 1e-9)
|
||||
return dict(n=len(r), total=r.sum(), mean=r.mean(), wr=100 * (r > 0).mean(), pf=pf, dd=dd)
|
||||
|
||||
|
||||
def main():
|
||||
book_csv, pq, bm, arm, dist = sys.argv[1], sys.argv[2], int(sys.argv[3]), float(sys.argv[4]), float(sys.argv[5])
|
||||
book = pd.read_csv(book_csv); book["entry_time"] = pd.to_datetime(book["entry_time"], utc=True, format="mixed")
|
||||
px = pd.read_parquet(pq)
|
||||
atr_prev = wilder_atr(px).shift(1)
|
||||
bars = dict(O=px["Open"].values, H=px["High"].values, L=px["Low"].values, C=px["Close"].values,
|
||||
burst=((px["High"] - px["Low"]) >= K * atr_prev).fillna(False).values,
|
||||
chour=(px.index + pd.Timedelta(minutes=bm)).hour.values)
|
||||
idx = pd.Index(px.index)
|
||||
pos = idx.get_indexer(book["entry_time"])
|
||||
ok = pos >= 0
|
||||
print(f"{book_csv}: {len(book)} trades, {ok.sum()} matched to bars; arm {arm} dist {dist}")
|
||||
book = book[ok].reset_index(drop=True); pos = pos[ok]
|
||||
half = np.where(np.arange(len(book)) < len(book) // 2, 1, 2)
|
||||
variants = {"base": {}, "flat_data": dict(hours_flat=(12, 13)), "flat_asia": dict(hours_flat=(0, 1)),
|
||||
"tight_data": dict(hours_tight=(12, 13)), "tight_asia": dict(hours_tight=(0, 1)),
|
||||
"react_exit": {}, "react_tight": {}}
|
||||
results = {}
|
||||
for v, kw in variants.items():
|
||||
rs, flags = [], []
|
||||
for i, t in book.iterrows():
|
||||
xp, held, reason, flag = simulate(bars, pos[i], t.direction, t.entry, t.sl, arm, dist, v, **kw)
|
||||
rs.append(r_of(t.direction, t.entry, t.sl, xp)); flags.append(flag)
|
||||
results[v] = (np.array(rs), np.array(flags))
|
||||
base = results["base"][0]
|
||||
match = np.isclose(base, book["r_multiple"].values, atol=0.02).mean()
|
||||
print(f"validation: replay reproduces the book's r_multiple (±0.02) on {100*match:.1f}% of trades; "
|
||||
f"sum replay {base.sum():+.2f} vs book {book.r_multiple.sum():+.2f}")
|
||||
print("\n== PROTECTION VARIANTS (whole book; 'affected' = trades where the rule fired) ==")
|
||||
print(f"{'variant':12s} {'n':>5s} {'affected':>8s} {'total R':>9s} {'Δ vs base':>9s} {'PF':>5s} {'maxDD':>6s} {'WR%':>5s} | affected only: base R -> variant R | Δ h1 / Δ h2")
|
||||
b = stats(base)
|
||||
for v, (r, f) in results.items():
|
||||
s = stats(r); aff = f == 1
|
||||
d1 = r[half == 1].sum() - base[half == 1].sum(); d2 = r[half == 2].sum() - base[half == 2].sum()
|
||||
print(f"{v:12s} {s['n']:5d} {aff.sum():8d} {s['total']:+9.2f} {s['total']-b['total']:+9.2f} {s['pf']:5.2f} {s['dd']:6.2f} {s['wr']:5.1f} | "
|
||||
f"{base[aff].sum():+8.2f} -> {r[aff].sum():+8.2f} (n={aff.sum()}) | {d1:+.2f} / {d2:+.2f}")
|
||||
# damage table: first burst while open (base path), with vs against
|
||||
print("\n== DAMAGE: first violent bar that hits an OPEN trade (base exits) ==")
|
||||
rows = []
|
||||
for i, t in book.iterrows():
|
||||
d = 1.0 if t.direction == "BUY" else -1.0; risk = abs(t.entry - t.sl)
|
||||
xp, held, reason, _ = simulate(bars, pos[i], t.direction, t.entry, t.sl, arm, dist, "base")
|
||||
j0 = pos[i]; j1 = j0 + held
|
||||
for j in range(j0 + 1, j1 + 1): # bursts after the entry bar, up to and including the exit bar
|
||||
if bars["burst"][j]:
|
||||
against = d * (bars["C"][j] - bars["O"][j]) < 0
|
||||
mtm_before = (d * (bars["C"][j - 1] - t.entry) - COST) / risk
|
||||
mtm_after = (d * (bars["C"][j] - t.entry) - COST) / risk
|
||||
rows.append(dict(against=against, bars_open=j - j0, mtm_before=mtm_before, mtm_after=mtm_after,
|
||||
final=r_of(t.direction, t.entry, t.sl, xp), ended_here=(j == j1), hour=bars["chour"][j],
|
||||
sched=bars["chour"][j] in (0, 1, 12, 13)))
|
||||
break
|
||||
dmg = pd.DataFrame(rows)
|
||||
if len(dmg):
|
||||
g = dmg.groupby("against").agg(n=("final", "size"), bars_open_med=("bars_open", "median"),
|
||||
mtm_before=("mtm_before", "mean"), mtm_after=("mtm_after", "mean"),
|
||||
final=("final", "mean"), ended_on_that_bar_pct=("ended_here", lambda s: 100 * s.mean()),
|
||||
in_sched_hours_pct=("sched", lambda s: 100 * s.mean()))
|
||||
print(g.round(3).to_string())
|
||||
ag = dmg[dmg.against]
|
||||
print(f"\nopposing bursts: {len(ag)} trades ({100*len(ag)/len(book):.1f}% of the book); "
|
||||
f"mean R just before {ag.mtm_before.mean():+.3f} -> just after {ag.mtm_after.mean():+.3f} -> final {ag.final.mean():+.3f}; "
|
||||
f"{100*(ag.final<0).mean():.0f}% end as losers; {100*ag.ended_here.mean():.0f}% are stopped ON that bar")
|
||||
print(f" those with an open PROFIT before the burst (mtm>0): n={int((ag.mtm_before>0).sum())}, final mean {ag[ag.mtm_before>0].final.mean():+.3f}, "
|
||||
f"{100*(ag[ag.mtm_before>0].final<0).mean():.0f}% end as losers")
|
||||
print(f" by hour of the opposing burst (n, mean final R): " + ", ".join(f"{h:02d}:{n}/{m:+.2f}" for h, n, m in
|
||||
ag.groupby("hour").final.agg(["size", "mean"]).reset_index().itertuples(index=False)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,348 @@
|
||||
# Stake-sizing campaign — 2026-09-05 (3-year Oanda window, $0.65)
|
||||
|
||||
## Purpose and owner decisions
|
||||
Owner asked (2026-09-05): predict direction and choose order size better; protect against
|
||||
trend exhaustion at signal time; map hours/days where institutions move gold violently;
|
||||
backtests ≤ 3 years. Decisions taken in planning:
|
||||
- Order size = a per-signal RELATIVE stake ("Stake: 0.7× normal"), not account-based units.
|
||||
- Exhaustion protection = RISK LESS, DON'T SKIP (every hard exhaustion block tested since
|
||||
2026-07 removed net-positive trades — the winners are the stretched runs).
|
||||
- Model only M5 and M15; H1/H4 measured descriptively.
|
||||
- scikit-learn / lightgbm allowed in ./venv for research only; the shipped artefact is
|
||||
numpy-only JSON. Neither library enters requirements.txt.
|
||||
- No tuned threshold, gate or exit changes (rule 1). Sizing is a post-decision overlay computed
|
||||
by `position_sizing.build_features` / `stake_multiplier` INSIDE `analyze_from_dataframes`, so
|
||||
live and backtest use the same call on the same objects (rule 4).
|
||||
|
||||
## Harness
|
||||
`backtest/gpu_backtest.py --pair XAUUSD --start 2023-09-05 --end 2026-09-05 --spread 0.65
|
||||
--workers 12`, pair-factory config (no --set/--quality/--strategies/--confirm-tfs), NumPy CPU
|
||||
backend (CuPy cannot locate CUDA_PATH on this box; the 2026-09-03 pinned books were also CPU).
|
||||
Cache: `oanda_XAU_USD_{5min,15min,1h,4h}_20230705_20260905.parquet`, `…_1d_20221110_20260905`.
|
||||
Feature books add `--export-features` (65 `f_*` columns + `stake_multiplier` + `r_sized`; the
|
||||
default export stays the pinned 63-column shape). Outputs in `backtest/research/sizing_2026-09/`
|
||||
(gitignored).
|
||||
|
||||
## Baseline (control books, unmodified backtester, 2026-09-05 23:36–23:41 UTC)
|
||||
| TF | trades | total R | WR | PF | max DD |
|
||||
|---|---|---|---|---|---|
|
||||
| M5 | 857 | +161.32 | 53.7% | 1.53 | 9.85 |
|
||||
| M15 | 1086 | +148.69 | 59.1% | 1.37 | 15.47 |
|
||||
| H1 | 255 | +70.52 | 64.3% | 1.98 | 4.43 |
|
||||
| H4 | 113 | +22.68 | 58.4% | 1.63 | 5.39 |
|
||||
|
||||
### Behaviour-neutrality checks (2026-09-05 23:57 UTC)
|
||||
- 12-month M5 canonical window (2025-08-14 → 2026-08-14, $0.65) re-run on the modified tree:
|
||||
`book_diff` vs `backtest/oanda_m5_12mo_065_nopb_20260903.csv` = ΔR +0.0, Δtrades +0, and the
|
||||
two CSVs have the SAME MD5 (`a28fed4becd056466c26d6a947e1a0e5`). Record-keeping fields,
|
||||
`SizingConfig`, and the `--export-features` hook change nothing when the flag is off.
|
||||
- Feature books: the 63 shared columns are value-identical to the control books on every TF
|
||||
(`assert_frame_equal`). A column-ORDER quirk was found and fixed the same day: pandas orders
|
||||
export columns by first appearance, so when the first trade of a book lacks an indicator
|
||||
(the 3-year M5 book's first trade had no candlestick pattern) that indicator's columns were
|
||||
appended after the extras. `trades_to_dataframe` now forces the extras to the tail; test
|
||||
extended with that exact case (435 tests green).
|
||||
|
||||
## Pre-registered acceptance rule (written 2026-09-05, BEFORE any model fit)
|
||||
A per-timeframe stake-multiplier model may ship only if, evaluated on OUT-OF-SAMPLE quarters
|
||||
only (all calendar quarters after a 12-month burn-in, each predicted by a model trained
|
||||
exclusively on trades whose entry time is ≥ 300 primary bars before that quarter's start,
|
||||
expanding window, standardisation fit on train only), ALL of the following hold:
|
||||
1. Sized total R > flat total R in BOTH halves of the OOS period (split at the median OOS
|
||||
entry time).
|
||||
2. Sized total R > flat total R over the full OOS period.
|
||||
3. Sized max drawdown (0-prepended cumulative sized-R curve, as `compute_metrics`) ≤ flat max
|
||||
drawdown × 1.10.
|
||||
4. Sized OOS book has PF ≥ 1.3 and positive total R in both halves (the rule-1 floor).
|
||||
5. Real OOS lift > the 95th percentile of the lift distribution from 50 label-shuffle refits
|
||||
(R permuted within training windows only; test labels and walk-forward geometry unchanged).
|
||||
6. Mean multiplier over OOS trades within 1.00 ± 0.05; every multiplier in [0.5, 1.5].
|
||||
Plus the mechanical check: the `--sizing` full-window replay reproduces the flat book's
|
||||
entry/exit/`r_multiple` columns exactly (sizing changes no entries).
|
||||
Model family and size map (fixed): primary = ridge regression on winsorised R (clip to
|
||||
[−1.5, +4]); secondary = L2 logistic on R>0 (ships only if the primary fails and it passes);
|
||||
simplicity baseline = one-feature model on `f_is_buy` (the shipped model must beat its OOS
|
||||
lift, else the simpler one ships if it passes); upper bound = lightgbm on all `f_*` (never
|
||||
ships; if its OOS lift is inside the shuffle null, stop); ablation = exhaustion-block-only model.
|
||||
Map: `m = 0.5 + 1.0 × clip((z − q10)/(q90 − q10), 0, 1)` with q10/q90 from TRAIN predictions,
|
||||
times a normaliser making the mean train multiplier 1.0, clipped to [0.5, 1.5], rounded to 0.05.
|
||||
`SHIPPED_FEATURES` (≤ 12 per TF, ≥ 2 from the exhaustion block) are chosen from the Stage-B
|
||||
sign-stable screen on the 3-year feature books and recorded below BEFORE the first fit; any
|
||||
later change restarts the shuffle control. If any clause fails on a timeframe, that timeframe
|
||||
ships nothing. No widening the clip, changing the split, dropping the purge, re-picking
|
||||
features or adding quarters to rescue a result.
|
||||
|
||||
## Pre-registered feature lists
|
||||
Derived MECHANICALLY (script in the session log, saved as `sizing_2026-09/preregistered_features.json`):
|
||||
sign-stable `f_*` features from the Stage-B screen (top-minus-bottom quintile mean-R gap has the
|
||||
same sign in both halves, both halves ≥ 10 trades per bucket), zero-variance columns removed,
|
||||
ranked by min(|gap_h1|, |gap_h2|), top 12; the ≥ 2-from-the-exhaustion-block clause was already
|
||||
satisfied on both TFs. Written 2026-09-06 00:05 UTC, before any Stage-D fit.
|
||||
- **M5 (29 sign-stable candidates):** f_sl_atr, f_confidence, f_rsi, f_mtf_c1, f_swing_dist_atr,
|
||||
f_ret_48_atr, f_tag_no_pullback, f_rsi_dir, f_first_friday, f_mtf_agreement,
|
||||
f_exhaustion_count, f_ret_24_atr. Exhaustion-block members: f_swing_dist_atr,
|
||||
f_tag_no_pullback, f_exhaustion_count (all three run POSITIVE in both halves).
|
||||
- **M15 (37 sign-stable candidates):** f_rsi, f_bb_pos, f_day_of_month, f_tag_ema200_near,
|
||||
f_atr_pct, f_is_buy, f_confidence, f_close_pos_in_bar, f_tag_weak_pullback, f_tag_exhaustion,
|
||||
f_penalty_total, f_ret_6_atr. Exhaustion-block members: f_tag_exhaustion, f_penalty_total
|
||||
(both NEGATIVE in both halves; f_tag_weak_pullback and f_tag_ema200_near also negative).
|
||||
Exhaustion-only ablation list (fixed by the plan): f_swing_dist_atr, f_ema21_dist_atr,
|
||||
f_ema200_dist_atr, f_exhaustion_count, f_penalty_total, f_tag_bb_overextended,
|
||||
f_tag_rsi_extended, f_tag_no_pullback, f_tag_exhaustion.
|
||||
|
||||
## Stage B — book breakdown (3-year feature books)
|
||||
Tool: `backtest/research/book_report.py` (outputs in `sizing_2026-09/report_{5min,15min,1h,4h}/`).
|
||||
Halves = median TRADE split (M5 cut 2025-12-12 because trade density doubled in the last year;
|
||||
M15 cut 2025-04-25; H1 2025-04-09).
|
||||
|
||||
**M5 (857 trades, +161.3R, PF 1.53, DD 9.85; halves +51.0R PF 1.33 / +110.3R PF 1.74; max 11
|
||||
wins / 8 losses in a row; longest flat stretch 88 trades).** Mix: GENERIC 469, XAU_CP 388, BUY
|
||||
765 / SELL 92 (SELLs +11.8R, positive both halves). Grade: A +0.26R/trade (227) > B +0.17 (608)
|
||||
> C −0.01 (21). Session (decision hour): NY 17–22 best (+0.31R/trade, PF 1.88), overlap 13–16
|
||||
weakest (+0.08, PF 1.22) — both stable. Weekday: Wed and Fri best and stable; Mon/Tue flip sign.
|
||||
Feature screen: 29 sign-stable `f_*` features. **Every stretch measure is POSITIVE**: wider
|
||||
stop in ATR (+0.22/+0.23 gap), higher RSI (+0.20/+0.19), farther from the last swing
|
||||
(+0.18/+0.27), 48-bar run-up (+0.18/+0.20), the `no_pullback` tag (+0.17/+0.28), more exhaustion
|
||||
tags (+0.12/+0.24). Only ATR-relative-to-500-bar-mean is negative (−0.11/−0.12: entries during
|
||||
a volatility spike do worse). "Risk less when stretched" has NO support on M5 — a model that
|
||||
honours the data will size stretched M5 trades UP, not down.
|
||||
|
||||
**M15 (1086 trades, +148.7R, PF 1.37, DD 15.47; halves +65.8R / +82.9R; max 12 wins / 8 losses;
|
||||
flat stretch 131 trades).** All GENERIC; BUY 739 (+158.4R, PF 1.62) vs SELL 347 (−9.7R, PF 0.93;
|
||||
−19.2R in half 1, +9.4R in half 2). Grade NOT monotone: C +0.45R/trade (82, PF 2.41) > A +0.14
|
||||
(481) > B +0.08 (516) — stable in both halves (matches the 2026-08-04 quant review). Session:
|
||||
Asian and NY ≈ +0.18–0.20R/trade, London and overlap ≈ +0.09–0.10 (all stable). Weekday:
|
||||
Wednesday flat/negative in both halves (229 trades, −0.6R); Thu/Fri flip sign between halves.
|
||||
Feature screen: 37 sign-stable features. Here the exhaustion block runs NEGATIVE: `exhaustion`
|
||||
tag (−0.19/−0.10), `weak_pullback` tag (−0.24/−0.11), `ema200_near` tag (−0.17/−0.32), penalty
|
||||
total (−0.16/−0.10), ATR percentile (−0.20/−0.15); while raw RSI (+0.31/+0.24) and Bollinger
|
||||
position (+0.36/+0.22) are positive. So on M15 "stretched by the aggregator's own penalty
|
||||
logic" is bad while "strong momentum reading" is good — the sizing model gets both.
|
||||
|
||||
**H1 (255 trades, +70.5R, PF 1.98, DD 4.43; halves +28.7R PF 1.79 / +41.9R PF 2.17)** and
|
||||
**H4 (113 trades, +22.7R, PF 1.63, DD 5.39)** are descriptive only. H1 is all-BUY by
|
||||
construction (shorts blocked 2026-09-03); Mon/Thu strongest, Friday the only losing weekday
|
||||
(−1.4R/50, flips between halves); 03:00 UTC decision hour is the best single hour (+20.0R/40,
|
||||
stable) and 04:00–05:00 negative.
|
||||
|
||||
**Labelling quirk (pre-existing, not fixed):** 33 M5 trades carry the backtester `strategy`
|
||||
tag GENERIC while their setup strategy is XAU_ATR (`f_strat_atr = 1`); the CSV tag and the
|
||||
setup's own string disagree for the ATR path. Labels only — R is unaffected.
|
||||
|
||||
**Stop-trigger check: PASSED** (both TFs have many sign-stable features) → proceed to Stage D.
|
||||
|
||||
## Stage C — institutional burst / time map
|
||||
Tool: `backtest/research/burst_map.py` on the 3-year raw bars (`burst_5min/`, `burst_15min/`
|
||||
under `sizing_2026-09/`). Burst = bar range ≥ K × prior Wilder ATR(14). Base rate at K=3:
|
||||
0.86 % of M5 bars (1840), 1.19 % of M15 bars (846).
|
||||
|
||||
**Where the violent bars cluster (bar-CLOSE hour, UTC, K=3 burst % of bars):**
|
||||
- 12:00–13:00 (US data window): M5 2.4 % / 2.3 %, M15 3.7 % / 5.0 % — 3–4× the base rate;
|
||||
Friday 12:00–13:00 is the densest weekday cell on M5 (3.4 % / 3.1 %). Largest mean 1-hour
|
||||
dollar moves are at close-hours 13–15 (M5: $7.8 / $9.4 / $8.5 vs ≈$5 elsewhere).
|
||||
- 01:00 (Asia open): M5 2.9 %, M15 3.3 %. Also 05:00 (Tokyo/London hand-over) ≈1.2 % on M5.
|
||||
- 22:00–23:00 (post-settlement thin book): ≈1.3–1.4 % on M5; Sunday reopen 22–23 h is the most
|
||||
burst-prone slot of all (2.4–3.0 % of bars) but the calendar already blocks Sunday entries
|
||||
before 18:00 NY.
|
||||
- Quietest hours: 03:00–04:00 and 16:00–17:00 (0.14–0.3 %).
|
||||
|
||||
**Post-burst drift (in the burst's own direction, ATR units):** ≈ 0 at every horizon on both
|
||||
TFs (M5 6/12/24/48-bar means −0.01/−0.01/+0.09/+0.05; M15 −0.08/+0.14/+0.18/+0.17; % positive
|
||||
46–51 %). London-session bursts on M5 show a mild short-horizon reversal (42 % positive at 6
|
||||
bars). No continuation or reversal edge worth a rule.
|
||||
|
||||
**Do bursts hurt the system's entries? No.** Trades DECIDED within 12 bars after a K=3 burst:
|
||||
M5 149 trades, mean +0.199R, PF 1.62 vs +0.186R / PF 1.52 for the rest; M15 242 trades,
|
||||
+0.160R / PF 1.47 vs +0.130R / PF 1.34. At K=4/5 the after-burst group is better still
|
||||
(M5 +0.276R PF 1.98; M15 +0.259R PF 1.80). This re-confirms the 2026-07-30 big-candle-gate
|
||||
rejection with 3 years of data: expansion bars are part of the entry signal, not a danger.
|
||||
(Group (b) "burst while open" is survivorship-shaped — winners live longer — and is not evidence.)
|
||||
|
||||
**The 40 largest 1-hour moves** are almost all one episode (2026-01-28 → 2026-02-06, incl. a
|
||||
−$394 hour at 15:45 UTC on 2026-01-29, and 2026-03-23), not recurring clock slots; only 3 of 40
|
||||
fall in the 12:30 data slot and 1 in the Wednesday 18:00 central-bank slot (slot labels only —
|
||||
no economic calendar file exists in the repo). Violent moves are episodic regime events; the
|
||||
recurring, predictable clustering is the 12:00–15:00 UTC US-data window and the 01:00 Asia open.
|
||||
|
||||
**Use in this campaign:** `f_bars_since_burst` and the hour/session features are already in the
|
||||
feature set, so Stage D can weigh them as a stake input; no entry gate is proposed (rule 1).
|
||||
|
||||
## Stage D — walk-forward results (2026-09-06 00:20 UTC)
|
||||
Driver: `backtest/research/sizing_model.py` (numpy closed-form ridge + Newton logistic; lightgbm
|
||||
4.7 + scikit-learn 1.9 installed in ./venv for the upper bound only; NOT in requirements.txt).
|
||||
Outputs: `sizing_2026-09/sm_{m5,m15}_{ridge,logistic}/` (summary.txt, oos_trades.csv,
|
||||
quarters.csv, coefs_by_quarter.csv, shuffle_lifts.csv). Validation of the driver: planted
|
||||
synthetic effect recovered (lift +116.7R, shuffle rank 100 %, all clauses PASS); pure-noise
|
||||
book correctly FAILS (rank 32 %); ridge matches `lstsq` to 2e-16; logistic gradient < 3e-15.
|
||||
|
||||
**M5 (5 OOS quarters 2025Q3→2026Q3, 619 trades — 2024Q4–2025Q2 skipped by the pre-registered
|
||||
min-train rule because only 96/119/142 trades precede them; 2026Q1 alone is +92R of the +125R
|
||||
flat OOS):**
|
||||
| model | flat R | sized R | lift | PF | DD | mean m | shuffle p95 | rank | verdict |
|
||||
|---|---|---|---|---|---|---|---|---|---|
|
||||
| ridge (primary) | +125.41 | +121.98 | −3.43 | 1.63 | 10.69 | 0.891 | +13.69 | 48 % | FAIL (1,2,5,6) |
|
||||
| logistic | +125.41 | +128.40 | +2.99 | 1.66 | 8.70 | 0.900 | +14.39 | 62 % | FAIL |
|
||||
| isbuy baseline | +125.41 | +123.22 | −2.20 | 1.55 | 11.92 | 1.000 | +7.05 | 4 % | FAIL |
|
||||
| exhaustion-only | +125.41 | +118.03 | −7.38 | 1.59 | 10.88 | 0.918 | +14.36 | 28 % | FAIL |
|
||||
| lightgbm (all 65) | +125.41 | +116.05 | −9.36 | 1.51 | 11.64 | 1.007 | +7.60 | 10 % | FAIL |
|
||||
Ridge weights (standardised, 5 quarters): `f_tag_no_pullback` +0.018 sign-stable 5/5 — the
|
||||
"no pullback" flag INCREASES the stake; `f_swing_dist_atr` and `f_exhaustion_count` ≈ 0 and
|
||||
sign-unstable. Exhaustion-only sizing LOSES 7.4R. **A clean null: nothing in the 65 features
|
||||
predicts M5 trade outcomes beyond noise, linear or non-linear.**
|
||||
|
||||
**M15 (8 OOS quarters 2024Q4→2026Q3, 762 trades; flat +115.93R PF 1.42 DD 13.73):**
|
||||
| model | sized R | lift | PF | DD | mean m | shuffle p95 | rank | clauses |
|
||||
|---|---|---|---|---|---|---|---|---|
|
||||
| ridge (primary) | +142.04 | +26.11 | 1.53 | 14.46 | 0.980 | +28.50 | 88 % | 1,2,3,4,6 PASS · **5 FAIL** |
|
||||
| logistic | +144.78 | +28.84 | 1.53 | 13.97 | 1.002 | +31.89 | 92 % | 1,2,3,4,6 PASS · **5 FAIL** |
|
||||
| isbuy baseline | +139.38 | +23.45 | 1.50 | 16.44 | 1.007 | +24.47 | 58 % | FAIL (3, 5) |
|
||||
| exhaustion-only | +123.83 | +7.90 | 1.45 | 14.85 | 1.010 | +19.77 | 70 % | FAIL |
|
||||
| lightgbm (all 65) | +123.87 | +7.94 | 1.44 | 14.74 | 0.996 | +17.66 | 60 % | FAIL |
|
||||
Ridge weights sign-stable in 8/8 quarters: `f_tag_exhaustion` −0.022, `f_tag_ema200_near`
|
||||
−0.033 (both SHRINK the stake — the "risk less when stretched" direction), `f_rsi` +0.028,
|
||||
`f_bb_pos` +0.030, `f_is_buy` +0.029, `f_close_pos_in_bar` +0.022, `f_ret_6_atr` +0.022;
|
||||
λ = 1000 every quarter (heavy shrinkage — the score is weak and the map is essentially a rank
|
||||
transform). Sized R beat flat in 6 of 8 quarters and in both halves, BUT the label-shuffle
|
||||
null has p95 = +28.5R: a random re-weighting of these 12 pre-screened features produces the
|
||||
same lift 12 % of the time, and the one-feature BUY/SELL baseline already delivers +23.5R of
|
||||
the +26.1R. **The model's information beyond "buys are better than sells" (≈ +2.7R over 762
|
||||
trades) is indistinguishable from noise.**
|
||||
|
||||
## Verdict (pre-registered rule applied, no rescue attempted)
|
||||
**FAIL on both timeframes → no stake multiplier ships.** No artifact, no `sizing.enabled`, no
|
||||
alert line, no `--sizing` backtester flag. What remains in the tree is behaviour-neutral
|
||||
research plumbing: `position_sizing.build_features` (65 decision-time features; the multiplier
|
||||
path returns 1.0 and no artifact exists), `SignalReport.penalty_tags/penalty_total/
|
||||
momentum_fading` (record-keeping), `TradeSetup.stake_multiplier` (None; reserved), the
|
||||
`--export-features` flag, `book_report.py`, `burst_map.py`, `sizing_model.py`, and Module 29
|
||||
tests. `SizingConfig` was REMOVED again before merge (rule 8: nothing reads it).
|
||||
|
||||
**What the owner should take from this (plain language):**
|
||||
1. On the 5-minute chart there is nothing to size on. Stretched trades are the good ones; the
|
||||
system's exhaustion flags carry no usable information about the outcome; a flexible model
|
||||
given all 65 facts does worse than flat sizing out of sample.
|
||||
2. On the 15-minute chart the only robust sizing signal is BUY vs SELL, which the gates already
|
||||
act on. The exhaustion flag does point the "risk less" way in every one of 8 unseen
|
||||
quarters, but the money it would make (≈ +3R per year) is inside noise at 1,086 trades.
|
||||
3. "Predict direction better" with a learned second opinion is closed at this sample size:
|
||||
the strongest non-linear model (all features, gradient boosting) fails on both charts.
|
||||
|
||||
**Open avenues (not rescues — each needs a NEW pre-registration):**
|
||||
- Re-run the M15 exhaustion-tag sizing when the book has ≈ 2× the trades (≈ 2028) or on a
|
||||
sealed forward sample; the sign is stable, the magnitude is not proven.
|
||||
- The shuffle null is wide because the size map rank-stretches a weak score to 0.5–1.5×; a
|
||||
narrower map (e.g. 0.75–1.25×) would shrink both lift and null — pre-register before testing.
|
||||
- Session and weekday structure (M5: NY best / overlap weakest; M15: Wednesday flat) is stable
|
||||
across halves but was rejected as a gate repeatedly; as a *size* input it was inside the
|
||||
12-feature model and did not survive the null.
|
||||
|
||||
## Addendum 2026-09-06 — violent bars: signs before them, and protecting OPEN trades
|
||||
Owner follow-up: "we might have an order running for hours before a violent move; we want
|
||||
protection." Tools: `backtest/research/burst_precursors.py` (signs before a burst) and
|
||||
`backtest/research/open_trade_bursts.py` (exact replay of every book trade on raw bars with the
|
||||
backtester's exit model — reproduces `r_multiple` on 100 % of trades — plus protection variants).
|
||||
Outputs: `sizing_2026-09/burst_precursors_{m5,m15}.txt`, `open_trade_bursts_{m5,m15}.txt`.
|
||||
|
||||
**Signs before a burst (both TFs, both halves stable):** the clock (close-hours 01, 12, 13 UTC
|
||||
hold ~40 % of bursts in 13 % of bars); tick activity in the last 3 bars vs 50-bar mean (top
|
||||
quintile ≈ 2× rate); the last bars already wider than normal (top quintile ≈ 2.2×; the
|
||||
tightest quintile ≈ 0.3–0.5× — NOT calm-before-storm at bar scale); a move already under way
|
||||
over the past hour (≈ 1.6×); a burst minutes ago (≈ 1.4×). Out of sample (fit half 1, score
|
||||
half 2) a logistic model reaches AUC 0.75–0.77 for the next bar; the top 5 % of moments carry a
|
||||
3.4–5.5 % burst chance vs 0.8–1.1 % base, and 24 % vs 9 % (M5) for "within the next hour".
|
||||
Burst direction continues the prior hour's drift 61 % of the time. Signs say "ripe", not "now".
|
||||
|
||||
**Damage to open trades (base exits):** M5 — 164 trades (19 %) meet an opposing burst while
|
||||
open: mean open R +0.42 just before → −0.06 just after → final +0.01; 43 % are stopped on that
|
||||
bar; 54 % end as losers. M15 — 246 trades (23 %): +0.27 → −0.20 → final −0.12; 39 % stopped on
|
||||
the bar. Trades already in profit before the burst still finish positive on average (M5 +0.53,
|
||||
M15 +0.27). Bursts WITH the trade (n 176 / 246) finish at +1.11 / +0.83R — they are where the
|
||||
winners come from.
|
||||
|
||||
**Protection variants — ALL negative on both TFs, both halves:**
|
||||
| variant | M5 ΔR (h1/h2) | M15 ΔR (h1/h2) |
|
||||
|---|---|---|
|
||||
| flatten before US-data hours (12–13 UTC) | −31.9 (−13.6/−18.3) | −103.0 (−67.4/−35.6) |
|
||||
| flatten before Asian open (00–01 UTC) | −38.7 (−22.0/−16.7) | −73.0 (−42.0/−30.9) |
|
||||
| protective 0.5R stop through the data hours | −29.6 (−12.3/−17.3) | −82.3 (−55.5/−26.8) |
|
||||
| protective 0.5R stop through the Asian open | −19.2 (−9.5/−9.7) | −35.1 (−17.3/−17.8) |
|
||||
| exit next open after an opposing burst | −19.1 (−6.5/−12.5) | −6.9 (−11.9/+5.0) |
|
||||
| 0.5R protective stop after an opposing burst | −9.0 (−3.0/−6.0) | −8.3 (−15.5/+7.3) |
|
||||
The existing stop/trail already contains the opposing-burst damage (≈ 0.4–0.5R per hit); every
|
||||
pre-emptive rule removes far more from the with-trade bursts than it saves. **Verdict: no
|
||||
open-trade protection rule ships; do not re-propose flatten/tighten-before-window or
|
||||
react-to-burst exits without a new mechanism.** Display-only "violent-move conditions" note on
|
||||
alerts is the only harmless use of the precursor signs.
|
||||
|
||||
## Addendum 2026-09-06 — calendar events, indicator states and time slots before violent bars
|
||||
Tools: `backtest/research/calendar_impact.py` (needs `backtest/cache/calendar.parquet`, scraped
|
||||
2023-09-01→2026-09-10 with `ff_calendar_scraper.py`: 15,124 events, 1,055 high-impact USD in
|
||||
window) and `backtest/research/burst_indicator_map.py` (states from the backtester's own
|
||||
`precompute_indicators`, taken at t−1). Outputs `sizing_2026-09/calendar_impact_{m5,m15}.txt`,
|
||||
`burst_indicator_map_{m5,m15}.txt`. Burst = range ≥ 3× prior ATR14; lifts quoted are stable in
|
||||
BOTH halves on both TFs unless noted.
|
||||
|
||||
**Calendar.** The M5 bar containing a high-impact USD release averages 4.4× normal range; 54 %
|
||||
of such releases produce a burst within the hour (base 9 %). By event: Non-Farm Payrolls (with
|
||||
Unemployment Rate / Avg Hourly Earnings) 97 %, 9.3× bar, $11.9 mean 1-h move; CPI 91 %, 8.3×,
|
||||
$9.0; FOMC decision/statement 87.5 %, 7.1×, $18.2 (press conference a further $20 hour); FOMC
|
||||
projections 92 %; Retail Sales 64 %; PPI 54 %; Core PCE 50 %; Unemployment Claims 44 %; GDP
|
||||
45–58 %; ISM PMIs 26 %; Powell speaks 33 %; Trump speaks 21 %; UoM sentiment ≈ 5 %. 20 % (M5)
|
||||
/ 31 % (M15) of ALL bursts fall within ±30 min of a high-impact USD release (3.7–4.2 % of time);
|
||||
in the 30 min AFTER one, next-bar burst odds are 8–17 % (7–16× base); 15 min before, 2–3 %.
|
||||
Open trades that held through a high-impact release did BETTER (M5 +0.41R vs +0.16R; M15 +0.27
|
||||
vs +0.07), even when the first 15-min move went against them (M5 +0.09, M15 +0.22). Flattening
|
||||
5 min before a release: −5.4R (M5) / −59.3R (M15); 0.5R protective stop through the hour:
|
||||
−2.4R / −43.1R. **Events are the biggest single cause of violent bars and still not a reason
|
||||
to touch an open trade.**
|
||||
|
||||
**Indicator states before a burst (next-bar lift, M5 / M15).** Price OUTSIDE the Bollinger
|
||||
bands: below 2.9× / 1.8–2.9×, above 2.1× / 2.2×, middle 0.2–0.8 band 0.7×. RSI < 30: 2.6–2.9× /
|
||||
1.8–2.2×; RSI ≥ 70: 1.9–2.1× / 1.6–1.9×; RSI 40–60: 0.7× (calmest). Close outside the Keltner
|
||||
channel: 1.4–2.2×; inside 0.76×. ADX RISING > 3: 1.4–1.5× / 1.2×; ADX FALLING < −3: 0.5–0.6×
|
||||
(safest state); the ADX level itself carries nothing. MACD histogram negative and still falling
|
||||
1.3× / 1.2–1.4×; histogram fading (either sign) 0.8×. Fully bearish 4-EMA stack 1.3–1.4× /
|
||||
1.3×; "mostly bull" 0.8× on M5. Regime: transitioning 1.4–1.6× / 1.5–1.6×, ranging 0.88×.
|
||||
Volatility regime: extreme 1.5–1.6× / 1.4–1.9×, low 0.6–0.8×. ROC beyond ±0.3 %: 1.2–1.9×
|
||||
(>1 %: 1.3–10×, n small). Bollinger-width squeeze: 0.59× on M5 (LESS likely next bar — narrow
|
||||
bands are not a warning); TTM squeeze (bands inside Keltner) 1.6× on M15 only. EMA200 distance,
|
||||
day of month, first-Friday-as-a-day: nothing stable (first Friday matters only through its
|
||||
12:30 slot).
|
||||
|
||||
**Time slots (M5 next-bar burst %, base 0.86 %):** Fri 01:xx 3.4 %, Fri 12:xx 3.4 %, Tue/Wed
|
||||
01:xx 3.2 %, Thu 12:xx 3.1 %, Fri 13:xx 3.1 %, Sun 22:xx reopen 3.0 %, Thu 01:xx 2.7 %, Wed
|
||||
12:xx 2.6 %, Sun 23:xx 2.4 %, Tue 00:xx 2.3 % (all 2.5–3.9× base). Quietest: 03–04 and 16–17.
|
||||
|
||||
**Direction of the burst (share going UP):** fully-bull stack 62 % (M5) / 50 % (M15); fully-bear
|
||||
stack 26 % / 29 %; RSI < 30 → 22 % / 30 % up (i.e. 70–78 % DOWN); RSI ≥ 70 → 64 % / 52 % up;
|
||||
MACD neg & falling → 30 % / 34 % up; trending_down regime → 26 % / 26 % up; trending_up → 60 % /
|
||||
46 % up. Bearish states foretell a DOWN burst reliably on both TFs; bullish states foretell an
|
||||
UP burst on M5 only.
|
||||
|
||||
**Reading:** violent bars arrive when price is already at an extreme with momentum accelerating
|
||||
(outside the bands, RSI extreme, ADX rising, MACD still falling, transitioning regime, extreme
|
||||
volatility) and/or a release is minutes away — "the storm is already brewing", not "calm before
|
||||
the storm". Calm mid-band bars with fading momentum and falling ADX are the safest. None of it
|
||||
changes trade handling (protection tests above); it is display material for a warning line.
|
||||
|
||||
## Addendum 2026-09-06 — violent-move WATCH rule (display-only feature, `violent_watch.py`)
|
||||
Owner asked for a warning. Delivery: a line on signal alerts + a standalone heads-up to the owner
|
||||
(once per hour per trigger set, daily cap), dark by default (`violent_watch` config block).
|
||||
The first draft rule ("hot" = calendar | ≥3 indicator readings | clock & ≥2 readings) was
|
||||
evaluated bar-by-bar on 3 years (`backtest/research/watch_rule_eval.py`, scratch variant
|
||||
sweep): it flagged 27 % of bars hot at only ≈2× odds and would have sent ≈13 heads-ups/day on
|
||||
M5 — rejected. Sweep of 11 variants (M5 / M15, both halves):
|
||||
| hot definition | bars hot | next-bar lift h1/h2 | next-hour rate | hot hours/day |
|
||||
|---|---|---|---|---|
|
||||
| calendar only (hi-impact USD ±30 min) | 3.4 % | 6.7×/4.4× · 9.1×/6.4× | 31 % · 28 % | 2.6 |
|
||||
| clock slot only | 18.8 % | 3.1×/2.0× · 3.6×/2.6× | 16 % · 13 % | 5.3 |
|
||||
| ≥2 price-extreme readings | 14.6 % | 2.3×/2.2× · 1.8×/2.0× | 11 % · 6 % | 10.7 |
|
||||
| **calendar ∨ (clock ∧ ≥1 price-extreme reading)** | **8.6 %** | **4.9×/3.1× · 5.9×/4.2×** | **21 % · 17 %** | **4.1 · 3.5** |
|
||||
| draft (cal ∨ readings≥3 ∨ clock∧readings≥2) | 26.9 % | 2.4×/2.0× · 2.4×/2.1× | 13 % · 8 % | 13.0 |
|
||||
**Shipped rule:** price-extreme readings = outside the Bollinger bands, RSI < 30 or ≥ 70, close
|
||||
outside the Keltner channel (the neutral readings — ADX rising, extreme volatility, transitioning
|
||||
— are dropped from the level logic; they are common and only ≈1.5×). `hot` = high-impact USD
|
||||
release within 30 min or 30 min after, OR a hot clock slot with ≥ 1 price-extreme reading; `watch`
|
||||
= clock slot alone or ≥ 2 price-extreme readings (≈2× odds). Direction sentence from the bear/bull
|
||||
polarity of the readings plus MACD acceleration and trend regime ("DOWN" on both TFs, "UP" on M5
|
||||
only). Heads-up only at `hot`. Expected ≈4 hot hours/day → ≤4 heads-ups/day after per-hour dedup.
|
||||
@@ -0,0 +1,961 @@
|
||||
"""Stage-D stake-sizing model driver (2026-09 campaign).
|
||||
|
||||
Walk-forward fit + evaluation of a per-signal stake multiplier on a trade
|
||||
book exported by ``gpu_backtest.py --export-features`` (63 base columns +
|
||||
``stake_multiplier``/``r_sized`` + 65 ``f_*`` decision-time features).
|
||||
|
||||
The protocol implemented here is the one pre-registered in
|
||||
``backtest/research/sizing_campaign_2026-09.md`` and is NOT to be varied to
|
||||
rescue a result:
|
||||
|
||||
* expanding walk-forward, 12-month burn-in, calendar-quarter refits, a
|
||||
``purge_bars × bar_minutes`` gap between the end of train and the start of
|
||||
the test quarter, standardisation fit on train only;
|
||||
* models: ridge on winsorised R (primary), L2 logistic on ``R > 0``
|
||||
(secondary), a one-feature ``f_is_buy`` simplicity baseline, an
|
||||
exhaustion-block-only ablation, and an optional lightgbm upper bound;
|
||||
* size map ``m = round_0.05(clip(normaliser × (0.5 + clip((z−lo)/(hi−lo),0,1)),
|
||||
0.5, 1.5))`` with ``lo``/``hi``/``normaliser`` calibrated on TRAIN
|
||||
predictions only;
|
||||
* a label-shuffle null (training labels permuted, test labels untouched);
|
||||
* six acceptance clauses, printed PASS/FAIL individually.
|
||||
|
||||
Usage:
|
||||
python backtest/research/sizing_model.py --book BOOK.csv --bar-minutes 5 \
|
||||
--features f_a,f_b,... [--model ridge|logistic] [--out DIR] \
|
||||
[--n-shuffle 50] [--skip-lgbm] \
|
||||
[--write-artifact PATH --tf 5min --confirm-tfs 15min,1h]
|
||||
|
||||
Analysis tooling only — never imported by the live code.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import subprocess
|
||||
import sys
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Sequence, Tuple
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from position_sizing import FEATURE_ORDER, FEATURE_VERSION # noqa: E402 (read-only import)
|
||||
|
||||
# ── Pre-registered constants (flag any change loudly) ──────────────────
|
||||
WINSOR_LO, WINSOR_HI = -1.5, 4.0
|
||||
LAMBDA_GRID: Tuple[float, ...] = (1.0, 10.0, 100.0, 1000.0)
|
||||
CV_FOLDS = 5
|
||||
MIN_TRAIN = 150
|
||||
MAP_Q_LO, MAP_Q_HI = 0.10, 0.90
|
||||
MAP_OUT_LO, MAP_OUT_HI = 0.5, 1.5
|
||||
MAP_CLIP_LO, MAP_CLIP_HI = 0.5, 1.5
|
||||
MAP_ROUND_TO = 0.05
|
||||
MEAN_MULT_TOL = 0.05
|
||||
DD_TOL = 1.10
|
||||
PF_FLOOR = 1.3
|
||||
NEWTON_MAX_ITER = 50
|
||||
NEWTON_TOL = 1e-8
|
||||
|
||||
#: The exhaustion-block ablation's fixed feature list.
|
||||
EXHAUSTION_FEATURES: Tuple[str, ...] = (
|
||||
"f_swing_dist_atr",
|
||||
"f_ema21_dist_atr",
|
||||
"f_ema200_dist_atr",
|
||||
"f_exhaustion_count",
|
||||
"f_penalty_total",
|
||||
"f_tag_bb_overextended",
|
||||
"f_tag_rsi_extended",
|
||||
"f_tag_no_pullback",
|
||||
"f_tag_exhaustion",
|
||||
)
|
||||
ISBUY_FEATURES: Tuple[str, ...] = ("f_is_buy",)
|
||||
|
||||
#: lightgbm never ships; running the full shuffle null through 200-tree fits
|
||||
#: costs minutes for a number nobody acts on, so its null is capped here.
|
||||
#: DEVIATION FROM THE PROTOCOL CONSTANT — printed in the summary when used.
|
||||
LGBM_MAX_SHUFFLE = 10
|
||||
|
||||
LGBM_PARAMS = dict(
|
||||
n_estimators=200,
|
||||
learning_rate=0.03,
|
||||
num_leaves=7,
|
||||
min_child_samples=30,
|
||||
subsample=0.8,
|
||||
colsample_bytree=0.8,
|
||||
reg_lambda=10,
|
||||
)
|
||||
|
||||
_LINES: List[str] = []
|
||||
|
||||
|
||||
def say(msg: str = "") -> None:
|
||||
"""Print and tee into the summary buffer."""
|
||||
print(msg)
|
||||
_LINES.append(msg)
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# Book loading
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
def load_book(path: Path) -> pd.DataFrame:
|
||||
"""Read a feature book, parse ``entry_time`` UTC, drop NaN-R rows."""
|
||||
df = pd.read_csv(path)
|
||||
if "entry_time" not in df.columns or "r_multiple" not in df.columns:
|
||||
raise SystemExit(f"{path}: not a trade book (no entry_time/r_multiple)")
|
||||
df["entry_time"] = pd.to_datetime(df["entry_time"], utc=True, format="mixed")
|
||||
n0 = len(df)
|
||||
bad = ~np.isfinite(pd.to_numeric(df["r_multiple"], errors="coerce").to_numpy(float))
|
||||
if bad.any():
|
||||
say(f"WARNING: dropping {int(bad.sum())}/{n0} rows with non-finite r_multiple")
|
||||
df = df.loc[~bad].copy()
|
||||
df["r_multiple"] = df["r_multiple"].astype(float)
|
||||
df = df.sort_values("entry_time", kind="mergesort").reset_index(drop=True)
|
||||
return df
|
||||
|
||||
|
||||
def resolve_features(df: pd.DataFrame, names: Sequence[str], label: str) -> List[str]:
|
||||
"""Validate a requested feature list against the book and FEATURE_ORDER."""
|
||||
missing = [n for n in names if n not in df.columns]
|
||||
if missing:
|
||||
raise SystemExit(f"{label}: columns absent from book: {', '.join(missing)}")
|
||||
unknown = [n for n in names if n.startswith("f_") and n not in FEATURE_ORDER]
|
||||
if unknown:
|
||||
say(f"NOTE: {label}: {', '.join(unknown)} not in position_sizing.FEATURE_ORDER")
|
||||
return list(names)
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as fh:
|
||||
for chunk in iter(lambda: fh.read(1 << 20), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def git_commit() -> str:
|
||||
try:
|
||||
return subprocess.check_output(
|
||||
["git", "rev-parse", "HEAD"], cwd=PROJECT_ROOT, text=True).strip()
|
||||
except Exception: # pragma: no cover - research tool
|
||||
return "unknown"
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# Walk-forward geometry
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
def quarter_starts(first: pd.Timestamp, last: pd.Timestamp) -> List[pd.Timestamp]:
|
||||
"""Every Jan/Apr/Jul/Oct 1 00:00 UTC in ``[first, last]`` inclusive."""
|
||||
out: List[pd.Timestamp] = []
|
||||
year = first.year
|
||||
while True:
|
||||
for month in (1, 4, 7, 10):
|
||||
ts = pd.Timestamp(year=year, month=month, day=1, tz="UTC")
|
||||
if ts < first:
|
||||
continue
|
||||
if ts > last:
|
||||
return out
|
||||
out.append(ts)
|
||||
year += 1
|
||||
|
||||
|
||||
def quarter_label(ts: pd.Timestamp) -> str:
|
||||
return f"{ts.year}Q{(ts.month - 1) // 3 + 1}"
|
||||
|
||||
|
||||
def build_folds(times: np.ndarray, n_folds: int) -> List[np.ndarray]:
|
||||
"""Contiguous-by-time index blocks (rows are already time-sorted)."""
|
||||
n = len(times)
|
||||
edges = np.linspace(0, n, n_folds + 1).astype(int)
|
||||
return [np.arange(edges[i], edges[i + 1]) for i in range(n_folds)
|
||||
if edges[i + 1] > edges[i]]
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# Standardisation
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
def fit_standardiser(X: np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
|
||||
"""Return (mean, scale, keep) from a training matrix.
|
||||
|
||||
NaNs are ignored when computing the mean (they are later filled with it).
|
||||
``scale`` is the population std with zeros replaced by 1.0; ``keep`` is
|
||||
False for zero-variance-in-train columns, which are dropped from the fit
|
||||
and carry a coefficient of 0.
|
||||
"""
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", category=RuntimeWarning)
|
||||
mean = np.nanmean(X, axis=0)
|
||||
std = np.nanstd(X, axis=0)
|
||||
mean = np.where(np.isfinite(mean), mean, 0.0)
|
||||
std = np.where(np.isfinite(std), std, 0.0)
|
||||
keep = std > 0
|
||||
scale = np.where(keep, std, 1.0)
|
||||
return mean, scale, keep
|
||||
|
||||
|
||||
def apply_standardiser(X: np.ndarray, mean: np.ndarray, scale: np.ndarray) -> np.ndarray:
|
||||
"""Fill NaN with the TRAIN mean, then standardise."""
|
||||
Z = np.array(X, dtype=float, copy=True)
|
||||
bad = ~np.isfinite(Z)
|
||||
if bad.any():
|
||||
Z[bad] = np.take(mean, np.where(bad)[1])
|
||||
return (Z - mean) / scale
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# Models (numpy only)
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
def _solve(A: np.ndarray, b: np.ndarray) -> np.ndarray:
|
||||
try:
|
||||
return np.linalg.solve(A, b)
|
||||
except np.linalg.LinAlgError: # pragma: no cover - degenerate design
|
||||
return np.linalg.lstsq(A, b, rcond=None)[0]
|
||||
|
||||
|
||||
def fit_ridge(X: np.ndarray, y: np.ndarray, lam: float) -> Tuple[np.ndarray, float]:
|
||||
"""Closed-form ridge with an UNPENALISED intercept.
|
||||
|
||||
Centring X and y is exactly equivalent to appending a 1-column carrying
|
||||
zero penalty. Returns ``(coef, intercept)`` with ``pred = intercept + X @ coef``.
|
||||
"""
|
||||
if X.shape[1] == 0:
|
||||
return np.zeros(0), float(y.mean()) if len(y) else 0.0
|
||||
xm = X.mean(axis=0)
|
||||
ym = float(y.mean())
|
||||
Xc = X - xm
|
||||
w = _solve(Xc.T @ Xc + lam * np.eye(X.shape[1]), Xc.T @ (y - ym))
|
||||
return w, ym - float(xm @ w)
|
||||
|
||||
|
||||
def fit_logistic(X: np.ndarray, y: np.ndarray, lam: float,
|
||||
max_iter: int = NEWTON_MAX_ITER,
|
||||
tol: float = NEWTON_TOL) -> Tuple[np.ndarray, float, float]:
|
||||
"""L2 logistic via Newton-Raphson, intercept unpenalised.
|
||||
|
||||
Returns ``(coef, intercept, grad_norm)`` where ``grad_norm`` is the
|
||||
infinity-norm of the penalised gradient at the returned point.
|
||||
"""
|
||||
n, p = X.shape
|
||||
A = np.hstack([np.ones((n, 1)), X])
|
||||
pen = np.zeros(p + 1)
|
||||
pen[1:] = lam
|
||||
beta = np.zeros(p + 1)
|
||||
# Warm-start the intercept at the empirical log-odds.
|
||||
frac = float(np.clip(y.mean(), 1e-6, 1 - 1e-6))
|
||||
beta[0] = math.log(frac / (1 - frac))
|
||||
grad = np.zeros(p + 1)
|
||||
for _ in range(max_iter):
|
||||
eta = A @ beta
|
||||
prob = 1.0 / (1.0 + np.exp(-np.clip(eta, -35, 35)))
|
||||
grad = A.T @ (y - prob) - pen * beta
|
||||
W = np.clip(prob * (1 - prob), 1e-10, None)
|
||||
H = A.T @ (A * W[:, None]) + np.diag(pen)
|
||||
step = _solve(H, grad)
|
||||
beta = beta + step
|
||||
if np.max(np.abs(step)) < tol:
|
||||
break
|
||||
eta = A @ beta
|
||||
prob = 1.0 / (1.0 + np.exp(-np.clip(eta, -35, 35)))
|
||||
grad = A.T @ (y - prob) - pen * beta
|
||||
return beta[1:], float(beta[0]), float(np.max(np.abs(grad)))
|
||||
|
||||
|
||||
def predict_linear(X: np.ndarray, coef: np.ndarray, intercept: float) -> np.ndarray:
|
||||
if X.shape[1] == 0:
|
||||
return np.full(len(X), intercept, dtype=float)
|
||||
return intercept + X @ coef
|
||||
|
||||
|
||||
def sigmoid(z: np.ndarray) -> np.ndarray:
|
||||
return 1.0 / (1.0 + np.exp(-np.clip(z, -35, 35)))
|
||||
|
||||
|
||||
def log_loss(y: np.ndarray, p: np.ndarray) -> float:
|
||||
p = np.clip(p, 1e-12, 1 - 1e-12)
|
||||
return float(-np.mean(y * np.log(p) + (1 - y) * np.log(1 - p)))
|
||||
|
||||
|
||||
def choose_lambda(X: np.ndarray, y: np.ndarray, kind: str,
|
||||
grid: Sequence[float] = LAMBDA_GRID,
|
||||
n_folds: int = CV_FOLDS) -> Tuple[float, Dict[float, float]]:
|
||||
"""5-fold time-contiguous CV inside the training window."""
|
||||
folds = build_folds(np.arange(len(y)), n_folds)
|
||||
if X.shape[1] == 0 or len(folds) < 2:
|
||||
return float(grid[0]), {}
|
||||
scores: Dict[float, float] = {}
|
||||
for lam in grid:
|
||||
vals: List[float] = []
|
||||
for hold in folds:
|
||||
mask = np.ones(len(y), dtype=bool)
|
||||
mask[hold] = False
|
||||
if mask.sum() < 2:
|
||||
continue
|
||||
if kind == "logistic":
|
||||
ytr = y[mask]
|
||||
if ytr.min() == ytr.max(): # single-class fold
|
||||
p = np.full(len(hold), float(np.clip(ytr.mean(), 1e-6, 1 - 1e-6)))
|
||||
else:
|
||||
c, b, _ = fit_logistic(X[mask], ytr, lam)
|
||||
p = sigmoid(predict_linear(X[hold], c, b))
|
||||
vals.append(log_loss(y[hold], p))
|
||||
else:
|
||||
c, b = fit_ridge(X[mask], y[mask], lam)
|
||||
resid = y[hold] - predict_linear(X[hold], c, b)
|
||||
vals.append(float(np.mean(resid ** 2)))
|
||||
scores[float(lam)] = float(np.mean(vals)) if vals else float("inf")
|
||||
best = min(scores, key=lambda k: scores[k])
|
||||
return best, scores
|
||||
|
||||
|
||||
def lgbm_available() -> bool:
|
||||
try:
|
||||
import lightgbm # noqa: F401
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def fit_lgbm(X: np.ndarray, y: np.ndarray, seed: int):
|
||||
import lightgbm as lgb
|
||||
model = lgb.LGBMRegressor(random_state=seed, verbose=-1, **LGBM_PARAMS)
|
||||
model.fit(X, y)
|
||||
return model
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# Size map
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
def round_to(x, step: float = MAP_ROUND_TO) -> np.ndarray:
|
||||
"""Round to the nearest ``step`` and kill float dust (0.05 → 2 dp)."""
|
||||
decimals = max(0, int(round(-math.log10(step)))) + 1
|
||||
return np.round(np.round(np.asarray(x, dtype=float) / step) * step, decimals)
|
||||
|
||||
|
||||
def calibrate_map(z_train: np.ndarray) -> Dict[str, float]:
|
||||
"""lo/hi/normaliser from TRAIN predictions only."""
|
||||
lo = float(np.quantile(z_train, MAP_Q_LO))
|
||||
hi = float(np.quantile(z_train, MAP_Q_HI))
|
||||
if not np.isfinite(lo) or not np.isfinite(hi) or hi <= lo:
|
||||
return {"lo": lo, "hi": hi, "normalizer": 1.0, "degenerate": True}
|
||||
raw = MAP_OUT_LO + (MAP_OUT_HI - MAP_OUT_LO) * np.clip((z_train - lo) / (hi - lo), 0, 1)
|
||||
mean_raw = float(np.mean(raw))
|
||||
norm = 1.0 / mean_raw if mean_raw > 0 else 1.0
|
||||
return {"lo": lo, "hi": hi, "normalizer": norm, "degenerate": False}
|
||||
|
||||
|
||||
def apply_map(z: np.ndarray, cal: Dict[str, float]) -> np.ndarray:
|
||||
z = np.asarray(z, dtype=float)
|
||||
if cal.get("degenerate"):
|
||||
return np.ones(len(z))
|
||||
lo, hi, norm = cal["lo"], cal["hi"], cal["normalizer"]
|
||||
raw = MAP_OUT_LO + (MAP_OUT_HI - MAP_OUT_LO) * np.clip((z - lo) / (hi - lo), 0, 1)
|
||||
return round_to(np.clip(raw * norm, MAP_CLIP_LO, MAP_CLIP_HI))
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# Metrics
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
def max_drawdown(r: np.ndarray) -> float:
|
||||
"""0-prepended cumulative-R drawdown (gpu_backtest.compute_metrics)."""
|
||||
r = np.asarray(r, dtype=float)
|
||||
if len(r) == 0:
|
||||
return 0.0
|
||||
cum = np.concatenate(([0.0], np.cumsum(r)))
|
||||
return float((np.maximum.accumulate(cum) - cum).max())
|
||||
|
||||
|
||||
def profit_factor(r: np.ndarray) -> float:
|
||||
r = np.asarray(r, dtype=float)
|
||||
gp = float(r[r > 0].sum()) if (r > 0).any() else 0.0
|
||||
gl = float(-r[r < 0].sum()) if (r < 0).any() else 0.0
|
||||
return gp / gl if gl > 0 else float("inf")
|
||||
|
||||
|
||||
def multiplier_hist(m: np.ndarray) -> pd.DataFrame:
|
||||
"""Counts in 0.1-wide bins from 0.5 to 1.5.
|
||||
|
||||
``np.histogram`` already closes the final bin on the right, so an
|
||||
exactly-1.5 multiplier lands in [1.4, 1.5] once and the counts sum to
|
||||
``len(m)``.
|
||||
"""
|
||||
edges = np.round(np.arange(MAP_OUT_LO, MAP_OUT_HI + 1e-9, 0.1), 2)
|
||||
counts, _ = np.histogram(np.asarray(m, dtype=float), bins=edges)
|
||||
counts = counts.astype(int)
|
||||
return pd.DataFrame({
|
||||
"bin_lo": edges[:-1], "bin_hi": edges[1:], "count": counts,
|
||||
"pct": np.where(len(m), counts / max(len(m), 1) * 100, 0.0),
|
||||
})
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# Walk-forward driver
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
def walk_forward(df: pd.DataFrame, features: Sequence[str], model: str,
|
||||
bar_minutes: int, burn_in_months: int, purge_bars: int,
|
||||
seed: int, y_train_override: Optional[np.ndarray] = None,
|
||||
verbose: bool = True) -> Dict:
|
||||
"""One full expanding walk-forward pass.
|
||||
|
||||
``y_train_override`` (full-book length, aligned to ``df``) replaces the
|
||||
TRAINING labels only — this is how the shuffle null is run. Test labels
|
||||
always come from the real ``r_multiple``.
|
||||
"""
|
||||
times = df["entry_time"].to_numpy()
|
||||
r_all = df["r_multiple"].to_numpy(float)
|
||||
r_train_src = r_all if y_train_override is None else np.asarray(y_train_override, float)
|
||||
X_all = df[list(features)].apply(pd.to_numeric, errors="coerce").to_numpy(float) \
|
||||
if features else np.zeros((len(df), 0))
|
||||
|
||||
book_start = df["entry_time"].iloc[0]
|
||||
book_end = df["entry_time"].iloc[-1]
|
||||
burn_end = book_start + pd.DateOffset(months=burn_in_months)
|
||||
purge = pd.Timedelta(minutes=purge_bars * bar_minutes)
|
||||
|
||||
qs = [q for q in quarter_starts(book_start, book_end) if q >= burn_end]
|
||||
rows: List[Dict] = []
|
||||
per_q: List[Dict] = []
|
||||
coef_rows: List[Dict] = []
|
||||
skipped: List[str] = []
|
||||
dropped_seen: set = set()
|
||||
|
||||
for q_start in qs:
|
||||
q_end = q_start + pd.DateOffset(months=3)
|
||||
train_mask = df["entry_time"] < (q_start - purge)
|
||||
test_mask = (df["entry_time"] >= q_start) & (df["entry_time"] < q_end)
|
||||
n_tr, n_te = int(train_mask.sum()), int(test_mask.sum())
|
||||
if n_tr < MIN_TRAIN or n_te == 0:
|
||||
skipped.append(f"{quarter_label(q_start)} (train={n_tr}, test={n_te})")
|
||||
continue
|
||||
|
||||
tr = np.flatnonzero(train_mask.to_numpy())
|
||||
te = np.flatnonzero(test_mask.to_numpy())
|
||||
Xtr_raw, Xte_raw = X_all[tr], X_all[te]
|
||||
mean, scale, keep = fit_standardiser(Xtr_raw)
|
||||
dropped = [features[i] for i in np.flatnonzero(~keep)]
|
||||
if verbose and dropped:
|
||||
tag = (quarter_label(q_start), tuple(dropped))
|
||||
if tag not in dropped_seen:
|
||||
dropped_seen.add(tag)
|
||||
say(f" WARNING {quarter_label(q_start)}: zero variance in train, "
|
||||
f"dropped from fit (coef 0): {', '.join(dropped)}")
|
||||
Xtr = apply_standardiser(Xtr_raw, mean, scale)[:, keep]
|
||||
Xte = apply_standardiser(Xte_raw, mean, scale)[:, keep]
|
||||
|
||||
r_tr = r_train_src[tr]
|
||||
lam: Optional[float] = None
|
||||
if model == "logistic":
|
||||
y_tr = (r_tr > 0).astype(float)
|
||||
lam, _ = choose_lambda(Xtr, y_tr, "logistic")
|
||||
coef, b, _ = fit_logistic(Xtr, y_tr, lam)
|
||||
z_tr = sigmoid(predict_linear(Xtr, coef, b))
|
||||
z_te = sigmoid(predict_linear(Xte, coef, b))
|
||||
elif model == "lgbm":
|
||||
y_tr = np.clip(r_tr, WINSOR_LO, WINSOR_HI)
|
||||
gbm = fit_lgbm(Xtr, y_tr, seed)
|
||||
z_tr = np.asarray(gbm.predict(Xtr), dtype=float)
|
||||
z_te = np.asarray(gbm.predict(Xte), dtype=float)
|
||||
coef, b = np.zeros(int(keep.sum())), 0.0
|
||||
else: # ridge / isbuy / exhaustion
|
||||
y_tr = np.clip(r_tr, WINSOR_LO, WINSOR_HI)
|
||||
lam, _ = choose_lambda(Xtr, y_tr, "ridge")
|
||||
coef, b = fit_ridge(Xtr, y_tr, lam)
|
||||
z_tr = predict_linear(Xtr, coef, b)
|
||||
z_te = predict_linear(Xte, coef, b)
|
||||
|
||||
cal = calibrate_map(z_tr)
|
||||
m_te = apply_map(z_te, cal)
|
||||
|
||||
if model != "lgbm":
|
||||
full_coef = np.zeros(len(features))
|
||||
full_coef[np.flatnonzero(keep)] = coef
|
||||
for name, c in zip(features, full_coef):
|
||||
coef_rows.append({"quarter": quarter_label(q_start), "feature": name,
|
||||
"coef_std": float(c), "lambda": lam,
|
||||
"dropped": name in dropped})
|
||||
coef_rows.append({"quarter": quarter_label(q_start), "feature": "_intercept",
|
||||
"coef_std": float(b), "lambda": lam, "dropped": False})
|
||||
|
||||
r_te = r_all[te]
|
||||
for i, idx in enumerate(te):
|
||||
rows.append({
|
||||
"entry_time": df["entry_time"].iloc[idx],
|
||||
"r_multiple": float(r_te[i]),
|
||||
"mult": float(m_te[i]),
|
||||
"r_sized": float(r_te[i] * m_te[i]),
|
||||
"quarter": quarter_label(q_start),
|
||||
"z": float(z_te[i]),
|
||||
})
|
||||
per_q.append({
|
||||
"quarter": quarter_label(q_start),
|
||||
"q_start": q_start, "n_train": n_tr, "n_test": n_te,
|
||||
"flat_r": float(r_te.sum()), "sized_r": float((r_te * m_te).sum()),
|
||||
"mean_m": float(m_te.mean()), "lambda": lam,
|
||||
"map_lo": cal["lo"], "map_hi": cal["hi"], "normalizer": cal["normalizer"],
|
||||
})
|
||||
|
||||
oos = pd.DataFrame(rows)
|
||||
return {"oos": oos, "quarters": pd.DataFrame(per_q),
|
||||
"coefs": pd.DataFrame(coef_rows), "skipped": skipped,
|
||||
"quarter_starts": qs}
|
||||
|
||||
|
||||
def evaluate(oos: pd.DataFrame) -> Dict:
|
||||
"""Headline OOS numbers for one model's walk-forward output."""
|
||||
if oos.empty:
|
||||
return {"n": 0, "flat_r": 0.0, "sized_r": 0.0, "lift": 0.0}
|
||||
r = oos["r_multiple"].to_numpy(float)
|
||||
rs = oos["r_sized"].to_numpy(float)
|
||||
m = oos["mult"].to_numpy(float)
|
||||
med = oos["entry_time"].median()
|
||||
first = (oos["entry_time"] < med).to_numpy()
|
||||
second = ~first
|
||||
return {
|
||||
"n": int(len(oos)),
|
||||
"flat_r": float(r.sum()), "sized_r": float(rs.sum()),
|
||||
"lift": float(rs.sum() - r.sum()),
|
||||
"flat_pf": profit_factor(r), "sized_pf": profit_factor(rs),
|
||||
"flat_dd": max_drawdown(r), "sized_dd": max_drawdown(rs),
|
||||
"mean_m": float(m.mean()), "min_m": float(m.min()), "max_m": float(m.max()),
|
||||
"median_time": med,
|
||||
"n_h1": int(first.sum()), "n_h2": int(second.sum()),
|
||||
"flat_h1": float(r[first].sum()), "flat_h2": float(r[second].sum()),
|
||||
"sized_h1": float(rs[first].sum()), "sized_h2": float(rs[second].sum()),
|
||||
}
|
||||
|
||||
|
||||
def acceptance(ev: Dict, real_lift: float, shuffle_p95: float) -> Dict:
|
||||
"""The six pre-registered clauses."""
|
||||
c1 = ev["sized_h1"] > ev["flat_h1"] and ev["sized_h2"] > ev["flat_h2"]
|
||||
c2 = ev["sized_r"] > ev["flat_r"]
|
||||
c3 = ev["sized_dd"] <= DD_TOL * ev["flat_dd"]
|
||||
c4 = (ev["sized_pf"] >= PF_FLOOR) and ev["sized_h1"] > 0 and ev["sized_h2"] > 0
|
||||
c5 = real_lift > shuffle_p95 if np.isfinite(shuffle_p95) else False
|
||||
c6 = (abs(ev["mean_m"] - 1.0) <= MEAN_MULT_TOL
|
||||
and ev["min_m"] >= MAP_CLIP_LO - 1e-9 and ev["max_m"] <= MAP_CLIP_HI + 1e-9)
|
||||
clauses = {"1_both_halves": bool(c1), "2_overall": bool(c2), "3_drawdown": bool(c3),
|
||||
"4_pf_and_halves": bool(c4), "5_beats_shuffle": bool(c5),
|
||||
"6_multiplier_bounds": bool(c6)}
|
||||
clauses["overall"] = bool(all(clauses.values()))
|
||||
return clauses
|
||||
|
||||
|
||||
def shuffle_null(df: pd.DataFrame, features: Sequence[str], model: str,
|
||||
bar_minutes: int, burn_in_months: int, purge_bars: int,
|
||||
n_shuffle: int, seed: int) -> np.ndarray:
|
||||
"""Lift distribution under permuted TRAINING labels (test labels real)."""
|
||||
rng = np.random.default_rng(seed)
|
||||
r_all = df["r_multiple"].to_numpy(float)
|
||||
lifts: List[float] = []
|
||||
for _ in range(n_shuffle):
|
||||
r_perm = r_all[rng.permutation(len(r_all))]
|
||||
res = walk_forward(df, features, model, bar_minutes, burn_in_months,
|
||||
purge_bars, seed, y_train_override=r_perm, verbose=False)
|
||||
ev = evaluate(res["oos"])
|
||||
lifts.append(ev["lift"])
|
||||
return np.asarray(lifts, dtype=float)
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# Final fit + artifact
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
def final_fit(df: pd.DataFrame, features: Sequence[str], model: str,
|
||||
seed: int) -> Dict:
|
||||
"""Fit on ALL trades; calibrate the map on in-sample predictions."""
|
||||
X_raw = df[list(features)].apply(pd.to_numeric, errors="coerce").to_numpy(float)
|
||||
r = df["r_multiple"].to_numpy(float)
|
||||
mean, scale, keep = fit_standardiser(X_raw)
|
||||
X = apply_standardiser(X_raw, mean, scale)[:, keep]
|
||||
if model == "logistic":
|
||||
y = (r > 0).astype(float)
|
||||
lam, _ = choose_lambda(X, y, "logistic")
|
||||
coef, b, gnorm = fit_logistic(X, y, lam)
|
||||
z = sigmoid(predict_linear(X, coef, b))
|
||||
target = "r_positive"
|
||||
else:
|
||||
y = np.clip(r, WINSOR_LO, WINSOR_HI)
|
||||
lam, _ = choose_lambda(X, y, "ridge")
|
||||
coef, b = fit_ridge(X, y, lam)
|
||||
gnorm = float("nan")
|
||||
z = predict_linear(X, coef, b)
|
||||
target = "r_winsorised"
|
||||
full_coef = np.zeros(len(features))
|
||||
full_coef[np.flatnonzero(keep)] = coef
|
||||
cal = calibrate_map(z)
|
||||
return {"mean": mean, "scale": scale, "keep": keep, "coef": full_coef,
|
||||
"intercept": float(b), "lambda": float(lam), "z": z,
|
||||
"mult": apply_map(z, cal), "map": cal, "target": target,
|
||||
"grad_norm": gnorm}
|
||||
|
||||
|
||||
def predict_multiplier(feature_dict: Dict[str, float], artifact: Dict) -> float:
|
||||
"""Runtime formula: standardise → linear score → size map → 0.05 grid."""
|
||||
names = artifact["features"]
|
||||
mean = artifact["mean"]
|
||||
scale = artifact["scale"]
|
||||
coef = artifact["coef"]
|
||||
z = float(artifact["intercept"])
|
||||
for i, name in enumerate(names):
|
||||
raw = feature_dict.get(name, mean[i])
|
||||
if raw is None or not np.isfinite(float(raw)):
|
||||
raw = mean[i]
|
||||
z += float(coef[i]) * (float(raw) - float(mean[i])) / float(scale[i])
|
||||
if artifact["model"] == "logistic":
|
||||
z = 1.0 / (1.0 + math.exp(-max(-35.0, min(35.0, z))))
|
||||
mp = artifact["map"]
|
||||
if mp["hi"] <= mp["lo"]:
|
||||
return 1.0
|
||||
frac = min(1.0, max(0.0, (z - mp["lo"]) / (mp["hi"] - mp["lo"])))
|
||||
raw_m = mp["out_lo"] + (mp["out_hi"] - mp["out_lo"]) * frac
|
||||
val = min(mp["clip_hi"], max(mp["clip_lo"], raw_m * mp["normalizer"]))
|
||||
step = mp["round_to"]
|
||||
return float(round(round(val / step) * step, 2))
|
||||
|
||||
|
||||
def build_artifact(fit: Dict, features: Sequence[str], model: str, df: pd.DataFrame,
|
||||
ev: Dict, clauses: Dict, real_lift: float, shuffle: np.ndarray,
|
||||
n_shuffle: int, book: Path, tf: str, confirm_tfs: List[str],
|
||||
burn_in_months: int, purge_bars: int, spread: float) -> Dict:
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"feature_version": FEATURE_VERSION,
|
||||
"pair": "XAU/USD",
|
||||
"primary_tf": tf,
|
||||
"confirmation_tfs": confirm_tfs,
|
||||
"model": model,
|
||||
"target": fit["target"],
|
||||
"features": list(features),
|
||||
"mean": [float(v) for v in fit["mean"]],
|
||||
"scale": [float(v) for v in fit["scale"]],
|
||||
"coef": [float(v) for v in fit["coef"]],
|
||||
"intercept": float(fit["intercept"]),
|
||||
"map": {
|
||||
"kind": "clipped_linear_on_quantiles",
|
||||
"lo": float(fit["map"]["lo"]), "hi": float(fit["map"]["hi"]),
|
||||
"out_lo": MAP_OUT_LO, "out_hi": MAP_OUT_HI,
|
||||
"normalizer": float(fit["map"]["normalizer"]),
|
||||
"clip_lo": MAP_CLIP_LO, "clip_hi": MAP_CLIP_HI,
|
||||
"round_to": MAP_ROUND_TO,
|
||||
},
|
||||
"training": {
|
||||
"window": [df["entry_time"].iloc[0].isoformat(),
|
||||
df["entry_time"].iloc[-1].isoformat()],
|
||||
"spread": spread,
|
||||
"n_train_trades": int(len(df)),
|
||||
"n_oos_trades": int(ev.get("n", 0)),
|
||||
"burn_in_months": burn_in_months,
|
||||
"purge_bars": purge_bars,
|
||||
"refit": "calendar_quarter",
|
||||
"final_fit": "full_window",
|
||||
"lambda": float(fit["lambda"]),
|
||||
"git_commit": git_commit(),
|
||||
"book_sha256": sha256_file(book),
|
||||
},
|
||||
"acceptance": {
|
||||
**{k: v for k, v in clauses.items()},
|
||||
"flat_r": ev.get("flat_r"), "sized_r": ev.get("sized_r"),
|
||||
"sized_pf": ev.get("sized_pf"), "flat_dd": ev.get("flat_dd"),
|
||||
"sized_dd": ev.get("sized_dd"),
|
||||
"sized_h1": ev.get("sized_h1"), "sized_h2": ev.get("sized_h2"),
|
||||
"flat_h1": ev.get("flat_h1"), "flat_h2": ev.get("flat_h2"),
|
||||
"real_lift": float(real_lift),
|
||||
"shuffle_p95_lift": float(np.percentile(shuffle, 95)) if len(shuffle) else None,
|
||||
"n_shuffle": int(n_shuffle),
|
||||
"mean_multiplier": ev.get("mean_m"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# Reporting
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
def fmt(x, nd=2, width=0) -> str:
|
||||
if x is None:
|
||||
return "n/a".rjust(width)
|
||||
if isinstance(x, float) and not np.isfinite(x):
|
||||
return ("inf" if x > 0 else "-inf").rjust(width)
|
||||
s = f"{x:.{nd}f}" if isinstance(x, (int, float, np.floating)) else str(x)
|
||||
return s.rjust(width) if width else s
|
||||
|
||||
|
||||
def print_quarters(q: pd.DataFrame) -> None:
|
||||
say(f"{'quarter':<9}{'n_tr':>6}{'n_te':>6}{'flat R':>10}{'sized R':>10}"
|
||||
f"{'mean m':>9}{'lambda':>9}")
|
||||
for _, row in q.iterrows():
|
||||
say(f"{row['quarter']:<9}{int(row['n_train']):>6}{int(row['n_test']):>6}"
|
||||
f"{fmt(row['flat_r'], 2, 10)}{fmt(row['sized_r'], 2, 10)}"
|
||||
f"{fmt(row['mean_m'], 3, 9)}"
|
||||
f"{('n/a' if row['lambda'] is None else fmt(row['lambda'], 0)):>9}")
|
||||
|
||||
|
||||
def print_hist(m: np.ndarray) -> None:
|
||||
h = multiplier_hist(m)
|
||||
say(f"{'bin':<12}{'count':>8}{'pct':>8}")
|
||||
for _, row in h.iterrows():
|
||||
bar = "#" * int(round(row["pct"] / 2))
|
||||
say(f"{row['bin_lo']:.1f}–{row['bin_hi']:.1f}{'':<4}{int(row['count']):>8}"
|
||||
f"{row['pct']:>7.1f}% {bar}")
|
||||
|
||||
|
||||
def print_coef_signs(coefs: pd.DataFrame, label: str) -> None:
|
||||
"""Mean standardised coefficient + sign stability across quarters."""
|
||||
if coefs.empty:
|
||||
return
|
||||
d = coefs[coefs["feature"] != "_intercept"]
|
||||
say(f" standardised coefficients across quarters ({label}):")
|
||||
say(f" {'feature':<24}{'mean':>10}{'min':>10}{'max':>10}{'+q':>5}{'-q':>5} sign-stable")
|
||||
for name, grp in d.groupby("feature", sort=False):
|
||||
c = grp["coef_std"].to_numpy(float)
|
||||
pos, neg = int((c > 0).sum()), int((c < 0).sum())
|
||||
stable = "yes" if (pos == 0 or neg == 0) and (pos + neg) > 0 else "no"
|
||||
say(f" {name:<24}{c.mean():>10.4f}{c.min():>10.4f}{c.max():>10.4f}"
|
||||
f"{pos:>5}{neg:>5} {stable}")
|
||||
|
||||
|
||||
def print_comparison(table: List[Dict]) -> None:
|
||||
say(f"{'model':<12}{'n':>6}{'flat R':>10}{'sized R':>10}{'lift':>9}"
|
||||
f"{'PF':>7}{'DD':>8}{'mean m':>8}{'p95 null':>10}{'rank%':>7} pass")
|
||||
for row in table:
|
||||
say(f"{row['model']:<12}{row['n']:>6}{fmt(row['flat_r'], 2, 10)}"
|
||||
f"{fmt(row['sized_r'], 2, 10)}{fmt(row['lift'], 2, 9)}"
|
||||
f"{fmt(row['sized_pf'], 2, 7)}{fmt(row['sized_dd'], 2, 8)}"
|
||||
f"{fmt(row['mean_m'], 3, 8)}{fmt(row['shuffle_p95'], 2, 10)}"
|
||||
f"{fmt(row['rank_pct'], 0, 7)} {'PASS' if row['pass'] else 'FAIL'}")
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# main
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--book", required=True, type=Path)
|
||||
ap.add_argument("--bar-minutes", required=True, type=int)
|
||||
ap.add_argument("--features", required=True,
|
||||
help="comma-separated f_* column names for the main model")
|
||||
ap.add_argument("--model", default="ridge", choices=("ridge", "logistic"))
|
||||
ap.add_argument("--burn-in-months", type=int, default=12)
|
||||
ap.add_argument("--purge-bars", type=int, default=300)
|
||||
ap.add_argument("--n-shuffle", type=int, default=50)
|
||||
ap.add_argument("--seed", type=int, default=20260905)
|
||||
ap.add_argument("--skip-lgbm", action="store_true")
|
||||
ap.add_argument("--out", type=Path, default=None)
|
||||
ap.add_argument("--write-artifact", type=Path, default=None)
|
||||
ap.add_argument("--tf", default=None, help="primary TF label for the artifact")
|
||||
ap.add_argument("--confirm-tfs", default="15min,1h")
|
||||
ap.add_argument("--spread", type=float, default=0.65,
|
||||
help="cost the book was produced at (recorded in the artifact)")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.write_artifact and not args.tf:
|
||||
raise SystemExit("--write-artifact requires --tf")
|
||||
|
||||
say("=" * 96)
|
||||
say("Stage-D stake-sizing walk-forward — RESEARCH ONLY")
|
||||
say("=" * 96)
|
||||
say(f"book : {args.book}")
|
||||
df = load_book(args.book)
|
||||
say(f"trades : {len(df)} "
|
||||
f"({df['entry_time'].iloc[0]} → {df['entry_time'].iloc[-1]})")
|
||||
say(f"bar minutes : {args.bar_minutes} purge = {args.purge_bars} bars = "
|
||||
f"{args.purge_bars * args.bar_minutes} min "
|
||||
f"({args.purge_bars * args.bar_minutes / 1440:.2f} d)")
|
||||
say(f"burn-in : {args.burn_in_months} months "
|
||||
f"(train-only until {df['entry_time'].iloc[0] + pd.DateOffset(months=args.burn_in_months)})")
|
||||
say(f"model / seed : {args.model} / {args.seed} shuffles: {args.n_shuffle}")
|
||||
say(f"winsor / lambda : [{WINSOR_LO}, {WINSOR_HI}] / {list(LAMBDA_GRID)} "
|
||||
f"CV folds {CV_FOLDS} (time-contiguous)")
|
||||
say(f"map : m = round_{MAP_ROUND_TO}(clip(norm × (0.5 + clip((z−lo)/(hi−lo),0,1)"
|
||||
f"), {MAP_CLIP_LO}, {MAP_CLIP_HI})) lo/hi = train q{MAP_Q_LO}/q{MAP_Q_HI}")
|
||||
say("")
|
||||
|
||||
main_feats = resolve_features(df, [f.strip() for f in args.features.split(",") if f.strip()],
|
||||
"--features")
|
||||
say(f"main features ({len(main_feats)}): {', '.join(main_feats)}")
|
||||
|
||||
specs: List[Tuple[str, List[str], str]] = [(args.model, main_feats, args.model)]
|
||||
for label, feats in (("isbuy", ISBUY_FEATURES), ("exhaustion", EXHAUSTION_FEATURES)):
|
||||
have = [f for f in feats if f in df.columns]
|
||||
if len(have) != len(feats):
|
||||
say(f"NOTE: {label} baseline missing "
|
||||
f"{', '.join(f for f in feats if f not in df.columns)} — using {len(have)}")
|
||||
if have:
|
||||
specs.append((label, have, "ridge"))
|
||||
use_lgbm = (not args.skip_lgbm) and lgbm_available()
|
||||
if not args.skip_lgbm and not use_lgbm:
|
||||
say("lightgbm not installed — upper bound skipped")
|
||||
if use_lgbm:
|
||||
all_f = [c for c in df.columns if c.startswith("f_")]
|
||||
specs.append(("lgbm", all_f, "lgbm"))
|
||||
say(f"lightgbm upper bound: {len(all_f)} f_* columns")
|
||||
say("")
|
||||
|
||||
results: Dict[str, Dict] = {}
|
||||
table: List[Dict] = []
|
||||
for label, feats, kind in specs:
|
||||
say("-" * 96)
|
||||
say(f"MODEL: {label} ({kind}, {len(feats)} feature(s))")
|
||||
say("-" * 96)
|
||||
res = walk_forward(df, feats, kind, args.bar_minutes, args.burn_in_months,
|
||||
args.purge_bars, args.seed, verbose=True)
|
||||
if res["skipped"]:
|
||||
say(f" skipped quarters: {'; '.join(res['skipped'])}")
|
||||
if res["oos"].empty:
|
||||
say(" no OOS quarters — nothing to evaluate")
|
||||
continue
|
||||
ev = evaluate(res["oos"])
|
||||
n_sh = args.n_shuffle if kind != "lgbm" else min(args.n_shuffle, LGBM_MAX_SHUFFLE)
|
||||
if kind == "lgbm" and n_sh != args.n_shuffle:
|
||||
say(f" NOTE (protocol deviation, flagged): lgbm shuffle null capped at "
|
||||
f"{n_sh} repeats (never ships)")
|
||||
sh = shuffle_null(df, feats, kind, args.bar_minutes, args.burn_in_months,
|
||||
args.purge_bars, n_sh, args.seed) if n_sh > 0 else np.array([])
|
||||
p95 = float(np.percentile(sh, 95)) if len(sh) else float("nan")
|
||||
rank = float((sh < ev["lift"]).mean() * 100) if len(sh) else float("nan")
|
||||
cl = acceptance(ev, ev["lift"], p95)
|
||||
|
||||
say(f" OOS quarters : {len(res['quarters'])} "
|
||||
f"({res['quarters']['quarter'].iloc[0]} → {res['quarters']['quarter'].iloc[-1]}), "
|
||||
f"{ev['n']} trades")
|
||||
say(f" flat R : {ev['flat_r']:+.2f} PF {fmt(ev['flat_pf'])} "
|
||||
f"maxDD {ev['flat_dd']:.2f}")
|
||||
say(f" sized R : {ev['sized_r']:+.2f} PF {fmt(ev['sized_pf'])} "
|
||||
f"maxDD {ev['sized_dd']:.2f}")
|
||||
say(f" lift : {ev['lift']:+.2f} R")
|
||||
say(f" halves (split {ev['median_time']}): "
|
||||
f"h1 flat {ev['flat_h1']:+.2f} / sized {ev['sized_h1']:+.2f} (n={ev['n_h1']}) | "
|
||||
f"h2 flat {ev['flat_h2']:+.2f} / sized {ev['sized_h2']:+.2f} (n={ev['n_h2']})")
|
||||
say(f" multiplier : mean {ev['mean_m']:.3f} min {ev['min_m']:.2f} "
|
||||
f"max {ev['max_m']:.2f}")
|
||||
say(f" shuffle null : n={len(sh)} mean {fmt(float(sh.mean()) if len(sh) else None)} "
|
||||
f"p95 {fmt(p95)} real-lift percentile rank {fmt(rank, 0)}%")
|
||||
say(" acceptance:")
|
||||
for k, v in cl.items():
|
||||
if k == "overall":
|
||||
continue
|
||||
say(f" {k:<22} {'PASS' if v else 'FAIL'}")
|
||||
say(f" {'OVERALL':<22} {'PASS' if cl['overall'] else 'FAIL'}")
|
||||
say(" per-quarter:")
|
||||
print_quarters(res["quarters"])
|
||||
print_coef_signs(res["coefs"], label)
|
||||
say("")
|
||||
|
||||
results[label] = {"res": res, "ev": ev, "shuffle": sh, "p95": p95,
|
||||
"rank": rank, "clauses": cl, "features": feats, "kind": kind}
|
||||
table.append({"model": label, "n": ev["n"], "flat_r": ev["flat_r"],
|
||||
"sized_r": ev["sized_r"], "lift": ev["lift"],
|
||||
"sized_pf": ev["sized_pf"], "sized_dd": ev["sized_dd"],
|
||||
"mean_m": ev["mean_m"], "shuffle_p95": p95, "rank_pct": rank,
|
||||
"pass": cl["overall"]})
|
||||
|
||||
if args.model not in results:
|
||||
raise SystemExit("the main model produced no OOS quarters — check burn-in / "
|
||||
"book window / MIN_TRAIN")
|
||||
|
||||
say("=" * 96)
|
||||
say("COMPARISON")
|
||||
say("=" * 96)
|
||||
print_comparison(table)
|
||||
say("")
|
||||
say(f"OOS multiplier histogram ({args.model}):")
|
||||
print_hist(results[args.model]["res"]["oos"]["mult"].to_numpy())
|
||||
say("")
|
||||
|
||||
main_label = args.model
|
||||
beats = None
|
||||
if main_label in results and "isbuy" in results:
|
||||
beats = results[main_label]["ev"]["lift"] > results["isbuy"]["ev"]["lift"]
|
||||
say(f"beats isbuy baseline: {'yes' if beats else 'no'} "
|
||||
f"({results[main_label]['ev']['lift']:+.2f} R vs "
|
||||
f"{results['isbuy']['ev']['lift']:+.2f} R)")
|
||||
say(f"OVERALL ACCEPTANCE ({main_label}): "
|
||||
f"{'PASS' if results[main_label]['clauses']['overall'] else 'FAIL'}")
|
||||
say("")
|
||||
|
||||
# ── artifact ──────────────────────────────────────────────────────
|
||||
artifact = None
|
||||
if args.write_artifact:
|
||||
say("-" * 96)
|
||||
say("FINAL FIT (all trades) + ARTIFACT")
|
||||
say("-" * 96)
|
||||
fit = final_fit(df, main_feats, args.model, args.seed)
|
||||
say(f" lambda (CV on full window): {fit['lambda']:g}")
|
||||
if np.isfinite(fit["grad_norm"]):
|
||||
say(f" logistic |grad|_inf at convergence: {fit['grad_norm']:.3e}")
|
||||
say(f" map lo/hi/normaliser: {fit['map']['lo']:.5f} / {fit['map']['hi']:.5f} / "
|
||||
f"{fit['map']['normalizer']:.5f}")
|
||||
artifact = build_artifact(
|
||||
fit, main_feats, args.model, df, results[main_label]["ev"],
|
||||
results[main_label]["clauses"], results[main_label]["ev"]["lift"],
|
||||
results[main_label]["shuffle"], args.n_shuffle, args.book, args.tf,
|
||||
[t.strip() for t in args.confirm_tfs.split(",") if t.strip()],
|
||||
args.burn_in_months, args.purge_bars, args.spread)
|
||||
# Self-check: the runtime formula must reproduce the in-sample map exactly.
|
||||
rows = df[main_feats].to_dict("records")
|
||||
repro = np.array([predict_multiplier(r, artifact) for r in rows])
|
||||
exact = bool(np.array_equal(repro, fit["mult"]))
|
||||
say(f" predict_multiplier reproduces in-sample multipliers exactly: "
|
||||
f"{'YES' if exact else 'NO'} (max |diff| "
|
||||
f"{float(np.max(np.abs(repro - fit['mult']))):.3e})")
|
||||
if not exact:
|
||||
raise SystemExit("predict_multiplier does not reproduce the final fit — abort")
|
||||
args.write_artifact.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.write_artifact.write_text(json.dumps(artifact, indent=2, default=str))
|
||||
say(f" wrote {args.write_artifact}")
|
||||
say("")
|
||||
|
||||
# ── outputs ───────────────────────────────────────────────────────
|
||||
if args.out:
|
||||
out = args.out
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
main_res = results[main_label]
|
||||
main_res["res"]["oos"].to_csv(out / "oos_trades.csv", index=False)
|
||||
main_res["res"]["quarters"].to_csv(out / "quarters.csv", index=False)
|
||||
coefs = pd.concat(
|
||||
[r["res"]["coefs"].assign(model=lbl) for lbl, r in results.items()
|
||||
if not r["res"]["coefs"].empty],
|
||||
ignore_index=True) if results else pd.DataFrame()
|
||||
coefs.to_csv(out / "coefs_by_quarter.csv", index=False)
|
||||
pd.DataFrame({
|
||||
"model": np.repeat([lbl for lbl in results], [len(r["shuffle"]) for r in results.values()]),
|
||||
"lift": np.concatenate([r["shuffle"] for r in results.values()])
|
||||
if results else np.array([]),
|
||||
}).to_csv(out / "shuffle_lifts.csv", index=False)
|
||||
multiplier_hist(main_res["res"]["oos"]["mult"].to_numpy()).to_csv(
|
||||
out / "multiplier_hist.csv", index=False)
|
||||
payload = {
|
||||
"book": str(args.book), "book_sha256": sha256_file(args.book),
|
||||
"git_commit": git_commit(), "bar_minutes": args.bar_minutes,
|
||||
"model": args.model, "features": main_feats, "seed": args.seed,
|
||||
"n_shuffle": args.n_shuffle, "burn_in_months": args.burn_in_months,
|
||||
"purge_bars": args.purge_bars,
|
||||
"beats_isbuy": beats,
|
||||
"models": {
|
||||
lbl: {
|
||||
"kind": r["kind"], "features": r["features"],
|
||||
"evaluation": {k: (None if isinstance(v, float) and not np.isfinite(v)
|
||||
else (str(v) if isinstance(v, pd.Timestamp) else v))
|
||||
for k, v in r["ev"].items()},
|
||||
"shuffle": {"n": int(len(r["shuffle"])),
|
||||
"mean": float(r["shuffle"].mean()) if len(r["shuffle"]) else None,
|
||||
"p95": None if not np.isfinite(r["p95"]) else r["p95"],
|
||||
"real_lift_percentile": None if not np.isfinite(r["rank"])
|
||||
else r["rank"]},
|
||||
"acceptance": r["clauses"],
|
||||
} for lbl, r in results.items()
|
||||
},
|
||||
"artifact": artifact,
|
||||
}
|
||||
(out / "results.json").write_text(json.dumps(payload, indent=2, default=str))
|
||||
(out / "summary.txt").write_text("\n".join(_LINES) + "\n")
|
||||
print(f"\nwrote outputs to {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Evaluate the violent-move WATCH rule (as shipped in violent_watch.py) bar by bar on history. Analysis tooling only.
|
||||
Level per bar from: calendar (high-impact USD within 30 min ahead / 30 min after), clock slots, indicator readings.
|
||||
Reports share of bars per level, burst rate next bar / next hour per level (both halves), and hot-hours per day.
|
||||
python backtest/research/watch_rule_eval.py PARQUET BAR_MINUTES
|
||||
"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import numpy as np, pandas as pd
|
||||
ROOT = Path(__file__).resolve().parents[2]; sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "backtest"))
|
||||
from gpu_backtest import precompute_indicators
|
||||
from config import TechnicalConfig
|
||||
|
||||
|
||||
def wilder_atr(df, period=14):
|
||||
pc = df["Close"].shift(1)
|
||||
tr = pd.concat([df["High"] - df["Low"], (df["High"] - pc).abs(), (df["Low"] - pc).abs()], axis=1).max(axis=1)
|
||||
return tr.ewm(alpha=1 / period, min_periods=period, adjust=False).mean()
|
||||
|
||||
|
||||
def main():
|
||||
pq, bm = sys.argv[1], int(sys.argv[2])
|
||||
px = pd.read_parquet(pq); px = px[(px.index >= pd.Timestamp("2023-08-05", tz="UTC")) & (px.index < pd.Timestamp("2026-09-05", tz="UTC"))]
|
||||
ic = precompute_indicators(px, TechnicalConfig())
|
||||
sh = lambda a: pd.Series(np.asarray(a, dtype=float), index=px.index).shift(1)
|
||||
rsi, adxs, hist, bbp, kcu, kcl = sh(ic.rsi), sh(ic.adx_slope), sh(ic.macd_hist), sh(ic.bb_position), sh(ic.kc_upper), sh(ic.kc_lower)
|
||||
ef, em, es, et, adx, atr1 = sh(ic.ema_fast), sh(ic.ema_medium), sh(ic.ema_slow), sh(ic.ema_trend), sh(ic.adx), sh(ic.atr)
|
||||
c1 = px["Close"].shift(1); hp = hist.shift(1)
|
||||
atr_prev = wilder_atr(px).shift(1); rng = px["High"] - px["Low"]
|
||||
burst = (rng >= 3.0 * atr_prev).astype(int); H = max(1, 60 // bm)
|
||||
next_hour = burst[::-1].rolling(H, min_periods=1).max()[::-1].astype(int)
|
||||
align = np.sign(c1 - ef) + np.sign(ef - em) + np.sign(em - es) + np.sign(es - et)
|
||||
regime = np.select([(adx >= 25) & (align > 0), (adx >= 25) & (align < 0), adx >= 25, adxs > 3.0], ["trending_up", "trending_down", "trending", "transitioning"], default="ranging")
|
||||
atr_pct = atr1.rolling(50).apply(lambda w: (w[-1] > w).mean() * 100, raw=True); extreme = atr_pct > 80
|
||||
# readings
|
||||
bear = ((bbp < 0).astype(int) + (rsi < 30).astype(int) + (c1 < kcl).astype(int) + ((hist < 0) & (hist < hp) & (hp < 0)).astype(int))
|
||||
bull = ((bbp > 1).astype(int) + (rsi >= 70).astype(int) + (c1 > kcu).astype(int) + ((hist > 0) & (hist > hp) & (hp > 0)).astype(int))
|
||||
neutral = (adxs > 3).astype(int) + extreme.astype(int) + (regime == "transitioning").astype(int)
|
||||
readings = bear + bull + neutral
|
||||
# clock (now = bar close time)
|
||||
ct = px.index + pd.Timedelta(minutes=bm); wd = ct.weekday; hr = ct.hour
|
||||
clock = ((wd <= 4) & np.isin(hr, [12, 13, 14])) | (hr == 1) | ((wd == 6) & np.isin(hr, [22, 23]))
|
||||
# calendar
|
||||
cal = pd.read_parquet(ROOT / "backtest/cache/calendar.parquet"); cal["ts"] = pd.to_datetime(cal.dateline, unit="s", utc=True)
|
||||
hi = np.sort(cal[(cal.currency == "USD") & (cal.impactName.str.lower() == "high")].ts.values.astype("datetime64[ns]"))
|
||||
t = ct.values.astype("datetime64[ns]"); nx = np.searchsorted(hi, t)
|
||||
to_next = np.where(nx < len(hi), (hi[np.minimum(nx, len(hi) - 1)] - t) / np.timedelta64(1, "m"), 1e9)
|
||||
since = np.where(nx - 1 >= 0, (t - hi[np.maximum(nx - 1, 0)]) / np.timedelta64(1, "m"), 1e9)
|
||||
calendar = ((to_next > 0) & (to_next <= 30)) | (since <= 30)
|
||||
# FINAL RULE (2026-09-06, chosen from the variant sweep recorded in sizing_campaign_2026-09.md):
|
||||
# extreme = price-at-extreme readings only (outside Bollinger, RSI beyond 30/70, outside Keltner)
|
||||
# hot = calendar | (clock & extreme >= 1) -> ~8.6% of bars, next-bar odds 3-5x normal, ~4 hot hours/day
|
||||
# watch = clock | extreme >= 2 -> ~2x normal
|
||||
extreme = ((bbp < 0) | (bbp > 1)).astype(int) + ((rsi < 30) | (rsi >= 70)).astype(int) + ((c1 < kcl) | (c1 > kcu)).astype(int)
|
||||
level = np.where(calendar | (clock & (extreme >= 1)), "hot", np.where(clock | (extreme >= 2), "watch", "none"))
|
||||
d = pd.DataFrame({"level": level, "burst": burst.values, "next_hour": next_hour.values, "readings": readings.values, "clock": clock, "calendar": calendar}, index=px.index)
|
||||
d = d[d.index >= pd.Timestamp("2023-09-05", tz="UTC")]
|
||||
half = np.where(np.arange(len(d)) < len(d) // 2, 1, 2)
|
||||
b1, b2 = d.burst[half == 1].mean(), d.burst[half == 2].mean(); h1, h2 = d.next_hour[half == 1].mean(), d.next_hour[half == 2].mean()
|
||||
print(f"{Path(pq).name}: {len(d)} bars; base next-bar {100*d.burst.mean():.2f}%, next-hour {100*d.next_hour.mean():.1f}%")
|
||||
rows = []
|
||||
for lv in ("none", "watch", "hot"):
|
||||
m = d.level == lv
|
||||
rows.append(dict(level=lv, share_of_bars=100 * m.mean(), next_bar_pct=100 * d.burst[m].mean(), lift_h1=d.burst[m & (half == 1)].mean() / b1, lift_h2=d.burst[m & (half == 2)].mean() / b2,
|
||||
next_hour_pct=100 * d.next_hour[m].mean(), hour_lift_h1=d.next_hour[m & (half == 1)].mean() / h1, hour_lift_h2=d.next_hour[m & (half == 2)].mean() / h2,
|
||||
share_of_all_bursts=100 * d.burst[m].sum() / d.burst.sum()))
|
||||
print(pd.DataFrame(rows).round(2).to_string(index=False))
|
||||
hot = d[d.level == "hot"]
|
||||
kinds = pd.Series(np.where(hot.calendar, "calendar", "clock+extreme reading"), index=hot.index)
|
||||
print("hot bars by cause:", (100 * kinds.value_counts(normalize=True)).round(1).to_dict())
|
||||
hot_hours = hot.index.floor("h").unique()
|
||||
days = pd.Series(hot_hours.date).value_counts()
|
||||
print(f"distinct HOT hours per calendar day: mean {days.mean():.1f}, median {days.median():.0f}, max {days.max()} (≈ heads-up messages/day before the daily cap)")
|
||||
hb = d.burst.groupby(d.index.floor("h")).max(); hh = pd.Series(1, index=hot_hours)
|
||||
print(f"of all hours with a violent bar, the hour was flagged HOT at its start or during: {100*hb[hb==1].index.isin(hot_hours).mean():.1f}%")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -55,6 +55,13 @@
|
||||
"max_per_day": 12,
|
||||
"candles_bars": 20
|
||||
},
|
||||
"violent_watch": {
|
||||
"enabled": false,
|
||||
"alert_line": true,
|
||||
"headsup": true,
|
||||
"max_headsup_per_day": 12,
|
||||
"headsup_min_level": "hot"
|
||||
},
|
||||
"signal_log": {
|
||||
"enabled": true,
|
||||
"path": "logs/signals.db",
|
||||
|
||||
@@ -50,11 +50,13 @@ from logging.handlers import TimedRotatingFileHandler
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from agent_panel import attach_panel
|
||||
from app_config import get_app_config, violent_watch_enabled
|
||||
from config import PlatformConfig, Timeframe, XAUUSD
|
||||
from main import report_to_dict
|
||||
from news_engine import NewsEngine
|
||||
from pair_signals import produce_signal
|
||||
from providers import make_data_provider
|
||||
from violent_watch import attach_watch
|
||||
import providers.base as _providers_base
|
||||
|
||||
logger = logging.getLogger("dashboard")
|
||||
@@ -359,6 +361,11 @@ class DashboardPublisher:
|
||||
try:
|
||||
report = produce_signal(XAUUSD, tf, self._provider(),
|
||||
CONFIRMATIONS[tf])
|
||||
# Display-only violent-move watch (dark by default). Never
|
||||
# raises and never touches the signal — only watch_line.
|
||||
if violent_watch_enabled():
|
||||
attach_watch(report, tf,
|
||||
get_app_config().violent_watch_settings)
|
||||
self._reports[tf.value] = report
|
||||
self.snapshots[tf.value] = {
|
||||
"status": "ok",
|
||||
|
||||
@@ -86,6 +86,7 @@
|
||||
<li><b>Grade (A+, A, B, C):</b> the bot's own mark for how clean the setup is. A+ is best. It never sends anything worse than C.</li>
|
||||
<li><b>Confidence and the indicator list:</b> extra detail for curious people. You don't need to understand it to trade the signal.</li>
|
||||
<li><b>⚠️ Warning lines:</b> things the bot noticed, like high volatility. They are "be aware," not "don't trade."</li>
|
||||
<li><b>🌪 Violent-move watch:</b> now and then the bot adds a line saying the next hour could produce very big candles — usually because an important American economic number is about to come out, or the market is in an hour that is historically wild, or the price has stretched a long way from normal. It is only a "hold on tight" note. It does not change the signal, and it is not a reason to close or move your stop loss early — the bot was tested doing exactly that and it lost money. Your stop loss is already your protection; leave it where it is.</li>
|
||||
<li><b>📰 News line:</b> a short plain-English note on what is moving the gold price right now and why, written from the last few hours of headlines. It is background information, not a reason to trade.</li>
|
||||
<li><b>The trade setup at the bottom is the only part you must act on.</b></li>
|
||||
</ul>
|
||||
|
||||
@@ -69,6 +69,7 @@
|
||||
<tr><td><b>Score / confidence</b></td><td>The regime-weighted composite of all nine indicators. Bigger absolute score = stronger consensus.</td></tr>
|
||||
<tr><td><b>MTF aligned</b></td><td>The two confirmation timeframes agree with the trade direction. Signals without alignment are filtered out on M5.</td></tr>
|
||||
<tr><td><b>Warnings ⚠️</b></td><td>Context, not veto. Backtesting shows warned trades perform about the same — strong momentum often triggers warnings.</td></tr>
|
||||
<tr><td><b>Violent-move watch 🌪</b></td><td>An occasional extra line saying the next hour looks unusually likely to produce a very large candle — because a high-impact US release is due (or just landed), because the clock is in a historically wild slot, or because price is stretched outside its bands. It is a heads-up about <i>volatility</i>, not a direction call, and it <b>never</b> changes the signal, the grade or the trade setup — the same alert would be sent without it. Do not close or tighten an open trade because of it: every such protective action was tested over three years and lost money. Your stop is the protection.</td></tr>
|
||||
<tr><td><b>Market context 📰</b></td><td>An AI-written note in plain English on what is moving the gold price right now, why it matters, and the next report to watch. It is built only from headlines published in the last 4 hours that lean one way (neutral items are dropped) plus today's high-impact releases. Informational only — it never blocks or grades a signal.</td></tr>
|
||||
<tr><td><b>SL / TP1 / RR</b></td><td>Stop loss sits behind the nearest swing level with an ATR buffer. TP1 is the nearest opposing level. <b>1R</b> = the entry-to-SL distance = the amount you risk.</td></tr>
|
||||
<tr><td><b>Trail plan 🔄</b></td><td>The real exit. Once price moves +0.6R in your favor, move the stop to trail 0.75R behind the best price and update it at each candle close. <b>TP1 is advisory — let winners run.</b></td></tr>
|
||||
|
||||
@@ -13,7 +13,7 @@ Design philosophy:
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Tuple
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
@@ -32,6 +32,13 @@ class IndicatorSignal:
|
||||
confidence: float # 0-100
|
||||
description: str
|
||||
weight: float = 1.0
|
||||
# RECORD-KEEPING ONLY (stamped by _compute_composite; nothing reads
|
||||
# them back). effective_weight = weight x regime multiplier actually
|
||||
# used, weighted_score = its contribution to the composite. Both stay
|
||||
# None for indicators the composite skipped (weight 0, `_`-prefixed
|
||||
# private series) and for analyses that never reached _compute_composite.
|
||||
effective_weight: Optional[float] = None
|
||||
weighted_score: Optional[float] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -50,6 +57,46 @@ class TechnicalAnalysis:
|
||||
pattern_signals: List[str] = field(default_factory=list)
|
||||
|
||||
|
||||
# NOTE: ROC is directionally redundant with MACD/Keltner/EMA
|
||||
# (0.8-0.94 correlation), but zeroing its weight was backtested
|
||||
# 2026-07 and LOST 13R — the redundant vote effectively raises
|
||||
# the composite bar and the thresholds are calibrated with it.
|
||||
# Don't remove without retuning score thresholds.
|
||||
_TRENDING_WEIGHTS = {
|
||||
"ema_trend": 2.0, "rsi": 1.0, "macd": 2.0,
|
||||
"bollinger": 0.5, "keltner": 1.5, "roc": 1.5,
|
||||
"candlestick": 0.8, "divergence": 1.0, "adx": 0.8,
|
||||
}
|
||||
_DEFAULT_REGIME_WEIGHTS = {
|
||||
"trending_up": _TRENDING_WEIGHTS,
|
||||
"trending_down": _TRENDING_WEIGHTS,
|
||||
"trending": _TRENDING_WEIGHTS,
|
||||
"ranging": {
|
||||
"ema_trend": 0.3, "rsi": 1.5, "macd": 1.5,
|
||||
"bollinger": 1.5, "keltner": 2.0, "roc": 0.5,
|
||||
"candlestick": 1.0, "divergence": 2.5, "adx": 0.5,
|
||||
},
|
||||
"transitioning": {
|
||||
"ema_trend": 1.5, "rsi": 1.0, "macd": 2.5,
|
||||
"bollinger": 1.0, "keltner": 2.5, "roc": 2.0,
|
||||
"candlestick": 1.0, "divergence": 2.0, "adx": 1.0,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def regime_weight_table(config: TechnicalConfig) -> Dict[str, Dict[str, float]]:
|
||||
"""The regime -> indicator weight multipliers _compute_composite runs
|
||||
with (``regime_weights_override`` wins, else the gold-optimized
|
||||
defaults above).
|
||||
|
||||
Pure and side-effect free: ONE source of truth shared by the composite
|
||||
and by signal_aggregator.engine_config_fingerprint. Returns a copy, so
|
||||
a caller can never mutate the shipped table."""
|
||||
override = getattr(config, "regime_weights_override", None)
|
||||
table = override if override else _DEFAULT_REGIME_WEIGHTS
|
||||
return {regime: dict(weights) for regime, weights in table.items()}
|
||||
|
||||
|
||||
class TechnicalEngine:
|
||||
"""
|
||||
Core technical analysis engine.
|
||||
@@ -1081,35 +1128,9 @@ class TechnicalEngine:
|
||||
total_weight = 0.0
|
||||
total_confidence = 0.0
|
||||
|
||||
# Use override weights if provided, else gold-optimized defaults
|
||||
if self.config.regime_weights_override:
|
||||
regime_weights = self.config.regime_weights_override
|
||||
else:
|
||||
# NOTE: ROC is directionally redundant with MACD/Keltner/EMA
|
||||
# (0.8-0.94 correlation), but zeroing its weight was backtested
|
||||
# 2026-07 and LOST 13R — the redundant vote effectively raises
|
||||
# the composite bar and the thresholds are calibrated with it.
|
||||
# Don't remove without retuning score thresholds.
|
||||
_trending = {
|
||||
"ema_trend": 2.0, "rsi": 1.0, "macd": 2.0,
|
||||
"bollinger": 0.5, "keltner": 1.5, "roc": 1.5,
|
||||
"candlestick": 0.8, "divergence": 1.0, "adx": 0.8,
|
||||
}
|
||||
regime_weights = {
|
||||
"trending_up": _trending,
|
||||
"trending_down": _trending,
|
||||
"trending": _trending,
|
||||
"ranging": {
|
||||
"ema_trend": 0.3, "rsi": 1.5, "macd": 1.5,
|
||||
"bollinger": 1.5, "keltner": 2.0, "roc": 0.5,
|
||||
"candlestick": 1.0, "divergence": 2.5, "adx": 0.5,
|
||||
},
|
||||
"transitioning": {
|
||||
"ema_trend": 1.5, "rsi": 1.0, "macd": 2.5,
|
||||
"bollinger": 1.0, "keltner": 2.5, "roc": 2.0,
|
||||
"candlestick": 1.0, "divergence": 2.0, "adx": 1.0,
|
||||
},
|
||||
}
|
||||
# Override weights if provided, else gold-optimized defaults
|
||||
# (regime_weight_table is the single source of truth).
|
||||
regime_weights = regime_weight_table(self.config)
|
||||
|
||||
active_regime_weights = regime_weights.get(result.market_regime, {})
|
||||
|
||||
@@ -1123,6 +1144,9 @@ class TechnicalEngine:
|
||||
|
||||
signal_score = ind.signal.value
|
||||
weighted_score = signal_score * weight * (ind.confidence / 100.0)
|
||||
# Record-keeping stamp (no maths depends on these two lines).
|
||||
ind.effective_weight = float(weight)
|
||||
ind.weighted_score = float(weighted_score)
|
||||
|
||||
total_score += weighted_score
|
||||
total_weight += weight
|
||||
|
||||
@@ -22,10 +22,12 @@ if hasattr(sys.stderr, "reconfigure"):
|
||||
sys.stderr.reconfigure(encoding="utf-8")
|
||||
|
||||
from agent_panel import attach_panel
|
||||
from app_config import get_app_config, violent_watch_enabled
|
||||
from config import PlatformConfig, Timeframe, SignalStrength, XAUUSD
|
||||
from market_outlook import OutlookResult, maybe_run_outlook
|
||||
from pair_signals import PAIR_PRODUCERS
|
||||
from signal_aggregator import SignalAggregator, SignalReport
|
||||
from violent_watch import attach_watch
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
@@ -160,6 +162,8 @@ def print_signal_report(report: SignalReport, detail: bool = False):
|
||||
if report.warnings:
|
||||
for w in report.warnings:
|
||||
print(f" {Colors.YELLOW}⚠{Colors.RESET} {w}")
|
||||
if report.watch_line:
|
||||
print(f" {Colors.YELLOW}🌪{Colors.RESET} {report.watch_line}")
|
||||
if report.event_description and report.event_risk != "clear":
|
||||
print(f" {Colors.YELLOW}📅{Colors.RESET} {report.event_description}")
|
||||
if report.news_description and report.news_description != "Neutral news sentiment":
|
||||
@@ -246,6 +250,7 @@ def report_to_dict(report: SignalReport) -> dict:
|
||||
"panel_created_at": report.panel_created_at,
|
||||
"panel_cache_status": report.panel_cache_status,
|
||||
"panel_payload_hash": report.panel_payload_hash,
|
||||
"watch_line": report.watch_line,
|
||||
"bullish_reasons": report.bullish_reasons,
|
||||
"bearish_reasons": report.bearish_reasons,
|
||||
"warnings": report.warnings,
|
||||
@@ -373,6 +378,12 @@ def run_live(args, config: PlatformConfig):
|
||||
if getattr(args, "panel", False):
|
||||
attach_panel(report, surface="cli", force=True)
|
||||
|
||||
# Display-only violent-move watch (dark by default; never touches
|
||||
# the signal, only report.watch_line).
|
||||
if violent_watch_enabled():
|
||||
attach_watch(report, config.primary_timeframe,
|
||||
get_app_config().violent_watch_settings)
|
||||
|
||||
outlook = None
|
||||
if getattr(args, "outlook", False):
|
||||
outlook = maybe_run_outlook(surface="cli", force=True)
|
||||
|
||||
@@ -0,0 +1,622 @@
|
||||
"""
|
||||
ForexBot2 — Position sizing (stake-multiplier overlay)
|
||||
======================================================
|
||||
|
||||
Single source of truth for **decision-time features** and the resulting
|
||||
**stake multiplier**. Everything here is designed to be called from
|
||||
inside the signal path so that live and backtest see byte-identical
|
||||
inputs (rule 4: live == backtest parity).
|
||||
|
||||
Design contract
|
||||
---------------
|
||||
* **Bounded lookbacks.** No feature may read further back than the live
|
||||
fetch depth of its timeframe (``LIVE_DEPTH_BARS``). The primary frame
|
||||
is hard-capped at ``MAX_LOOKBACK_BARS`` rows, confirmation frames at
|
||||
60 rows, and the burst scan at 200 rows, so a feature computed on a
|
||||
full backtest frame equals the same feature computed on the shorter
|
||||
live tail.
|
||||
* **Closed bars only.** The frames handed in are exactly the frames
|
||||
``analyze_from_dataframes`` received: UTC tz-aware, oldest-first,
|
||||
capitalized OHLCV columns, forming candle already dropped. The last
|
||||
row is the decision bar; nothing peeks past it.
|
||||
* **Never raises into a scan.** ``build_features`` and
|
||||
``stake_multiplier`` swallow every exception (logged at DEBUG) and
|
||||
fall back to neutral defaults. A sizing failure can never break a
|
||||
signal.
|
||||
* **Advisory today.** ``stake_multiplier`` returns ``1.0`` in every case
|
||||
at this stage — artifact evaluation lands in a later step. The guard
|
||||
order is already implemented so the wiring is testable now.
|
||||
|
||||
Dependencies are deliberately limited to numpy/pandas plus ``config``
|
||||
(the ``Timeframe`` enum): this module ships live and must not drag the
|
||||
provider/news stack into any process that only wants features.
|
||||
|
||||
VERDICT 2026-09-06 (backtest/research/sizing_campaign_2026-09.md): the walk-forward
|
||||
stake-multiplier model FAILED the pre-registered acceptance rule on both M5 and
|
||||
M15 (M5 a clean null; M15 lift inside the label-shuffle null and explained by
|
||||
BUY-vs-SELL). NO artifact ships and nothing reads a multiplier live. This module
|
||||
is research plumbing: `build_features` feeds the backtester's --export-features
|
||||
flag; `stake_multiplier` returns 1.0 unless a validated artifact is ever added
|
||||
under a NEW pre-registration.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
from pathlib import Path
|
||||
from typing import Dict, Iterable, Optional, Sequence, Tuple
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from config import SignalStrength, Timeframe
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
#: Bump whenever FEATURE_ORDER or any feature's definition changes.
|
||||
#: A trained artifact carrying a different value is ignored.
|
||||
FEATURE_VERSION = 1
|
||||
|
||||
#: Bars each timeframe actually holds in LIVE mode. Any feature whose
|
||||
#: lookback exceeds its timeframe's depth here would be unreproducible
|
||||
#: live, so the windows below stay far inside these numbers.
|
||||
LIVE_DEPTH_BARS: Dict[Timeframe, int] = {
|
||||
Timeframe.M5: 5000,
|
||||
Timeframe.M15: 5000,
|
||||
Timeframe.H1: 1460,
|
||||
Timeframe.H4: 380,
|
||||
Timeframe.D1: 300,
|
||||
}
|
||||
|
||||
#: Hard cap on any primary-timeframe window used in this module.
|
||||
MAX_LOOKBACK_BARS = 500
|
||||
|
||||
#: Bars of each confirmation timeframe used by the cross-TF features.
|
||||
CONFIRM_LOOKBACK_BARS = 60
|
||||
|
||||
#: Window for the "bars since the last volatility burst" feature. Also
|
||||
#: the sentinel returned when no burst is present in the window.
|
||||
BURST_LOOKBACK_BARS = 200
|
||||
|
||||
#: Burst threshold: bar range >= this multiple of the PRIOR bar's ATR.
|
||||
BURST_ATR_MULTIPLE = 3.0
|
||||
|
||||
#: Candle duration in seconds. Deliberate local copy of
|
||||
#: ``providers.base.BaseDataProvider.CANDLE_SECONDS`` — importing the
|
||||
#: providers package here would pull requests/twelvedata into every
|
||||
#: feature-only consumer. A test asserts the copies agree.
|
||||
CANDLE_SECONDS: Dict[Timeframe, int] = {
|
||||
Timeframe.M5: 300,
|
||||
Timeframe.M15: 900,
|
||||
Timeframe.H1: 3600,
|
||||
Timeframe.H4: 14400,
|
||||
Timeframe.D1: 86400,
|
||||
}
|
||||
|
||||
#: Grade -> ordinal used by ``f_quality_rank`` (D/F both fall to 0).
|
||||
_QUALITY_RANK = {"A+": 4.0, "A": 3.0, "B": 2.0, "C": 1.0}
|
||||
|
||||
_BULLISH = (SignalStrength.BUY, SignalStrength.STRONG_BUY)
|
||||
_BEARISH = (SignalStrength.SELL, SignalStrength.STRONG_SELL)
|
||||
|
||||
|
||||
FEATURE_ORDER: Tuple[str, ...] = (
|
||||
# ── Time ──────────────────────────────────────────────────────
|
||||
"f_hour_utc",
|
||||
"f_hour_sin",
|
||||
"f_hour_cos",
|
||||
"f_weekday",
|
||||
"f_sess_asian",
|
||||
"f_sess_london",
|
||||
"f_sess_overlap",
|
||||
"f_sess_ny",
|
||||
"f_minutes_to_hour",
|
||||
"f_day_of_month",
|
||||
"f_first_friday",
|
||||
# ── Volatility / setup geometry ───────────────────────────────
|
||||
"f_atr",
|
||||
"f_atr_pct",
|
||||
"f_atr_rel_500",
|
||||
"f_range_pctile_500",
|
||||
"f_bar_range_atr",
|
||||
"f_close_pos_in_bar",
|
||||
"f_risk_price",
|
||||
"f_sl_atr",
|
||||
"f_tp1_atr",
|
||||
"f_rr_planned",
|
||||
# ── Exhaustion block ──────────────────────────────────────────
|
||||
"f_swing_dist_atr",
|
||||
"f_ema21_dist_atr",
|
||||
"f_ema200_dist_atr",
|
||||
"f_ema200_dist_pct",
|
||||
"f_exhaustion_count",
|
||||
"f_tag_exhaustion",
|
||||
"f_tag_no_pullback",
|
||||
"f_tag_weak_pullback",
|
||||
"f_tag_rsi_extended",
|
||||
"f_tag_bb_overextended",
|
||||
"f_tag_ema200_near",
|
||||
"f_tag_ema200_far",
|
||||
"f_penalty_total",
|
||||
"f_momentum_fading",
|
||||
# ── Signal quality ────────────────────────────────────────────
|
||||
"f_score_dir",
|
||||
"f_confidence",
|
||||
"f_quality_rank",
|
||||
"f_ind_agree_frac",
|
||||
"f_ind_oppose_frac",
|
||||
"f_is_buy",
|
||||
"f_strat_atr",
|
||||
"f_strat_pb",
|
||||
"f_strat_cp",
|
||||
"f_strat_generic",
|
||||
# ── Multi-timeframe ───────────────────────────────────────────
|
||||
"f_mtf_primary",
|
||||
"f_mtf_c1",
|
||||
"f_mtf_c2",
|
||||
"f_mtf_agreement",
|
||||
# ── Primary-TF indicators ─────────────────────────────────────
|
||||
"f_rsi",
|
||||
"f_rsi_dir",
|
||||
"f_bb_pos",
|
||||
"f_bb_pos_dir",
|
||||
"f_adx",
|
||||
"f_kc_ext_atr",
|
||||
"f_roc_dir",
|
||||
"f_macd_hist_dir",
|
||||
# ── Price path ────────────────────────────────────────────────
|
||||
"f_ret_6_atr",
|
||||
"f_ret_12_atr",
|
||||
"f_ret_24_atr",
|
||||
"f_ret_48_atr",
|
||||
"f_consec_closes_dir",
|
||||
"f_bars_since_burst",
|
||||
# ── Cross-timeframe extension ─────────────────────────────────
|
||||
"f_h1_ext_atr",
|
||||
"f_c_other_ext_atr",
|
||||
)
|
||||
|
||||
#: Value each feature falls back to when it cannot be computed. Only
|
||||
#: ``f_bars_since_burst`` has a non-zero sentinel (its "none in window"
|
||||
#: cap); everything else degrades to a neutral 0.0.
|
||||
_DEFAULTS: Dict[str, float] = {name: 0.0 for name in FEATURE_ORDER}
|
||||
_DEFAULTS["f_bars_since_burst"] = float(BURST_LOOKBACK_BARS)
|
||||
|
||||
#: Penalty tags recorded by SignalAggregator._compute_final_signal that
|
||||
#: this module exposes as one-hot features.
|
||||
_TAG_FEATURES = (
|
||||
("f_tag_exhaustion", "exhaustion"),
|
||||
("f_tag_no_pullback", "no_pullback"),
|
||||
("f_tag_weak_pullback", "weak_pullback"),
|
||||
("f_tag_rsi_extended", "rsi_extended"),
|
||||
("f_tag_bb_overextended", "bb_overextended"),
|
||||
("f_tag_ema200_near", "ema200_near"),
|
||||
("f_tag_ema200_far", "ema200_far"),
|
||||
)
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
# Small helpers
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
|
||||
def _finite(value, default: float = 0.0) -> float:
|
||||
"""Coerce to a finite Python float, falling back to ``default``."""
|
||||
try:
|
||||
out = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return float(default)
|
||||
if not math.isfinite(out):
|
||||
return float(default)
|
||||
return out
|
||||
|
||||
|
||||
def wilder_atr(df: pd.DataFrame, period: int = 14) -> pd.Series:
|
||||
"""Wilder ATR, identical convention to ``engine._compute_atr``.
|
||||
|
||||
True Range, then an EWM with ``alpha = 1/period``,
|
||||
``min_periods = period`` and ``adjust=False``.
|
||||
"""
|
||||
high, low, close = df["High"], df["Low"], df["Close"]
|
||||
tr1 = high - low
|
||||
tr2 = (high - close.shift()).abs()
|
||||
tr3 = (low - close.shift()).abs()
|
||||
tr = pd.concat([tr1, tr2, tr3], axis=1).max(axis=1)
|
||||
return tr.ewm(alpha=1.0 / period, min_periods=period, adjust=False).mean()
|
||||
|
||||
|
||||
def decision_time(primary_df: pd.DataFrame, tf: Timeframe) -> pd.Timestamp:
|
||||
"""The clock every gate uses: last CLOSED bar label + its duration."""
|
||||
last = pd.Timestamp(primary_df.index[-1])
|
||||
return last + pd.Timedelta(seconds=CANDLE_SECONDS.get(tf, 3600))
|
||||
|
||||
|
||||
def session_label(hour_utc: int) -> str:
|
||||
"""Trading-session bucket for a UTC hour."""
|
||||
h = int(hour_utc)
|
||||
if 3 <= h <= 6:
|
||||
return "asian"
|
||||
if 7 <= h <= 12:
|
||||
return "london"
|
||||
if 13 <= h <= 16:
|
||||
return "overlap"
|
||||
if 17 <= h <= 22:
|
||||
return "ny"
|
||||
return "off"
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
# Artifact plumbing
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
|
||||
#: path -> (mtime, artifact-or-None)
|
||||
_ARTIFACT_CACHE: Dict[str, Tuple[float, Optional[dict]]] = {}
|
||||
|
||||
|
||||
def artifact_path_for(primary_tf: Timeframe, base_dir) -> Path:
|
||||
"""Where the trained artifact for one primary timeframe lives."""
|
||||
return Path(base_dir) / f"xauusd_{primary_tf.value}.json"
|
||||
|
||||
|
||||
def load_artifact(path) -> Optional[dict]:
|
||||
"""Load a sizing artifact (JSON object), cached on (path, mtime).
|
||||
|
||||
Returns ``None`` when the file is missing, unreadable, not JSON, or
|
||||
not a JSON object. Never raises.
|
||||
"""
|
||||
try:
|
||||
p = Path(path)
|
||||
mtime = p.stat().st_mtime
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
key = str(p)
|
||||
cached = _ARTIFACT_CACHE.get(key)
|
||||
if cached is not None and cached[0] == mtime:
|
||||
return cached[1]
|
||||
|
||||
data: Optional[dict]
|
||||
try:
|
||||
with p.open("r", encoding="utf-8") as fh:
|
||||
loaded = json.load(fh)
|
||||
data = loaded if isinstance(loaded, dict) else None
|
||||
except Exception:
|
||||
logger.debug("position_sizing: artifact %s is unreadable", key,
|
||||
exc_info=True)
|
||||
data = None
|
||||
|
||||
_ARTIFACT_CACHE[key] = (mtime, data)
|
||||
return data
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
# Feature construction
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
|
||||
def _direction(setup, report) -> float:
|
||||
"""+1.0 for a long, -1.0 for a short — from the TRADE, not the score."""
|
||||
direction = getattr(setup, "direction", None)
|
||||
if direction is None:
|
||||
return 1.0 if _finite(getattr(report, "final_score", 0.0)) >= 0 else -1.0
|
||||
return 1.0 if str(direction).upper() == "BUY" else -1.0
|
||||
|
||||
|
||||
def _ext_vs_sma(df: pd.DataFrame, d: float) -> float:
|
||||
"""Direction-signed distance from the 20-bar SMA in ATR units.
|
||||
|
||||
Computed on the LAST ``CONFIRM_LOOKBACK_BARS`` rows only, so the
|
||||
value is identical on a full history and on its live-depth tail.
|
||||
"""
|
||||
if df is None or len(df) < 21:
|
||||
return 0.0
|
||||
tail = df.iloc[-CONFIRM_LOOKBACK_BARS:]
|
||||
sma = tail["Close"].rolling(20).mean().iloc[-1]
|
||||
atr = wilder_atr(tail, 14).iloc[-1]
|
||||
atr_f = _finite(atr, 0.0)
|
||||
if atr_f <= 0:
|
||||
return 0.0
|
||||
return d * (_finite(tail["Close"].iloc[-1]) - _finite(sma)) / atr_f
|
||||
|
||||
|
||||
def _fill_time(out: Dict[str, float], dt: pd.Timestamp) -> None:
|
||||
hour = int(dt.hour)
|
||||
out["f_hour_utc"] = float(hour)
|
||||
out["f_hour_sin"] = float(math.sin(2.0 * math.pi * hour / 24.0))
|
||||
out["f_hour_cos"] = float(math.cos(2.0 * math.pi * hour / 24.0))
|
||||
weekday = int(dt.weekday())
|
||||
out["f_weekday"] = float(weekday)
|
||||
label = session_label(hour)
|
||||
out["f_sess_asian"] = 1.0 if label == "asian" else 0.0
|
||||
out["f_sess_london"] = 1.0 if label == "london" else 0.0
|
||||
out["f_sess_overlap"] = 1.0 if label == "overlap" else 0.0
|
||||
out["f_sess_ny"] = 1.0 if label == "ny" else 0.0
|
||||
out["f_minutes_to_hour"] = float((60 - int(dt.minute)) % 60)
|
||||
out["f_day_of_month"] = float(int(dt.day))
|
||||
out["f_first_friday"] = 1.0 if (weekday == 4 and int(dt.day) <= 7) else 0.0
|
||||
|
||||
|
||||
def _fill_geometry(out: Dict[str, float], P: pd.DataFrame, setup,
|
||||
atr: float, atr_ok: bool, norm, d: float) -> None:
|
||||
out["f_atr"] = max(0.0, _finite(atr))
|
||||
ranges = (P["High"] - P["Low"]).astype(float)
|
||||
last_range = _finite(ranges.iloc[-1])
|
||||
mean_range = _finite(ranges.mean())
|
||||
out["f_atr_rel_500"] = (_finite(atr) / mean_range) if mean_range > 0 else 0.0
|
||||
n = int(len(ranges))
|
||||
out["f_range_pctile_500"] = (
|
||||
float((ranges.to_numpy() < last_range).sum()) / n if n else 0.0)
|
||||
out["f_bar_range_atr"] = norm(last_range)
|
||||
|
||||
high = _finite(P["High"].iloc[-1])
|
||||
low = _finite(P["Low"].iloc[-1])
|
||||
close = _finite(P["Close"].iloc[-1])
|
||||
span = high - low
|
||||
if span > 0:
|
||||
out["f_close_pos_in_bar"] = (
|
||||
(close - low) / span if d > 0 else (high - close) / span)
|
||||
else:
|
||||
out["f_close_pos_in_bar"] = 0.5
|
||||
|
||||
entry = _finite(getattr(setup, "entry_price", None))
|
||||
stop = _finite(getattr(setup, "stop_loss", None))
|
||||
tp1 = _finite(getattr(setup, "take_profit_1", None))
|
||||
risk = abs(entry - stop)
|
||||
out["f_risk_price"] = risk
|
||||
out["f_sl_atr"] = norm(risk)
|
||||
out["f_tp1_atr"] = norm(abs(tp1 - entry))
|
||||
out["f_rr_planned"] = _finite(getattr(setup, "risk_reward", None))
|
||||
|
||||
|
||||
def _fill_exhaustion(out: Dict[str, float], report, KL: Dict[str, float],
|
||||
px: float, d: float, norm) -> None:
|
||||
if d > 0:
|
||||
swings = [KL[k] for k in ("swing_low_1", "swing_low_2", "swing_low_3")
|
||||
if KL.get(k) is not None]
|
||||
out["f_swing_dist_atr"] = norm(px - min(swings)) if swings else 0.0
|
||||
else:
|
||||
swings = [KL[k] for k in ("swing_high_1", "swing_high_2", "swing_high_3")
|
||||
if KL.get(k) is not None]
|
||||
out["f_swing_dist_atr"] = norm(max(swings) - px) if swings else 0.0
|
||||
|
||||
ema21 = KL.get("ema_medium")
|
||||
out["f_ema21_dist_atr"] = (
|
||||
norm(d * (px - _finite(ema21))) if ema21 is not None else 0.0)
|
||||
|
||||
ema200 = KL.get("ema_trend")
|
||||
ema200_f = _finite(ema200, 0.0) if ema200 is not None else 0.0
|
||||
if ema200 is not None and ema200_f != 0.0:
|
||||
out["f_ema200_dist_atr"] = norm(d * (px - ema200_f))
|
||||
out["f_ema200_dist_pct"] = d * (px - ema200_f) / ema200_f * 100.0
|
||||
else:
|
||||
out["f_ema200_dist_atr"] = 0.0
|
||||
out["f_ema200_dist_pct"] = 0.0
|
||||
|
||||
out["f_exhaustion_count"] = _finite(getattr(report, "exhaustion_count", 0))
|
||||
tags = getattr(report, "penalty_tags", None) or {}
|
||||
for feature, tag in _TAG_FEATURES:
|
||||
out[feature] = 1.0 if tag in tags else 0.0
|
||||
out["f_penalty_total"] = _finite(getattr(report, "penalty_total", 0.0))
|
||||
out["f_momentum_fading"] = 1.0 if getattr(report, "momentum_fading", False) else 0.0
|
||||
|
||||
|
||||
def _fill_quality(out: Dict[str, float], report, setup, IND: dict,
|
||||
d: float) -> None:
|
||||
out["f_score_dir"] = d * _finite(getattr(report, "final_score", 0.0))
|
||||
out["f_confidence"] = _finite(getattr(report, "final_confidence", 0.0))
|
||||
out["f_quality_rank"] = _QUALITY_RANK.get(
|
||||
getattr(report, "signal_quality", ""), 0.0)
|
||||
|
||||
agree = oppose = total = 0
|
||||
for name, ind in (IND or {}).items():
|
||||
if str(name).startswith("_"):
|
||||
continue
|
||||
if _finite(getattr(ind, "weight", 0.0)) == 0.0:
|
||||
continue
|
||||
total += 1
|
||||
signal = getattr(ind, "signal", None)
|
||||
if d > 0:
|
||||
if signal in _BULLISH:
|
||||
agree += 1
|
||||
elif signal in _BEARISH:
|
||||
oppose += 1
|
||||
else:
|
||||
if signal in _BEARISH:
|
||||
agree += 1
|
||||
elif signal in _BULLISH:
|
||||
oppose += 1
|
||||
out["f_ind_agree_frac"] = (agree / total) if total else 0.0
|
||||
out["f_ind_oppose_frac"] = (oppose / total) if total else 0.0
|
||||
|
||||
out["f_is_buy"] = 1.0 if d > 0 else 0.0
|
||||
strategy = str(getattr(setup, "strategy", "") or "")
|
||||
# The ATR path stamps "XAU_ATR"; "XAU" is accepted for legacy setups.
|
||||
out["f_strat_atr"] = 1.0 if strategy in ("XAU_ATR", "XAU") else 0.0
|
||||
out["f_strat_pb"] = 1.0 if strategy == "XAU_PB" else 0.0
|
||||
out["f_strat_cp"] = 1.0 if strategy == "XAU_CP" else 0.0
|
||||
out["f_strat_generic"] = 1.0 if strategy == "GENERIC" else 0.0
|
||||
|
||||
|
||||
def _fill_mtf(out: Dict[str, float], report, primary_tf: Timeframe,
|
||||
confirmation_tfs: Sequence[Timeframe], d: float) -> None:
|
||||
scores = getattr(report, "mtf_scores", None) or {}
|
||||
out["f_mtf_primary"] = d * _finite(scores.get(primary_tf.value, 0.0))
|
||||
confirms = list(confirmation_tfs or [])
|
||||
for idx, key in ((0, "f_mtf_c1"), (1, "f_mtf_c2")):
|
||||
if idx < len(confirms):
|
||||
out[key] = d * _finite(scores.get(confirms[idx].value, 0.0))
|
||||
else:
|
||||
out[key] = 0.0
|
||||
out["f_mtf_agreement"] = 1.0 if getattr(report, "mtf_agreement", False) else 0.0
|
||||
|
||||
|
||||
def _fill_indicators(out: Dict[str, float], IND: dict, d: float, norm) -> None:
|
||||
IND = IND or {}
|
||||
|
||||
def _val(name, default):
|
||||
ind = IND.get(name)
|
||||
return _finite(getattr(ind, "value", None), default) if ind else default
|
||||
|
||||
rsi = _val("rsi", 50.0)
|
||||
out["f_rsi"] = rsi
|
||||
out["f_rsi_dir"] = d * (rsi - 50.0)
|
||||
bb_pos = _val("bollinger", 0.5) # bollinger.value = bb_position
|
||||
out["f_bb_pos"] = bb_pos
|
||||
out["f_bb_pos_dir"] = d * (bb_pos - 0.5)
|
||||
out["f_adx"] = _val("adx", 0.0)
|
||||
# keltner.value = close - kc_mid (signed extension from the mid line)
|
||||
out["f_kc_ext_atr"] = norm(d * _val("keltner", 0.0))
|
||||
out["f_roc_dir"] = d * _val("roc", 0.0)
|
||||
# macd.value = histogram
|
||||
out["f_macd_hist_dir"] = d * _val("macd", 0.0)
|
||||
|
||||
|
||||
def _fill_path(out: Dict[str, float], P: pd.DataFrame, d: float, norm) -> None:
|
||||
close = P["Close"].astype(float).to_numpy()
|
||||
n = len(close)
|
||||
for lag, key in ((6, "f_ret_6_atr"), (12, "f_ret_12_atr"),
|
||||
(24, "f_ret_24_atr"), (48, "f_ret_48_atr")):
|
||||
out[key] = norm(d * (close[-1] - close[-1 - lag])) if n >= lag + 1 else 0.0
|
||||
|
||||
run = 0.0
|
||||
if n >= 2:
|
||||
diffs = np.diff(close)
|
||||
last = diffs[-1]
|
||||
if last > 0:
|
||||
sign = 1.0
|
||||
elif last < 0:
|
||||
sign = -1.0
|
||||
else:
|
||||
sign = 0.0
|
||||
if sign != 0.0:
|
||||
count = 0
|
||||
for value in diffs[::-1]:
|
||||
if (value > 0 and sign > 0) or (value < 0 and sign < 0):
|
||||
count += 1
|
||||
else:
|
||||
break
|
||||
run = sign * count
|
||||
out["f_consec_closes_dir"] = max(-20.0, min(20.0, d * run))
|
||||
|
||||
out["f_bars_since_burst"] = float(BURST_LOOKBACK_BARS)
|
||||
prev_atr = wilder_atr(P, 14).shift(1).to_numpy(dtype=float)
|
||||
ranges = (P["High"] - P["Low"]).astype(float).to_numpy()
|
||||
start = max(0, n - BURST_LOOKBACK_BARS)
|
||||
for i in range(n - 1, start - 1, -1):
|
||||
threshold = prev_atr[i]
|
||||
if not math.isfinite(threshold) or threshold <= 0:
|
||||
continue
|
||||
if ranges[i] >= BURST_ATR_MULTIPLE * threshold:
|
||||
out["f_bars_since_burst"] = float((n - 1) - i)
|
||||
break
|
||||
|
||||
|
||||
def _fill_cross_tf(out: Dict[str, float], dfs: Dict[Timeframe, pd.DataFrame],
|
||||
confirmation_tfs: Sequence[Timeframe], d: float) -> None:
|
||||
out["f_h1_ext_atr"] = _ext_vs_sma(dfs.get(Timeframe.H1), d)
|
||||
out["f_c_other_ext_atr"] = 0.0
|
||||
for tf in (confirmation_tfs or []):
|
||||
if tf is Timeframe.H1:
|
||||
continue
|
||||
df = dfs.get(tf)
|
||||
if df is None or len(df) < 21:
|
||||
continue
|
||||
out["f_c_other_ext_atr"] = _ext_vs_sma(df, d)
|
||||
break
|
||||
|
||||
|
||||
def build_features(
|
||||
report,
|
||||
setup,
|
||||
dfs: Dict[Timeframe, pd.DataFrame],
|
||||
primary_tf: Timeframe,
|
||||
confirmation_tfs: Sequence[Timeframe],
|
||||
) -> Dict[str, float]:
|
||||
"""Decision-time feature vector for one signal.
|
||||
|
||||
Returns a dict whose keys are exactly ``FEATURE_ORDER`` (in order),
|
||||
every value a finite Python float. Degenerate inputs fall back to
|
||||
the neutral defaults instead of raising — this runs inside a scan.
|
||||
"""
|
||||
out: Dict[str, float] = dict(_DEFAULTS)
|
||||
|
||||
try:
|
||||
P = dfs.get(primary_tf)
|
||||
if P is None or len(P) == 0:
|
||||
return {k: out[k] for k in FEATURE_ORDER}
|
||||
P = P.iloc[-MAX_LOOKBACK_BARS:]
|
||||
|
||||
d = _direction(setup, report)
|
||||
primary = getattr(report, "primary_analysis", None)
|
||||
KL = dict(getattr(primary, "key_levels", None) or {})
|
||||
IND = getattr(primary, "indicators", None) or {}
|
||||
|
||||
atr_raw = KL.get("atr")
|
||||
atr = _finite(atr_raw, 0.0)
|
||||
atr_ok = atr > 0
|
||||
atr_div = atr if atr_ok else 1e-9
|
||||
|
||||
def norm(value: float) -> float:
|
||||
"""ATR-normalise, or 0.0 when ATR is missing/non-positive."""
|
||||
return (_finite(value) / atr_div) if atr_ok else 0.0
|
||||
|
||||
px = _finite(P["Close"].iloc[-1])
|
||||
out["f_atr_pct"] = _finite(KL.get("atr_pct"), 0.0)
|
||||
|
||||
for step, args in (
|
||||
(_fill_time, (out, decision_time(P, primary_tf))),
|
||||
(_fill_geometry, (out, P, setup, atr, atr_ok, norm, d)),
|
||||
(_fill_exhaustion, (out, report, KL, px, d, norm)),
|
||||
(_fill_quality, (out, report, setup, IND, d)),
|
||||
(_fill_mtf, (out, report, primary_tf, confirmation_tfs, d)),
|
||||
(_fill_indicators, (out, IND, d, norm)),
|
||||
(_fill_path, (out, P, d, norm)),
|
||||
(_fill_cross_tf, (out, dfs, confirmation_tfs, d)),
|
||||
):
|
||||
try:
|
||||
step(*args)
|
||||
except Exception:
|
||||
logger.debug("position_sizing: %s failed", step.__name__,
|
||||
exc_info=True)
|
||||
except Exception:
|
||||
logger.debug("position_sizing.build_features failed", exc_info=True)
|
||||
|
||||
return {k: _finite(out.get(k), _DEFAULTS[k]) for k in FEATURE_ORDER}
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
# Stake multiplier
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
|
||||
def stake_multiplier(
|
||||
report,
|
||||
setup,
|
||||
dfs: Dict[Timeframe, pd.DataFrame],
|
||||
primary_tf: Timeframe,
|
||||
confirmation_tfs: Sequence[Timeframe],
|
||||
artifact_dir=None,
|
||||
) -> float:
|
||||
"""Stake multiplier for a taken setup. Always ``1.0`` at this stage.
|
||||
|
||||
The guard order is already final — no artifact directory, a missing
|
||||
artifact file, an artifact built for a different ``FEATURE_VERSION``,
|
||||
or ANY exception all yield the neutral ``1.0``. Artifact evaluation
|
||||
lands in a later step; until then this is wiring, not a decision.
|
||||
"""
|
||||
try:
|
||||
if not artifact_dir:
|
||||
return 1.0
|
||||
artifact = load_artifact(artifact_path_for(primary_tf, artifact_dir))
|
||||
if artifact is None:
|
||||
return 1.0
|
||||
if artifact.get("feature_version") != FEATURE_VERSION:
|
||||
logger.debug(
|
||||
"position_sizing: artifact feature_version %r != %d — neutral",
|
||||
artifact.get("feature_version"), FEATURE_VERSION)
|
||||
return 1.0
|
||||
# Step 1: evaluation not implemented yet.
|
||||
return 1.0
|
||||
except Exception:
|
||||
logger.debug("position_sizing.stake_multiplier failed", exc_info=True)
|
||||
return 1.0
|
||||
Executable
+115
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
label_scan_outcomes.py — batch driver for the scan_outcomes table.
|
||||
|
||||
Labels model-free forward returns (1/4/24 h, $ and ATR-normalized, plus
|
||||
MFE/MAE, a 3xATR burst flag and a completeness flag) for every logged scan
|
||||
whose 24 h window has closed, from the closed-candle streams already
|
||||
stored in the DB. No provider, no network.
|
||||
|
||||
The live loggers do the same work opportunistically at the end of every
|
||||
scan (time-boxed to ~2 s); this script is the catch-up pass for a backlog
|
||||
— a freshly imported database, or rows whose bars only arrived later.
|
||||
|
||||
Idempotent: a row is rewritten only when its numbers change, so a settled
|
||||
database costs one read pass. Safe beside the live writers (short per-row
|
||||
transactions with a busy timeout). Read-only preview with --dry-run.
|
||||
|
||||
python scripts/label_scan_outcomes.py --dry-run
|
||||
python scripts/label_scan_outcomes.py
|
||||
python scripts/label_scan_outcomes.py --db logs/signals.db --budget 5000
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import app_config # noqa: E402
|
||||
from signal_log import SCAN_LABEL_MIN_AGE_H, SignalLogger # noqa: E402
|
||||
|
||||
CHUNK = 2000
|
||||
|
||||
|
||||
def _counts(con: sqlite3.Connection, symbol: str,
|
||||
labelled: bool = True) -> dict:
|
||||
"""Candidate / labelled / complete / pending counts (read-only).
|
||||
|
||||
The age filter uses logged_at (always an ISO string) as a proxy for
|
||||
the bar close — the labeller itself parses the real bar label."""
|
||||
scans = ("SELECT COUNT(*) FROM signals s WHERE s.symbol = ?"
|
||||
" AND s.indicators_json IS NOT NULL AND s.price > 0")
|
||||
due = (scans + " AND s.logged_at <"
|
||||
f" datetime('now', '-{int(SCAN_LABEL_MIN_AGE_H)} hours')")
|
||||
out = {
|
||||
"scan rows with indicators": con.execute(scans, (symbol,)).fetchone()[0],
|
||||
"of those, window elapsed": con.execute(due, (symbol,)).fetchone()[0],
|
||||
}
|
||||
if labelled:
|
||||
out["labelled"] = con.execute(
|
||||
"SELECT COUNT(*) FROM scan_outcomes").fetchone()[0]
|
||||
out["complete"] = con.execute(
|
||||
"SELECT COUNT(*) FROM scan_outcomes WHERE complete = 1"
|
||||
).fetchone()[0]
|
||||
out["pending (labelled, incomplete)"] = con.execute(
|
||||
"SELECT COUNT(*) FROM scan_outcomes WHERE complete = 0"
|
||||
).fetchone()[0]
|
||||
return out
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[1])
|
||||
ap.add_argument("--db", default=None, help="defaults to signal_log.path")
|
||||
ap.add_argument("--dry-run", action="store_true",
|
||||
help="read-only candidate counts, writes nothing")
|
||||
ap.add_argument("--budget", type=int, default=100000,
|
||||
help="maximum rows to write in this run")
|
||||
ap.add_argument("--symbol", default="XAU/USD")
|
||||
args = ap.parse_args(argv)
|
||||
path = args.db or app_config.signal_log_path()
|
||||
if not os.path.exists(path):
|
||||
print(f"no database at {path}")
|
||||
return 1
|
||||
|
||||
if args.dry_run:
|
||||
con = sqlite3.connect(f"file:{os.path.abspath(path)}?mode=ro", uri=True)
|
||||
try:
|
||||
have = {r[0] for r in con.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table'")}
|
||||
if "scan_outcomes" not in have:
|
||||
print("scan_outcomes table not present yet (created on the"
|
||||
" first non-dry run / next logged scan)")
|
||||
for key, value in _counts(con, args.symbol,
|
||||
labelled="scan_outcomes" in have).items():
|
||||
print(f" {key}: {value}")
|
||||
finally:
|
||||
con.close()
|
||||
return 0
|
||||
|
||||
lg = SignalLogger(path)
|
||||
try:
|
||||
print("before:", _counts(lg._conn, args.symbol))
|
||||
written = 0
|
||||
previous_complete = -1
|
||||
while written < args.budget:
|
||||
n = lg.label_scan_outcomes(
|
||||
budget=min(CHUNK, args.budget - written), time_budget=120.0)
|
||||
written += n
|
||||
complete = lg._conn.execute(
|
||||
"SELECT COUNT(*) FROM scan_outcomes WHERE complete = 1"
|
||||
).fetchone()[0]
|
||||
# Stop on a pass that wrote nothing, or one that could not
|
||||
# complete a single further row (only re-labelling stragglers).
|
||||
if n == 0 or complete == previous_complete:
|
||||
break
|
||||
previous_complete = complete
|
||||
print(f"labelled (rows written): {written}")
|
||||
print("after:", _counts(lg._conn, args.symbol))
|
||||
finally:
|
||||
lg.close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+53
-7
@@ -9,7 +9,7 @@ import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, List, Optional
|
||||
from zoneinfo import ZoneInfo
|
||||
@@ -21,7 +21,7 @@ _NY_TZ = ZoneInfo("America/New_York")
|
||||
from chart_patterns import detect_chart_pattern
|
||||
from config import PlatformConfig, SignalStrength, Timeframe
|
||||
from providers import BaseDataProvider, make_data_provider
|
||||
from engine import TechnicalEngine, TechnicalAnalysis
|
||||
from engine import TechnicalEngine, TechnicalAnalysis, regime_weight_table
|
||||
from news_engine import NewsEngine, FundamentalContext
|
||||
import signal_log
|
||||
|
||||
@@ -70,6 +70,10 @@ class TradeSetup:
|
||||
# platform defaults; 0.0 means trailing is genuinely disabled.
|
||||
trail_arm_r: Optional[float] = None
|
||||
trail_dist_r: Optional[float] = None
|
||||
# Post-decision stake multiplier from position_sizing (advisory
|
||||
# overlay). None = unsized — every gate/exit path ignores it, and
|
||||
# the backtester's R accounting is unaffected unless it opts in.
|
||||
stake_multiplier: Optional[float] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -108,6 +112,12 @@ class SignalReport:
|
||||
panel_payload_hash: str = ""
|
||||
panel_verdict_source: str = "" # "macro" / "chair" / ""
|
||||
|
||||
# Violent-move watch line (advisory/DISPLAY-ONLY). Stamped post-hoc
|
||||
# by surfaces via violent_watch.attach_watch — never populated
|
||||
# inside analyze_from_dataframes (backtest parity) and never read by
|
||||
# any gate, grade or setup generator.
|
||||
watch_line: str = ""
|
||||
|
||||
final_signal: SignalStrength = SignalStrength.NEUTRAL
|
||||
final_score: float = 0.0
|
||||
final_confidence: float = 0.0
|
||||
@@ -128,6 +138,13 @@ class SignalReport:
|
||||
market_regime: str = ""
|
||||
volatility_regime: str = ""
|
||||
exhaustion_count: int = 0
|
||||
# Record-keeping only (no gate reads these): the penalty tags that
|
||||
# fired in _compute_final_signal (tag -> RAW pre-decay amount), the
|
||||
# decayed total actually subtracted from confidence, and whether the
|
||||
# momentum-fading half of the exhaustion block was true.
|
||||
penalty_tags: Dict[str, float] = field(default_factory=dict)
|
||||
penalty_total: float = 0.0
|
||||
momentum_fading: bool = False
|
||||
|
||||
|
||||
class SignalAggregator:
|
||||
@@ -305,11 +322,13 @@ class SignalAggregator:
|
||||
logger.debug("signal log: news context unavailable",
|
||||
exc_info=True)
|
||||
extra = {"scan_kind": "blocked" if blocked else "scan"}
|
||||
try:
|
||||
extra["config_fingerprint"] = self.gate_config_fingerprint()
|
||||
except Exception:
|
||||
logger.debug("signal log: config fingerprint unavailable",
|
||||
exc_info=True)
|
||||
for key, fingerprint in (
|
||||
("config_fingerprint", self.gate_config_fingerprint),
|
||||
("engine_fingerprint", self.engine_config_fingerprint)):
|
||||
try:
|
||||
extra[key] = fingerprint()
|
||||
except Exception:
|
||||
logger.debug("signal log: %s unavailable", key, exc_info=True)
|
||||
signal_log.log_signal(
|
||||
report,
|
||||
context=context,
|
||||
@@ -348,6 +367,27 @@ class SignalAggregator:
|
||||
json.dumps(payload, sort_keys=True, default=str).encode("utf-8")
|
||||
).hexdigest()[:12]
|
||||
|
||||
def engine_config_fingerprint(self) -> str:
|
||||
"""Short hash of the SCORING configuration this process runs with.
|
||||
|
||||
gate_config_fingerprint covers the post-grade gates; this covers
|
||||
what produced the composite in the first place — indicator
|
||||
parameters, signal thresholds/geometry, the regime weight
|
||||
multipliers actually in force (override included) and the
|
||||
timeframe set. Logged on every row so indicator-weight research
|
||||
can tell which rows share a scoring configuration."""
|
||||
payload = {
|
||||
"technical": asdict(self.config.technical),
|
||||
"signals": asdict(self.config.signals),
|
||||
"regime_weights": regime_weight_table(self.config.technical),
|
||||
"primary": self.config.primary_timeframe.value,
|
||||
"confirm": [tf.value for tf in self.config.confirmation_timeframes],
|
||||
"display_name": self.config.display_name,
|
||||
}
|
||||
return hashlib.sha1(
|
||||
json.dumps(payload, sort_keys=True, default=str).encode("utf-8")
|
||||
).hexdigest()[:12]
|
||||
|
||||
def _apply_regime_filter_live(
|
||||
self, report: SignalReport, dfs: Dict[Timeframe, pd.DataFrame]
|
||||
) -> None:
|
||||
@@ -1552,6 +1592,12 @@ class SignalAggregator:
|
||||
if total_penalty >= MAX_PENALTY:
|
||||
break
|
||||
|
||||
# Record-keeping for position_sizing / signal-log forensics.
|
||||
# Purely additive — nothing below reads these fields.
|
||||
report.penalty_tags = {name: float(amount) for name, amount in penalties}
|
||||
report.penalty_total = float(total_penalty)
|
||||
report.momentum_fading = bool(momentum_fading)
|
||||
|
||||
confidence -= total_penalty
|
||||
confidence = max(0.0, min(95.0, confidence))
|
||||
|
||||
|
||||
+390
-1
@@ -56,6 +56,29 @@ or news_engine) so this module stays at the bottom of the import graph.
|
||||
(``scan`` / ``blocked`` for weekend/day early returns, which carry no
|
||||
news or candles) and ``config_fingerprint`` (hash of the gate config the
|
||||
producing process ran with — surfaces config drift between processes).
|
||||
|
||||
2026-09 weight-research additions:
|
||||
|
||||
* Row columns ``bar_ts`` (the primary bar LABEL this scan closed on,
|
||||
stored verbatim so it joins to indicators_json), ``engine_fingerprint``
|
||||
(hash of the SCORING config — indicator params, thresholds, regime
|
||||
weight multipliers, timeframes), ``code_version`` (short git SHA),
|
||||
``penalty_tags_json`` / ``penalty_total`` / ``momentum_fading`` /
|
||||
``exhaustion_count`` (the decay/exhaustion record), and the post-hoc
|
||||
``watch_level`` / ``watch_kinds`` / ``watch_line`` written by
|
||||
``update_watch`` from violent_watch.attach_watch.
|
||||
* Every per-indicator entry in ``indicators_json`` also carries
|
||||
``eff_weight`` (base weight x regime multiplier — the weight the
|
||||
composite actually used) and ``wscore`` (its contribution). Both are
|
||||
null for indicators the composite skipped. Divergence and candlestick
|
||||
entries exist only when they fire: absence is "no vote", not zero.
|
||||
* Table ``scan_outcomes`` — model-free forward returns per scan
|
||||
(1/4/24 h, $ and ATR-normalized, plus MFE/MAE, a 3xATR burst flag and
|
||||
a completeness flag), labelled from the stored candle streams by
|
||||
``label_scan_outcomes`` (called opportunistically at the end of log(),
|
||||
time-boxed; batch driver: scripts/label_scan_outcomes.py). Readers
|
||||
must dedup by the FIRST row per (timeframe, bar_ts) — several surfaces
|
||||
log the same bar.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
@@ -64,9 +87,10 @@ import logging
|
||||
import math
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
import app_config
|
||||
@@ -74,8 +98,13 @@ from advisory_outcome import (
|
||||
advance_outcome,
|
||||
canonical_r,
|
||||
initial_outcome,
|
||||
normalize_timeframe,
|
||||
timeframe_value,
|
||||
)
|
||||
# Private on purpose — the scan-outcome labeller must apply EXACTLY the
|
||||
# advisory model's bar validation and scheduled-closure rule (NY-anchored
|
||||
# weekend + the daily 17:00 NY maintenance hour), never a second copy.
|
||||
from advisory_outcome import _TF_SECONDS, _has_continuity, _valid_bars
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -128,6 +157,27 @@ CREATE TABLE IF NOT EXISTS news_snapshots (
|
||||
first_seen TEXT NOT NULL,
|
||||
last_seen TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS scan_outcomes (
|
||||
signal_id INTEGER PRIMARY KEY REFERENCES signals(id),
|
||||
timeframe TEXT,
|
||||
bar_ts TEXT,
|
||||
atr REAL,
|
||||
close0 REAL,
|
||||
fwd_1h REAL,
|
||||
fwd_4h REAL,
|
||||
fwd_24h REAL,
|
||||
fwd_1h_atr REAL,
|
||||
fwd_4h_atr REAL,
|
||||
fwd_24h_atr REAL,
|
||||
mfe_24h_atr REAL,
|
||||
mae_24h_atr REAL,
|
||||
burst_1h INTEGER,
|
||||
bars_used INTEGER,
|
||||
complete INTEGER,
|
||||
labeled_at TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_scan_outcomes_tf_bar
|
||||
ON scan_outcomes (timeframe, bar_ts);
|
||||
CREATE TABLE IF NOT EXISTS outlooks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
logged_at TEXT NOT NULL,
|
||||
@@ -170,6 +220,18 @@ _ADDITIVE_SIGNAL_COLS = (
|
||||
("scan_kind", "TEXT"),
|
||||
("config_fingerprint", "TEXT"),
|
||||
("panel_verdict_source", "TEXT"),
|
||||
# 2026-09 weight-research columns (see module docstring).
|
||||
("bar_ts", "TEXT"),
|
||||
("engine_fingerprint", "TEXT"),
|
||||
("code_version", "TEXT"),
|
||||
("penalty_tags_json", "TEXT"),
|
||||
("penalty_total", "REAL"),
|
||||
("momentum_fading", "INTEGER"),
|
||||
("exhaustion_count", "INTEGER"),
|
||||
# Violent-move watch, written post-hoc by update_watch.
|
||||
("watch_level", "TEXT"),
|
||||
("watch_kinds", "TEXT"),
|
||||
("watch_line", "TEXT"),
|
||||
)
|
||||
|
||||
# Read-side compatibility view `signals_with_news`: legacy rows keep
|
||||
@@ -203,6 +265,28 @@ _GATE_PREFIXES = (
|
||||
)
|
||||
_ACTIONABLE_GRADES = ("A+", "A", "B", "C")
|
||||
|
||||
# ── Scan outcomes (forward returns for indicator-weight research) ─────
|
||||
# Horizons are CALENDAR hours from the signal bar's close; a bar label is
|
||||
# its OPEN time, so the bar closing at label+interval is the one counted.
|
||||
SCAN_OUTCOME_HORIZONS_H = (1, 4, 24)
|
||||
SCAN_OUTCOME_WINDOW_H = 24
|
||||
# A 24 h window can only be judged once it has elapsed; the extra hour
|
||||
# leaves room for the last bar of the window to be stored.
|
||||
SCAN_LABEL_MIN_AGE_H = 25
|
||||
# Missing bars tolerated inside a window ON TOP of the scheduled closures
|
||||
# advisory_outcome._has_continuity already allows.
|
||||
SCAN_LABEL_MAX_GAP_BARS = 3
|
||||
# "Burst" bar: range >= this multiple of the scan's ATR (violent_watch).
|
||||
SCAN_BURST_ATR_MULTIPLE = 3.0
|
||||
# Soft wall-clock cap for ONE opportunistic (in-scan) labelling pass.
|
||||
SCAN_LABEL_TIME_BUDGET = 2.0
|
||||
# Minimum spacing between opportunistic passes in ONE process. Rows that
|
||||
# can never complete (a real hole in the stored stream) are re-examined
|
||||
# on every pass, and that read costs ~0.5 s on the live database — too
|
||||
# much to pay on the alert path for every logged scan, and pointless more
|
||||
# often than a candle close. Explicit calls (batch script, tests) ignore it.
|
||||
SCAN_LABEL_MIN_INTERVAL = 60.0
|
||||
|
||||
_ADDITIVE_OUTLOOK_COLS = (
|
||||
("news_json", "TEXT"),
|
||||
("indicators_json", "TEXT"),
|
||||
@@ -220,6 +304,8 @@ _SIGNAL_INSERT_COLS = (
|
||||
"setup_key", "headlines_hash", "events_hash", "gate_reason",
|
||||
"actionable", "cleaned_warnings_json", "bar_range_median", "scan_kind",
|
||||
"config_fingerprint",
|
||||
"bar_ts", "engine_fingerprint", "code_version", "penalty_tags_json",
|
||||
"penalty_total", "momentum_fading", "exhaustion_count",
|
||||
)
|
||||
_INSERT = (
|
||||
"INSERT INTO signals (" + ", ".join(_SIGNAL_INSERT_COLS) + ") VALUES ("
|
||||
@@ -263,6 +349,30 @@ def _blob_hash(text: str) -> str:
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
_code_version_cache: Optional[str] = None
|
||||
|
||||
|
||||
def _code_version() -> str:
|
||||
"""Short git SHA of the tree this process runs from (computed once).
|
||||
|
||||
Record-keeping only — "" whenever git is unavailable, slow (2 s cap)
|
||||
or the tree is not a checkout. There is no VERSION/__version__
|
||||
convention in this repo, so the commit is the version."""
|
||||
global _code_version_cache
|
||||
if _code_version_cache is None:
|
||||
_code_version_cache = ""
|
||||
try:
|
||||
done = subprocess.run(
|
||||
["git", "rev-parse", "--short", "HEAD"],
|
||||
cwd=os.path.dirname(os.path.abspath(__file__)),
|
||||
capture_output=True, text=True, timeout=2.0)
|
||||
if done.returncode == 0:
|
||||
_code_version_cache = done.stdout.strip()
|
||||
except Exception:
|
||||
logger.debug("signal_log: code version unavailable", exc_info=True)
|
||||
return _code_version_cache
|
||||
|
||||
|
||||
def setup_key(setup: Optional[Dict[str, Any]], timeframe: Any) -> Optional[str]:
|
||||
"""Identity of a logged trade setup for event dedup.
|
||||
|
||||
@@ -310,6 +420,118 @@ def gate_reason(
|
||||
return None
|
||||
|
||||
|
||||
_SCAN_OUTCOME_COLS = (
|
||||
"timeframe", "bar_ts", "atr", "close0",
|
||||
"fwd_1h", "fwd_4h", "fwd_24h",
|
||||
"fwd_1h_atr", "fwd_4h_atr", "fwd_24h_atr",
|
||||
"mfe_24h_atr", "mae_24h_atr", "burst_1h", "bars_used", "complete",
|
||||
)
|
||||
_SCAN_OUTCOME_INSERT = (
|
||||
"INSERT OR REPLACE INTO scan_outcomes (signal_id, "
|
||||
+ ", ".join(_SCAN_OUTCOME_COLS) + ", labeled_at) VALUES ("
|
||||
+ ", ".join("?" for _ in range(len(_SCAN_OUTCOME_COLS) + 2)) + ")"
|
||||
)
|
||||
|
||||
|
||||
def _utc(stamp: datetime) -> datetime:
|
||||
"""Tz-aware UTC (stored bar labels are UTC; a naive one is assumed)."""
|
||||
if stamp.tzinfo is None:
|
||||
return stamp.replace(tzinfo=timezone.utc)
|
||||
return stamp.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _window_continuous(bars: List[Dict[str, Any]], anchor: datetime,
|
||||
timeframe: Any,
|
||||
max_gap_bars: int = SCAN_LABEL_MAX_GAP_BARS) -> bool:
|
||||
"""True when ``bars`` walk forward from ``anchor`` without a real hole.
|
||||
|
||||
The scheduled-closure judgement is advisory_outcome._has_continuity's
|
||||
(NY-anchored weekend + the daily maintenance hour) applied one step at
|
||||
a time; on top of it a gap of at most ``max_gap_bars`` missing bars is
|
||||
tolerated, because a scan-outcome window is measured from closes, not
|
||||
replayed bar by bar like an advisory trade."""
|
||||
seconds = _TF_SECONDS.get(normalize_timeframe(timeframe))
|
||||
if not seconds:
|
||||
return False
|
||||
previous = anchor
|
||||
for bar in bars:
|
||||
stamp = bar["_timestamp"]
|
||||
if stamp <= previous:
|
||||
continue
|
||||
if not _has_continuity([bar], previous, timeframe):
|
||||
missing = int((stamp - previous).total_seconds() // seconds) - 1
|
||||
if missing > max_gap_bars:
|
||||
return False
|
||||
previous = stamp
|
||||
return True
|
||||
|
||||
|
||||
def scan_outcome(bar_ts: Any, timeframe: Any, bars: Iterable[Dict[str, Any]],
|
||||
atr: Optional[float], price: Optional[float]
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Forward-return label for ONE scan, from stored closed candles.
|
||||
|
||||
``bars`` is the union of stored bars for the row's primary timeframe.
|
||||
``close0`` is the close of the bar labelled ``bar_ts`` (the row's own
|
||||
last closed bar), falling back to the logged scan price. A horizon is
|
||||
reported only once it has ELAPSED inside the stream (an unfinished
|
||||
window reports NULLs rather than a truncated move), and ``complete``
|
||||
additionally requires an unbroken window. Pure: no DB, no clock."""
|
||||
seconds = _TF_SECONDS.get(normalize_timeframe(timeframe))
|
||||
anchor = None
|
||||
try:
|
||||
anchor = _utc(datetime.fromisoformat(str(bar_ts).replace(" ", "T")))
|
||||
except (TypeError, ValueError):
|
||||
anchor = None
|
||||
if not seconds or anchor is None:
|
||||
return None
|
||||
|
||||
valid = _valid_bars(bars)
|
||||
for bar in valid:
|
||||
bar["_timestamp"] = _utc(bar["_timestamp"])
|
||||
close0 = next((b["close"] for b in valid if b["_timestamp"] == anchor),
|
||||
None)
|
||||
if close0 is None:
|
||||
close0 = _finite(price)
|
||||
if close0 is None:
|
||||
return None
|
||||
atr = atr if (atr is not None and atr > 0) else None
|
||||
|
||||
forward = [b for b in valid if b["_timestamp"] > anchor]
|
||||
reach = forward[-1]["_timestamp"] if forward else anchor
|
||||
out: Dict[str, Any] = {
|
||||
"timeframe": str(getattr(timeframe, "value", timeframe)),
|
||||
"bar_ts": str(bar_ts), "atr": atr, "close0": close0,
|
||||
"mfe_24h_atr": None, "mae_24h_atr": None, "burst_1h": None,
|
||||
}
|
||||
window: List[Dict[str, Any]] = []
|
||||
for hours in SCAN_OUTCOME_HORIZONS_H:
|
||||
edge = anchor + timedelta(hours=hours)
|
||||
upto = [b for b in forward if b["_timestamp"] <= edge]
|
||||
elapsed = reach >= edge and bool(upto)
|
||||
move = (upto[-1]["close"] - close0) if elapsed else None
|
||||
out[f"fwd_{hours}h"] = move
|
||||
out[f"fwd_{hours}h_atr"] = (move / atr) if (move is not None and atr) \
|
||||
else None
|
||||
if hours == 1 and elapsed and atr:
|
||||
out["burst_1h"] = int(any(
|
||||
(b["high"] - b["low"]) >= SCAN_BURST_ATR_MULTIPLE * atr
|
||||
for b in upto))
|
||||
if hours == SCAN_OUTCOME_WINDOW_H:
|
||||
window = upto
|
||||
if elapsed and atr:
|
||||
out["mfe_24h_atr"] = (max(b["high"] for b in upto)
|
||||
- close0) / atr
|
||||
out["mae_24h_atr"] = (close0
|
||||
- min(b["low"] for b in upto)) / atr
|
||||
complete = (reach >= anchor + timedelta(hours=SCAN_OUTCOME_WINDOW_H)
|
||||
and bool(window)
|
||||
and _window_continuous(window, anchor, timeframe))
|
||||
out["bars_used"] = len(window)
|
||||
out["complete"] = int(complete)
|
||||
return out
|
||||
|
||||
|
||||
# ── Serializers (duck-typed; deliberately NOT imported from ──
|
||||
# ── dashboard_publisher — the dashboard's hard independence rule) ──
|
||||
|
||||
@@ -396,7 +618,12 @@ def _analysis_to_dict(analysis: Any) -> Optional[Dict[str, Any]]:
|
||||
"signal": getattr(signal, "name", str(signal)),
|
||||
"value": _finite(getattr(indicator, "value", None)),
|
||||
"confidence": _finite(getattr(indicator, "confidence", None)),
|
||||
# `weight` is the BASE weight; `eff_weight` is that weight
|
||||
# times the regime multiplier the composite actually used and
|
||||
# `wscore` its contribution (null for skipped indicators).
|
||||
"weight": _finite(getattr(indicator, "weight", None)),
|
||||
"eff_weight": _finite(getattr(indicator, "effective_weight", None)),
|
||||
"wscore": _finite(getattr(indicator, "weighted_score", None)),
|
||||
"description": getattr(indicator, "description", "") or "",
|
||||
}
|
||||
composite = getattr(analysis, "composite_signal", None)
|
||||
@@ -510,6 +737,11 @@ def _report_row(
|
||||
cleaned = list(getattr(report, "cleaned_warnings", None) or [])
|
||||
setup_dict = json.loads(setup_json) if setup_json else None
|
||||
bar_range = _finite(getattr(report, "spread_estimate", None))
|
||||
# The primary bar LABEL this scan closed on, stored verbatim (the
|
||||
# same string indicators_json carries, so readers can join on it).
|
||||
primary = getattr(report, "primary_analysis", None)
|
||||
bar_ts = str(getattr(primary, "timestamp", "") or "") or None
|
||||
penalty_tags = getattr(report, "penalty_tags", None) or {}
|
||||
return {
|
||||
"logged_at": datetime.now(timezone.utc).isoformat(),
|
||||
"source": source,
|
||||
@@ -554,6 +786,17 @@ def _report_row(
|
||||
"bar_range_median": bar_range,
|
||||
"scan_kind": str(extra.get("scan_kind") or "scan"),
|
||||
"config_fingerprint": extra.get("config_fingerprint"),
|
||||
# Weight-research record keeping (nothing live reads these back).
|
||||
"bar_ts": bar_ts,
|
||||
"engine_fingerprint": extra.get("engine_fingerprint"),
|
||||
"code_version": _code_version() or None,
|
||||
"penalty_tags_json": (_dumps({str(k): _finite(v)
|
||||
for k, v in penalty_tags.items()})
|
||||
if penalty_tags else None),
|
||||
"penalty_total": _finite(getattr(report, "penalty_total", None)),
|
||||
"momentum_fading": int(bool(getattr(report, "momentum_fading", False))),
|
||||
"exhaustion_count": int(
|
||||
_finite(getattr(report, "exhaustion_count", 0)) or 0),
|
||||
}
|
||||
|
||||
|
||||
@@ -573,6 +816,7 @@ class SignalLogger:
|
||||
# switch plus the additive migration need brief exclusive locks.
|
||||
self._conn.execute("PRAGMA busy_timeout=5000")
|
||||
self._lock = threading.Lock()
|
||||
self._next_scan_label = 0.0 # opportunistic labeller throttle
|
||||
with self._lock:
|
||||
self._with_lock_retry(
|
||||
lambda: self._conn.execute("PRAGMA journal_mode=WAL"))
|
||||
@@ -642,6 +886,11 @@ class SignalLogger:
|
||||
self._conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_signals_event"
|
||||
" ON signals (advisory_event_id)")
|
||||
# bar_ts is an additive column, so its index cannot live in
|
||||
# _SCHEMA (which runs before the ALTERs on a legacy database).
|
||||
self._conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_signals_tf_bar"
|
||||
" ON signals (timeframe, bar_ts)")
|
||||
self._conn.commit()
|
||||
|
||||
def _migrate_outlook_columns(self) -> None:
|
||||
@@ -703,6 +952,17 @@ class SignalLogger:
|
||||
except Exception:
|
||||
logger.warning("signal_log: advisory outcome resolver failed",
|
||||
exc_info=True)
|
||||
# Opportunistic, time-boxed and throttled: the same "advance what
|
||||
# the stored bars now allow" pass, for scan outcomes. Keeps the
|
||||
# research table current without a daemon or a cron entry.
|
||||
try:
|
||||
if time.monotonic() >= self._next_scan_label:
|
||||
self._next_scan_label = (time.monotonic()
|
||||
+ SCAN_LABEL_MIN_INTERVAL)
|
||||
self.label_scan_outcomes()
|
||||
except Exception:
|
||||
logger.warning("signal_log: scan-outcome labeller failed",
|
||||
exc_info=True)
|
||||
return rowid
|
||||
|
||||
# ── news interning ────────────────────────────────────────────
|
||||
@@ -822,6 +1082,26 @@ class SignalLogger:
|
||||
advisory_status, advisory_json, rowid))
|
||||
self._conn.commit()
|
||||
|
||||
def update_watch(self, rowid: int, level: str, kinds: Any,
|
||||
line: str) -> None:
|
||||
"""Fill the violent-move watch fields on an already-logged row.
|
||||
|
||||
Post-hoc writer mirroring update_panel: violent_watch.attach_watch
|
||||
runs on the surfaces AFTER analyze_pair inserted the row. Level and
|
||||
kinds are written even when the display line is switched off — the
|
||||
evaluation ("did a burst follow a 'hot' scan?") needs the
|
||||
assessment, not the text."""
|
||||
if isinstance(kinds, (list, tuple, set)):
|
||||
kinds_text = ",".join(str(k) for k in sorted(kinds))
|
||||
else:
|
||||
kinds_text = str(kinds or "")
|
||||
with self._lock:
|
||||
self._conn.execute(
|
||||
"UPDATE signals SET watch_level=?, watch_kinds=?,"
|
||||
" watch_line=? WHERE id=?",
|
||||
(str(level or ""), kinds_text, str(line or ""), int(rowid)))
|
||||
self._conn.commit()
|
||||
|
||||
def resolve_pending(
|
||||
self, symbol: str, candles_json: Optional[str], limit: int = 100
|
||||
) -> int:
|
||||
@@ -927,6 +1207,90 @@ class SignalLogger:
|
||||
continue
|
||||
return [bars[k] for k in sorted(bars)]
|
||||
|
||||
def label_scan_outcomes(self, budget: int = 150, now: Any = None,
|
||||
time_budget: Optional[float] = None) -> int:
|
||||
"""Label forward returns for scans whose 24 h window has closed.
|
||||
|
||||
Candidates are rows that carry indicator snapshots (a real scan)
|
||||
and a price, whose signal bar closed more than 25 h ago and which
|
||||
have no COMPLETE label yet; the least recently labelled go first,
|
||||
so a permanently short stream can never starve the rest. Returns
|
||||
the number of rows actually WRITTEN — a row whose numbers did not
|
||||
change is skipped, which makes repeated passes idempotent and lets
|
||||
a batch driver stop at 0.
|
||||
|
||||
Bounded on purpose: at most ``budget`` candidates are examined and
|
||||
the pass stops after ``time_budget`` seconds (default 2 s) so the
|
||||
opportunistic call inside log() stays cheap. Per-row failures are
|
||||
logged and skipped."""
|
||||
moment = now or datetime.now(timezone.utc)
|
||||
cutoff = _utc(moment) - timedelta(hours=SCAN_LABEL_MIN_AGE_H)
|
||||
deadline = time.monotonic() + float(
|
||||
SCAN_LABEL_TIME_BUDGET if time_budget is None else time_budget)
|
||||
written = 0
|
||||
with self._lock:
|
||||
rows = self._conn.execute(
|
||||
"SELECT s.id, s.symbol, s.timeframe, s.price,"
|
||||
" COALESCE(s.bar_ts, json_extract(s.indicators_json,"
|
||||
" '$.\"' || s.timeframe || '\".timestamp')) AS bar,"
|
||||
" json_extract(s.indicators_json,"
|
||||
" '$.\"' || s.timeframe || '\".key_levels.atr') AS atr"
|
||||
" FROM signals s"
|
||||
" LEFT JOIN scan_outcomes o ON o.signal_id = s.id"
|
||||
" WHERE s.indicators_json IS NOT NULL AND s.price > 0"
|
||||
" AND (o.signal_id IS NULL OR o.complete = 0)"
|
||||
" AND bar IS NOT NULL"
|
||||
" ORDER BY COALESCE(o.labeled_at, '') ASC, s.id ASC"
|
||||
" LIMIT ?", (int(budget),)).fetchall()
|
||||
# One bar stream per (symbol, timeframe) for the whole batch,
|
||||
# built from the OLDEST bar in it — _stream_since is a scan
|
||||
# over every row logged since that bar, far too expensive to
|
||||
# repeat per candidate.
|
||||
due: List[Tuple[Any, ...]] = []
|
||||
oldest: Dict[Tuple[str, str], Tuple[datetime, str]] = {}
|
||||
for rowid, symbol, tf, price, bar, atr in rows:
|
||||
try:
|
||||
stamp = _utc(datetime.fromisoformat(
|
||||
str(bar).replace(" ", "T")))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if stamp > cutoff:
|
||||
continue # window has not elapsed yet
|
||||
key = (str(symbol), str(tf))
|
||||
if key not in oldest or stamp < oldest[key][0]:
|
||||
oldest[key] = (stamp, str(bar))
|
||||
due.append((rowid, symbol, tf, price, bar, atr, key))
|
||||
streams: Dict[Tuple[str, str], List[Dict[str, Any]]] = {}
|
||||
for rowid, symbol, tf, price, bar, atr, key in due:
|
||||
if time.monotonic() > deadline:
|
||||
break
|
||||
try:
|
||||
if key not in streams:
|
||||
streams[key] = self._stream_since(
|
||||
symbol, tf, oldest[key][1])
|
||||
label = scan_outcome(bar, tf, streams[key],
|
||||
_finite(atr), price)
|
||||
if label is None:
|
||||
continue
|
||||
values = tuple(label[col] for col in _SCAN_OUTCOME_COLS)
|
||||
prior = self._conn.execute(
|
||||
"SELECT " + ", ".join(_SCAN_OUTCOME_COLS)
|
||||
+ " FROM scan_outcomes WHERE signal_id=?",
|
||||
(rowid,)).fetchone()
|
||||
if prior is not None and tuple(prior) == values:
|
||||
continue
|
||||
self._conn.execute(
|
||||
_SCAN_OUTCOME_INSERT,
|
||||
(int(rowid),) + values
|
||||
+ (datetime.now(timezone.utc).isoformat(),))
|
||||
self._conn.commit()
|
||||
written += 1
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"signal_log: scan-outcome labelling failed for row %s",
|
||||
rowid, exc_info=True)
|
||||
return written
|
||||
|
||||
def advance_with_bars(self, rowid: int, bars: Iterable[Dict[str, Any]],
|
||||
evidence: str = "external_bars") -> str:
|
||||
"""Advance ONE pending outcome from caller-supplied bars (backfill
|
||||
@@ -1222,6 +1586,31 @@ def log_panel_update(report: Any) -> None:
|
||||
exc_info=True)
|
||||
|
||||
|
||||
def log_watch_update(report: Any, watch: Any) -> None:
|
||||
"""Write the post-hoc violent-move watch fields onto this scan's row.
|
||||
|
||||
Mirrors log_panel_update: matches on the private rowid hand-off from
|
||||
log_signal, and a report that was never logged (backtest replay,
|
||||
logging disabled, another process) is a silent no-op. NEVER raises."""
|
||||
try:
|
||||
rowid = getattr(report, "_signal_log_rowid", None)
|
||||
if rowid is None or not app_config.signal_log_enabled():
|
||||
return
|
||||
path = app_config.signal_log_path()
|
||||
with _loggers_lock:
|
||||
lg = _loggers.get(path)
|
||||
if lg is None:
|
||||
return
|
||||
lg.update_watch(
|
||||
int(rowid),
|
||||
str(getattr(watch, "level", "") or ""),
|
||||
getattr(watch, "kinds", None) or [],
|
||||
getattr(report, "watch_line", "") or "")
|
||||
except Exception:
|
||||
logger.warning("signal_log: failed to persist violent-move watch",
|
||||
exc_info=True)
|
||||
|
||||
|
||||
def log_outlook(outlook: Any, surface: str = "live",
|
||||
payload: Any = None) -> None:
|
||||
"""Persist one advisory outlook when signal logging is enabled.
|
||||
|
||||
+7
-1
@@ -382,7 +382,13 @@ function listBlock(r) {
|
||||
const group = (title, arr, cls, icon) => arr?.length
|
||||
? `<div class="small text-secondary fw-semibold mt-2 mb-1">${title} (${arr.length})</div>
|
||||
<div class="small">${arr.map(t => `<div class="${cls}"><i class="bi ${icon} me-1"></i>${esc(t)}</div>`).join("")}</div>` : "";
|
||||
return group("Warnings", r.warnings, "text-warning", "bi-exclamation-triangle-fill")
|
||||
// Violent-move watch: display-only advisory line, stamped post-hoc by
|
||||
// the publisher. Empty unless the feature is enabled.
|
||||
const watch = r.watch_line
|
||||
? `<div class="small text-warning fw-semibold mt-2">${esc(r.watch_line)}</div>`
|
||||
: "";
|
||||
return watch
|
||||
+ group("Warnings", r.warnings, "text-warning", "bi-exclamation-triangle-fill")
|
||||
+ group("Bullish", r.bullish_reasons, "text-success", "bi-arrow-up-circle-fill")
|
||||
+ group("Bearish", r.bearish_reasons, "text-danger", "bi-arrow-down-circle-fill");
|
||||
}
|
||||
|
||||
+75
-2
@@ -50,7 +50,7 @@ import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, List, Optional, Set
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
|
||||
from telegram import InlineKeyboardButton, InlineKeyboardMarkup, Update
|
||||
from telegram.constants import ParseMode
|
||||
@@ -68,7 +68,7 @@ from config import (
|
||||
)
|
||||
from app_config import (
|
||||
get_app_config, telegram_token, telegram_chat_id,
|
||||
telegram_channel_id,
|
||||
telegram_channel_id, violent_watch_enabled,
|
||||
)
|
||||
from providers import BaseDataProvider, make_data_provider
|
||||
from agent_panel import attach_panel, panel_health
|
||||
@@ -77,6 +77,7 @@ from market_outlook import maybe_run_outlook
|
||||
from news_engine import FundamentalContext, NewsEngine
|
||||
from pair_signals import PAIR_PRODUCERS
|
||||
from signal_aggregator import SignalAggregator, SignalReport
|
||||
from violent_watch import ViolentWatch, attach_watch, should_send_headsup
|
||||
|
||||
_APP = get_app_config()
|
||||
|
||||
@@ -410,6 +411,10 @@ def format_signal_message(report: SignalReport) -> str:
|
||||
extra.append(f"📋 {ind_summary}")
|
||||
for w in report.warnings:
|
||||
extra.append(f"⚠️ {_escape_md(w)}")
|
||||
# Display-only violent-move watch (violent_watch.attach_watch stamps
|
||||
# it post-hoc; it is empty unless the feature is enabled).
|
||||
if report.watch_line:
|
||||
extra.append(_escape_md(report.watch_line))
|
||||
if report.news_summary:
|
||||
extra.append(f"📰 {_escape_md(report.news_summary)}")
|
||||
if report.panel_verdict:
|
||||
@@ -753,6 +758,33 @@ def _seconds_until_boundary(interval_minutes: int, buffer: int = 10) -> float:
|
||||
return remaining
|
||||
|
||||
|
||||
# ── Violent-move watch (display-only) ────────────────────────────────
|
||||
# Per-process heads-up budget: {"day": date, "count": int, "sent": {...}}.
|
||||
_watch_state: Dict[str, Any] = {}
|
||||
|
||||
|
||||
def _format_headsup(report: SignalReport, w: "ViolentWatch") -> str:
|
||||
"""Standalone owner heads-up. Advisory only — it never asks anyone
|
||||
to touch an open trade: every protective action tested lost money
|
||||
(sizing_campaign_2026-09.md), so the stop stays the protection."""
|
||||
lines = [
|
||||
f"🌪 *Heads-up — {_escape_md(report.display_name)}*: "
|
||||
f"violent-move conditions for the next hour.",
|
||||
_escape_md(" · ".join(w.triggers)),
|
||||
]
|
||||
if w.likely_direction == "down":
|
||||
lines.append("If a big move comes, it has usually gone DOWN from "
|
||||
"readings like these.")
|
||||
elif w.likely_direction == "up":
|
||||
lines.append("If a big move comes, it has usually gone UP from "
|
||||
"readings like these.")
|
||||
lines.append(
|
||||
"If you have a trade open, expect big bars. Your stop is the "
|
||||
"protection — past tests show closing or tightening early loses "
|
||||
"money.")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
async def _scan_timeframe(
|
||||
bot,
|
||||
tf: Timeframe,
|
||||
@@ -779,6 +811,38 @@ async def _scan_timeframe(
|
||||
|
||||
_update_level_cache(reports)
|
||||
|
||||
# Violent-move watch: DISPLAY-ONLY. Runs before the alert loop so
|
||||
# the stamped line is present when format_signal_message renders,
|
||||
# and can never break an alert (whole block is best-effort).
|
||||
try:
|
||||
vw_settings = get_app_config().violent_watch_settings
|
||||
if violent_watch_enabled():
|
||||
now = datetime.now(timezone.utc)
|
||||
for report in reports:
|
||||
w = attach_watch(report, tf, vw_settings, now)
|
||||
if not (tf == _HEADSUP_TF
|
||||
and vw_settings.get("headsup", True)
|
||||
and OWNER_ID is not None):
|
||||
continue
|
||||
min_level = vw_settings.get("headsup_min_level", "hot")
|
||||
if w.level == "hot" or (min_level == "watch"
|
||||
and w.level == "watch"):
|
||||
ok, why = should_send_headsup(
|
||||
_watch_state, now, w.kinds,
|
||||
int(vw_settings.get("max_headsup_per_day", 12)))
|
||||
if ok:
|
||||
await _send_alert(bot, OWNER_ID,
|
||||
_format_headsup(report, w))
|
||||
else:
|
||||
logger.info(
|
||||
f"violent watch: heads-up skipped ({why})")
|
||||
else:
|
||||
logger.debug(
|
||||
f"violent watch: {report.symbol} {tf.value} "
|
||||
f"level={w.level} (min {min_level}) — no heads-up")
|
||||
except Exception:
|
||||
logger.exception("violent watch failed")
|
||||
|
||||
candidates = [
|
||||
r for r in reports
|
||||
if r.final_signal != SignalStrength.NEUTRAL # same gate as breach path
|
||||
@@ -1100,6 +1164,15 @@ _AUTOSCAN_LOOPS = [
|
||||
(tf, _TF_INTERVAL_MINUTES[tf]) for tf in _APP.scan_timeframes
|
||||
]
|
||||
|
||||
# The violent-move heads-up is evaluated on the FASTEST scanned loop
|
||||
# only — it is a "next hour" warning, so once per fastest candle is the
|
||||
# most frequent it can be, and every slower loop would just repeat it.
|
||||
_HEADSUP_TF: Optional[Timeframe] = min(
|
||||
_APP.scan_timeframes,
|
||||
key=lambda tf: _TF_INTERVAL_MINUTES.get(tf, 10 ** 6),
|
||||
default=None,
|
||||
)
|
||||
|
||||
|
||||
_ws_task: Optional[asyncio.Task] = None
|
||||
|
||||
|
||||
+1262
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,426 @@
|
||||
"""
|
||||
ForexBot2 — violent-move watch (DISPLAY-ONLY heads-up)
|
||||
=======================================================
|
||||
Flags the conditions under which a "violent bar" (range >= 3x the prior
|
||||
ATR14) is unusually likely, so a human reading an alert knows the next
|
||||
hour may be wild. It is a WARNING LINE and a standalone heads-up
|
||||
message — nothing here ever changes a signal, grade, gate, trade setup
|
||||
or backtest result.
|
||||
|
||||
Hard independence rule (mirrors market_outlook.py / agent_panel.py):
|
||||
this module is NEVER imported by `signal_aggregator`, `engine`, or
|
||||
anything under `backtest/`. Surfaces (telegram_bot, main, the dashboard
|
||||
publisher) attach the result POST-HOC via `attach_watch`, exactly the
|
||||
way panels are attached.
|
||||
|
||||
Evidence — 3-year study, backtest/research/sizing_campaign_2026-09.md
|
||||
(addenda):
|
||||
* A violent bar is a bar whose range >= 3x the prior ATR14.
|
||||
* Hot clock slots run about 3-4x the normal rate: bars closing
|
||||
12:00-14:59 UTC on weekdays, the 01:xx UTC hour on any weekday, and
|
||||
the Sunday reopen (22:xx-23:xx UTC).
|
||||
* The 30 minutes after a high-impact US release run about 10x normal.
|
||||
Share of releases followed by a violent bar within the hour: NFP
|
||||
97%, CPI 91%, FOMC 88%, retail sales 64%, PPI 54%, PCE 50%,
|
||||
jobless claims 44%.
|
||||
* Indicator readings taken BEFORE the bar each lift the odds 1.3-3x:
|
||||
price outside the Bollinger bands, RSI < 30 or >= 70, close outside
|
||||
the Keltner channel, a rising ADX (slope > 3), volatility regime
|
||||
"extreme", market regime "transitioning", and MACD histogram
|
||||
negative and accelerating down (the engine emits STRONG_SELL for
|
||||
exactly that case).
|
||||
* Only the three PRICE-EXTREME readings (Bollinger, RSI, Keltner)
|
||||
earn a level. A bar-by-bar sweep of the 3-year window showed the
|
||||
first draft — which let any two readings raise the level — flagged
|
||||
27% of all bars "hot" at merely ~2x odds (~13 heads-ups/day). The
|
||||
shipped rule (calendar, or a clock slot plus one price extreme)
|
||||
flags 8.6% of bars: next-bar odds 3-5x normal, ~20% chance within
|
||||
the hour, about 4 hot hours a day. The remaining readings (ADX
|
||||
slope, volatility regime, regime transition, MACD acceleration) are
|
||||
CONTEXT: they are printed once a level exists, and never create one.
|
||||
* Direction: bearish readings preceded a DOWN burst 70-78% of the
|
||||
time on BOTH M5 and M15. Bullish readings preceded an UP burst only
|
||||
~60% of the time and only on M5 — so an "up" call is made on M5
|
||||
alone.
|
||||
* Open trades are NOT to be touched. Every protective action tested
|
||||
(early close, tightened stop, partial exit) lost money; the stop
|
||||
already IS the protection. The heads-up says so explicitly.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from config import SignalStrength, Timeframe
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Calendar window either side of a high-impact US release (minutes).
|
||||
RELEASE_WINDOW_MINUTES = 30
|
||||
# Hot clock slots (UTC), from the 3-year bar study.
|
||||
US_DATA_HOURS = (12, 13, 14)
|
||||
ASIAN_OPEN_HOUR = 1
|
||||
SUNDAY_REOPEN_HOURS = (22, 23)
|
||||
# ADX slope above this counts as "trend strength rising".
|
||||
ADX_SLOPE_HOT = 3.0
|
||||
|
||||
_LEVEL_NONE = "none"
|
||||
_LEVEL_WATCH = "watch"
|
||||
_LEVEL_HOT = "hot"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ViolentWatch:
|
||||
"""One scan's violent-move assessment. Advisory/display-only."""
|
||||
level: str = _LEVEL_NONE # "none" | "watch" | "hot"
|
||||
triggers: List[str] = field(default_factory=list) # plain language
|
||||
kinds: List[str] = field(default_factory=list) # calendar/clock/extreme
|
||||
readings: int = 0 # PRICE-EXTREME readings (0..3)
|
||||
likely_direction: str = "" # "down" | "up" | ""
|
||||
|
||||
def line(self) -> str:
|
||||
"""One-line alert text; "" when nothing fired."""
|
||||
if self.level == _LEVEL_NONE or not self.triggers:
|
||||
return ""
|
||||
if self.level == _LEVEL_HOT:
|
||||
head = "🌪 Violent-move watch (HOT): "
|
||||
odds = " — odds of a violent bar about 3–5× normal"
|
||||
else:
|
||||
head = "🌪 Violent-move watch (watch): "
|
||||
odds = " — about 2× normal"
|
||||
tail = ""
|
||||
if self.likely_direction == "down":
|
||||
tail = " — if one comes it has usually gone DOWN"
|
||||
elif self.likely_direction == "up":
|
||||
tail = " — if one comes it has usually gone UP"
|
||||
return head + " · ".join(self.triggers) + odds + tail
|
||||
|
||||
|
||||
# ── Calendar ─────────────────────────────────────────────────────────
|
||||
|
||||
def _is_high_impact_usd(event: Any) -> bool:
|
||||
"""USD + high impact, tolerating either the `impact` string or the
|
||||
`is_high_impact` property (EconomicEvent exposes both)."""
|
||||
try:
|
||||
if str(getattr(event, "currency", "")).upper() != "USD":
|
||||
return False
|
||||
impact = getattr(event, "impact", None)
|
||||
if isinstance(impact, str):
|
||||
return impact.strip().lower() == "high"
|
||||
return bool(getattr(event, "is_high_impact", False))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _calendar_trigger(events: Optional[List[Any]],
|
||||
now: datetime) -> Optional[str]:
|
||||
"""Soonest high-impact USD release inside +/-30 min of `now`."""
|
||||
if not events:
|
||||
return None
|
||||
window = timedelta(minutes=RELEASE_WINDOW_MINUTES)
|
||||
best: Optional[Tuple[float, str]] = None
|
||||
for ev in events or []:
|
||||
if not _is_high_impact_usd(ev):
|
||||
continue
|
||||
when = getattr(ev, "time", None)
|
||||
if not isinstance(when, datetime) or when.tzinfo is None:
|
||||
continue
|
||||
try:
|
||||
delta = when - now
|
||||
except Exception:
|
||||
continue
|
||||
name = str(getattr(ev, "event", "") or "US release").strip()
|
||||
if timedelta(0) < delta <= window:
|
||||
minutes = max(1, int(round(delta.total_seconds() / 60)))
|
||||
text = f"US high-impact release in {minutes} min ({name})"
|
||||
rank = abs(delta.total_seconds())
|
||||
elif -window <= delta < timedelta(0):
|
||||
minutes = max(1, int(round(-delta.total_seconds() / 60)))
|
||||
text = f"US release just out ({name}, {minutes} min ago)"
|
||||
rank = abs(delta.total_seconds())
|
||||
else:
|
||||
continue
|
||||
if best is None or rank < best[0]:
|
||||
best = (rank, text)
|
||||
return None if best is None else best[1]
|
||||
|
||||
|
||||
# ── Clock ────────────────────────────────────────────────────────────
|
||||
|
||||
def _clock_trigger(now: datetime) -> Optional[str]:
|
||||
"""Hot clock slot for `now` (UTC), or None."""
|
||||
try:
|
||||
moment = now.astimezone(timezone.utc) if now.tzinfo else now
|
||||
except Exception:
|
||||
return None
|
||||
weekday, hour = moment.weekday(), moment.hour # Mon=0 .. Sun=6
|
||||
if weekday <= 4 and hour in US_DATA_HOURS:
|
||||
return "US data hours (12:00–15:00 UTC)"
|
||||
if weekday <= 4 and hour == ASIAN_OPEN_HOUR:
|
||||
return "Asian open hour (01:00 UTC)"
|
||||
if weekday == 6 and hour in SUNDAY_REOPEN_HOURS:
|
||||
return "Sunday reopen"
|
||||
return None
|
||||
|
||||
|
||||
# ── Indicator readings ───────────────────────────────────────────────
|
||||
|
||||
def _indicator_value(analysis: Any, name: str) -> Optional[float]:
|
||||
try:
|
||||
ind = (analysis.indicators or {}).get(name)
|
||||
if ind is None:
|
||||
return None
|
||||
value = float(ind.value)
|
||||
except Exception:
|
||||
return None
|
||||
if value != value: # NaN
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def _indicator_signal(analysis: Any, name: str) -> Optional[SignalStrength]:
|
||||
try:
|
||||
ind = (analysis.indicators or {}).get(name)
|
||||
return None if ind is None else ind.signal
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _keltner_side(analysis: Any) -> Optional[str]:
|
||||
""""above" / "below" / None — is the close outside the channel?
|
||||
|
||||
The engine's keltner indicator carries `value = close - kc_mid` and
|
||||
stashes kc_upper/kc_lower/kc_mid in `key_levels` (engine.py
|
||||
_compute_keltner). Comparing the value against the band half-widths
|
||||
is therefore exact AND survives the squeeze branches, which
|
||||
overwrite the indicator's signal/description but still publish the
|
||||
same levels. The description text is only a fallback for analyses
|
||||
that were built without key_levels.
|
||||
"""
|
||||
value = _indicator_value(analysis, "keltner")
|
||||
levels = getattr(analysis, "key_levels", None) or {}
|
||||
try:
|
||||
upper = levels.get("kc_upper")
|
||||
lower = levels.get("kc_lower")
|
||||
mid = levels.get("kc_mid")
|
||||
if value is not None and None not in (upper, lower, mid):
|
||||
if value > float(upper) - float(mid):
|
||||
return "above"
|
||||
if value < float(lower) - float(mid):
|
||||
return "below"
|
||||
return None
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
desc = str((analysis.indicators or {})["keltner"].description).lower()
|
||||
except Exception:
|
||||
return None
|
||||
if "above keltner upper" in desc:
|
||||
return "above"
|
||||
if "below keltner lower" in desc:
|
||||
return "below"
|
||||
return None
|
||||
|
||||
|
||||
def _collect_readings(analysis: Any) -> Tuple[List[str], List[str], int, int]:
|
||||
"""(price_extreme_phrases, context_phrases, bearish, bullish).
|
||||
|
||||
Only the PRICE-EXTREME group earns a level (see the module
|
||||
docstring); the context group is printed alongside once a level
|
||||
exists but can never create one. Both groups feed direction.
|
||||
"""
|
||||
extreme: List[str] = []
|
||||
context: List[str] = []
|
||||
bear = bull = 0
|
||||
if analysis is None:
|
||||
return extreme, context, bear, bull
|
||||
|
||||
# ── Price extremes (level-bearing) ──
|
||||
bb = _indicator_value(analysis, "bollinger")
|
||||
if bb is not None:
|
||||
if bb < 0:
|
||||
extreme.append("price below the Bollinger bands")
|
||||
bear += 1
|
||||
elif bb > 1:
|
||||
extreme.append("price above the Bollinger bands")
|
||||
bull += 1
|
||||
|
||||
rsi = _indicator_value(analysis, "rsi")
|
||||
if rsi is not None:
|
||||
if rsi < 30:
|
||||
extreme.append(f"RSI {rsi:.0f}, oversold extreme")
|
||||
bear += 1
|
||||
elif rsi >= 70:
|
||||
extreme.append(f"RSI {rsi:.0f}, overbought extreme")
|
||||
bull += 1
|
||||
|
||||
kc_side = _keltner_side(analysis)
|
||||
if kc_side is not None:
|
||||
extreme.append("price outside the Keltner channel")
|
||||
if kc_side == "above":
|
||||
bull += 1
|
||||
else:
|
||||
bear += 1
|
||||
|
||||
# ── Context (never raises the level) ──
|
||||
slope = getattr(analysis, "_adx_slope", None)
|
||||
try:
|
||||
if slope is not None and float(slope) > ADX_SLOPE_HOT:
|
||||
context.append("trend strength rising")
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
if str(getattr(analysis, "volatility_regime", "")) == "extreme":
|
||||
context.append("volatility extreme")
|
||||
|
||||
regime = str(getattr(analysis, "market_regime", ""))
|
||||
if regime == "transitioning":
|
||||
context.append("regime transitioning")
|
||||
|
||||
macd = _indicator_signal(analysis, "macd")
|
||||
if macd == SignalStrength.STRONG_SELL:
|
||||
context.append("downside momentum accelerating")
|
||||
bear += 1
|
||||
elif macd == SignalStrength.STRONG_BUY:
|
||||
context.append("upside momentum accelerating")
|
||||
bull += 1
|
||||
|
||||
# Regime contributes direction only — it has no phrase of its own
|
||||
# beyond "regime transitioning" above.
|
||||
if regime == "trending_down":
|
||||
bear += 1
|
||||
elif regime == "trending_up":
|
||||
bull += 1
|
||||
|
||||
return extreme, context, bear, bull
|
||||
|
||||
|
||||
# ── Assessment ───────────────────────────────────────────────────────
|
||||
|
||||
def assess(report: Any,
|
||||
now: datetime,
|
||||
events: Optional[List[Any]] = None,
|
||||
primary_tf: Optional[Timeframe] = None) -> ViolentWatch:
|
||||
"""Score violent-move conditions. Every input may be missing or
|
||||
malformed — the answer then degrades to "no trigger". Never raises."""
|
||||
triggers: List[str] = []
|
||||
kinds: List[str] = []
|
||||
|
||||
calendar = _calendar_trigger(events, now)
|
||||
if calendar:
|
||||
triggers.append(calendar)
|
||||
kinds.append("calendar")
|
||||
|
||||
clock = _clock_trigger(now)
|
||||
if clock:
|
||||
triggers.append(clock)
|
||||
kinds.append("clock")
|
||||
|
||||
analysis = getattr(report, "primary_analysis", None)
|
||||
extreme_phrases, context_phrases, bear, bull = _collect_readings(analysis)
|
||||
triggers.extend(extreme_phrases)
|
||||
readings = len(extreme_phrases)
|
||||
if readings >= 1:
|
||||
kinds.append("extreme")
|
||||
|
||||
# Level rule (bar-by-bar validated, see module docstring): only
|
||||
# price extremes and the clock/calendar create a level.
|
||||
if calendar or (clock and readings >= 1):
|
||||
level = _LEVEL_HOT
|
||||
elif clock or readings >= 2:
|
||||
level = _LEVEL_WATCH
|
||||
else:
|
||||
level = _LEVEL_NONE
|
||||
|
||||
# Context readings are colour, printed only once a level exists.
|
||||
if level != _LEVEL_NONE:
|
||||
triggers.extend(context_phrases)
|
||||
|
||||
direction = ""
|
||||
if bear >= 2 and bear > bull:
|
||||
direction = "down"
|
||||
elif bull >= 2 and bull > bear:
|
||||
# Bullish readings only earned a call on M5 in the study.
|
||||
tf_value = getattr(primary_tf, "value", primary_tf)
|
||||
if tf_value == Timeframe.M5.value:
|
||||
direction = "up"
|
||||
|
||||
return ViolentWatch(level=level, triggers=triggers, kinds=kinds,
|
||||
readings=readings, likely_direction=direction)
|
||||
|
||||
|
||||
def _log_watch(report: Any, watch: "ViolentWatch") -> None:
|
||||
"""Persist the assessment onto the scan's signals.db row (fail-safe).
|
||||
|
||||
The row is inserted inside analyze_pair — BEFORE a surface attaches
|
||||
the watch — so these columns arrive by post-hoc update, exactly like
|
||||
the panel verdict. Level and kinds are written even when the display
|
||||
line is off: evaluating the watch needs the assessment, not the text.
|
||||
"""
|
||||
try:
|
||||
from signal_log import log_watch_update
|
||||
log_watch_update(report, watch)
|
||||
except Exception:
|
||||
logger.debug("violent watch: logging failed", exc_info=True)
|
||||
|
||||
|
||||
def attach_watch(report: Any,
|
||||
primary_tf: Optional[Timeframe],
|
||||
settings: Dict[str, Any],
|
||||
now: Optional[datetime] = None) -> ViolentWatch:
|
||||
"""Stamp `report.watch_line` with the display line (advisory only).
|
||||
|
||||
Contract: NEVER raises. A broken watch must not cost an alert send
|
||||
or a dashboard publish — a failure logs and returns level "none".
|
||||
"""
|
||||
try:
|
||||
settings = settings or {}
|
||||
if not settings.get("enabled", False):
|
||||
return ViolentWatch()
|
||||
moment = now or datetime.now(timezone.utc)
|
||||
events: Optional[List[Any]] = None
|
||||
try:
|
||||
# Lazy import keeps this module import-light and free of any
|
||||
# signal-path dependency. The calendar rides NewsEngine's
|
||||
# class-level cache, so a warm cache costs no network I/O.
|
||||
from config import PlatformConfig
|
||||
from news_engine import NewsEngine
|
||||
events = NewsEngine(PlatformConfig().news).get_week_events(["USD"])
|
||||
except Exception as exc:
|
||||
logger.debug("violent watch: calendar unavailable (%s)", exc)
|
||||
watch = assess(report, moment, events, primary_tf)
|
||||
if settings.get("alert_line", True):
|
||||
report.watch_line = watch.line()
|
||||
_log_watch(report, watch)
|
||||
return watch
|
||||
except Exception:
|
||||
logger.exception("violent watch: assessment failed")
|
||||
return ViolentWatch()
|
||||
|
||||
|
||||
def should_send_headsup(state: Dict[str, Any],
|
||||
now: datetime,
|
||||
kinds: List[str],
|
||||
max_per_day: int) -> Tuple[bool, str]:
|
||||
"""Rate-limit the standalone heads-up message.
|
||||
|
||||
`state` is a mutable dict owned by the caller:
|
||||
{"day": date, "count": int, "sent": {(hour_iso, kinds): True}}
|
||||
One message per UTC hour per trigger combination, capped per UTC day.
|
||||
"""
|
||||
day = now.date()
|
||||
if state.get("day") != day:
|
||||
state["day"] = day
|
||||
state["count"] = 0
|
||||
state["sent"] = {}
|
||||
sent = state.setdefault("sent", {})
|
||||
if int(state.get("count", 0)) >= int(max_per_day):
|
||||
return False, "daily cap"
|
||||
hour_bucket = now.replace(minute=0, second=0, microsecond=0).isoformat()
|
||||
key = (hour_bucket, tuple(sorted(kinds)))
|
||||
if key in sent:
|
||||
return False, "already sent this hour for these triggers"
|
||||
sent[key] = True
|
||||
state["count"] = int(state.get("count", 0)) + 1
|
||||
return True, "ok"
|
||||
Reference in New Issue
Block a user