Add session filter + auto-toggle, extend trade journal diagnostics

Session filter restricts trades to a UTC hour window (default 07-16,
London + NY) — opt-in via --session-filter, with --session-start /
--session-end overrides. Wired into both the backtester (cheap hour
check next to the regime gate) and live mode (downgrades signal to
NEUTRAL/F when current UTC hour is outside the window).

Auto-toggle (--auto-session-filter) tracks rolling mean pnl_pct over
the last N closed trades; when it falls below threshold, the session
filter activates, releasing once expectancy recovers. Default lookback
10 trades, threshold -0.3%.

Trade dataclass extended with diagnostic fields captured at entry:
entry_hour, entry_dow, ADX, ATR%, market_regime, volatility_regime,
mtf_agreement. These let --journal-csv exports surface loss-pattern
clusters when a window underperforms.

Validated on 2024 trend / 2023 chop / 2026 recent windows. Auto-toggle
preserves ~50% of trend edge while still cutting ~25% of recent-regime
losses — pure regime filter is still best on cumulative, so both
session flags ship default-off.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

backtest.py

@@ -14,8 +14,9 @@ import itertools
 import logging
 import multiprocessing as mp
 import os
+from collections import deque
 from dataclasses import replace
-from typing import Dict, List, Optional, Tuple
+from typing import Deque, Dict, List, Optional, Tuple
 
 import pandas as pd
 
@@ -123,6 +124,13 @@ class WalkForwardBacktester:
         ]
         trades: List[Trade] = []
 
+        # Auto-session feedback: rolling pnl_pct over last K closed trades.
+        # When mean falls below threshold, the session filter activates;
+        # once expectancy recovers, it releases.
+        self._recent_pnl_pcts: Deque[float] = deque(
+            maxlen=max(1, bt.auto_session_lookback_trades)
+        )
+
         open_position: Optional[Dict] = None
 
         # Optional progress bar
@@ -148,6 +156,7 @@ class WalkForwardBacktester:
                     trades.append(closed)
                     balance += closed.pnl_currency
                     equity_points.append((next_bar.name, balance))
+                    self._recent_pnl_pcts.append(float(closed.pnl_pct))
                     open_position = None
 
             # ── Step 2: maybe open a new position ──────────────────
@@ -166,6 +175,7 @@ class WalkForwardBacktester:
                     and report.trade_setup is not None
                     and t + 1 < len(primary_df)
                     and self._regime_passes(decision_time)
+                    and self._session_passes(decision_time)
                 ):
                     next_bar = primary_df.iloc[t + 1]
                     open_position = self._open_trade(report, next_bar, balance)
@@ -244,6 +254,25 @@ class WalkForwardBacktester:
             return False
         return float(last) >= bt.regime_filter_min_adx
 
+    def _session_passes(self, decision_time: pd.Timestamp) -> bool:
+        """UTC hour gate — decision_time = entry bar's open time.
+
+        Filter is active when EITHER manual `session_filter` is on, OR
+        `auto_session_filter` is on and the rolling mean pnl_pct over the
+        last N closed trades is below threshold.
+        """
+        bt = self.config.backtest
+        active = bt.session_filter
+        if not active and bt.auto_session_filter:
+            recent = getattr(self, "_recent_pnl_pcts", None)
+            if recent is not None and len(recent) >= bt.auto_session_lookback_trades:
+                mean_pnl = sum(recent) / len(recent)
+                active = mean_pnl < bt.auto_session_threshold_pct
+        if not active:
+            return True
+        h = int(decision_time.hour)
+        return bt.session_start_hour <= h < bt.session_end_hour
+
     @staticmethod
     def _compute_adx_series(df: pd.DataFrame, period: int = 14) -> pd.Series:
         """Wilder's ADX series — same algorithm as engine._compute_adx."""
@@ -325,6 +354,20 @@ class WalkForwardBacktester:
         risk_pips = max(ts.risk_pips, 1e-9)
         pip_currency_value = risk_amount / risk_pips  # $ per pip
 
+        # ── Diagnostic snapshot at entry ─────────────────────────
+        analysis = report.primary_analysis
+        adx_val = 0.0
+        atr_pct = 0.0
+        if analysis is not None:
+            adx_sig = analysis.indicators.get("adx")
+            if adx_sig is not None:
+                adx_val = float(adx_sig.value)
+            atr_pct = float(analysis.key_levels.get("atr_pct", 0.0))
+
+        entry_ts = next_bar.name
+        entry_hour = int(entry_ts.hour) if hasattr(entry_ts, "hour") else -1
+        entry_dow = int(entry_ts.dayofweek) if hasattr(entry_ts, "dayofweek") else -1
+
         return {
             "direction": direction,
             "entry_time": next_bar.name,
@@ -338,6 +381,14 @@ class WalkForwardBacktester:
             "score_at_entry": float(report.final_score),
             "confidence_at_entry": float(report.final_confidence),
             "bar_index_open": next_bar.name,
+            # Diagnostics
+            "entry_hour": entry_hour,
+            "entry_dow": entry_dow,
+            "adx_at_entry": adx_val,
+            "atr_pct_at_entry": atr_pct,
+            "market_regime": report.market_regime or "",
+            "volatility_regime": report.volatility_regime or "",
+            "mtf_agreement": bool(report.mtf_agreement),
         }
 
     def _check_exit(
@@ -419,6 +470,13 @@ class WalkForwardBacktester:
             grade=position["grade"],
             score_at_entry=position["score_at_entry"],
             confidence_at_entry=position["confidence_at_entry"],
+            entry_hour=position.get("entry_hour", -1),
+            entry_dow=position.get("entry_dow", -1),
+            adx_at_entry=position.get("adx_at_entry", 0.0),
+            atr_pct_at_entry=position.get("atr_pct_at_entry", 0.0),
+            market_regime=position.get("market_regime", ""),
+            volatility_regime=position.get("volatility_regime", ""),
+            mtf_agreement=position.get("mtf_agreement", False),
         )
 
 
@@ -444,6 +502,18 @@ def _apply_overrides(config: PlatformConfig, overrides: Dict[str, float]) -> Pla
         new_config.backtest.regime_filter = bool(overrides["regime_filter"])
     if "regime_filter_min_adx" in overrides:
         new_config.backtest.regime_filter_min_adx = float(overrides["regime_filter_min_adx"])
+    if "session_filter" in overrides:
+        new_config.backtest.session_filter = bool(overrides["session_filter"])
+    if "session_start_hour" in overrides:
+        new_config.backtest.session_start_hour = int(overrides["session_start_hour"])
+    if "session_end_hour" in overrides:
+        new_config.backtest.session_end_hour = int(overrides["session_end_hour"])
+    if "auto_session_filter" in overrides:
+        new_config.backtest.auto_session_filter = bool(overrides["auto_session_filter"])
+    if "auto_session_lookback_trades" in overrides:
+        new_config.backtest.auto_session_lookback_trades = int(overrides["auto_session_lookback_trades"])
+    if "auto_session_threshold_pct" in overrides:
+        new_config.backtest.auto_session_threshold_pct = float(overrides["auto_session_threshold_pct"])
     if "risk_per_trade_pct" in overrides:
         new_config.backtest.risk_per_trade_pct = float(overrides["risk_per_trade_pct"])
     if "disable_indicator" in overrides:

config.py

@@ -170,6 +170,22 @@ class BacktestConfig:
     regime_filter_timeframe: Timeframe = Timeframe.H4
     regime_filter_min_adx: float = 25.0
 
+    # Session gate — only trade during a specified UTC hour window.
+    # 2026 diagnostic backtest showed 07-15 UTC (London + NY) had
+    # 47% WR / +127 pips while off-hours were 0% WR / -571 pips.
+    session_filter: bool = False
+    session_start_hour: int = 7   # UTC, inclusive
+    session_end_hour: int = 16    # UTC, exclusive — so trades fire on 07..15
+
+    # Auto-toggle: enable session filter ONLY when recent realised trades
+    # show adverse expectancy (mean pnl_pct over the last N closed trades
+    # below threshold). Releases automatically once expectancy recovers.
+    # Designed so trending regimes keep all hours, while adverse/chop
+    # regimes auto-restrict to liquid sessions.
+    auto_session_filter: bool = False
+    auto_session_lookback_trades: int = 10
+    auto_session_threshold_pct: float = -0.3
+
     # Parallelism (used by Optimizer; single-run backtest is sequential)
     parallel_workers: int = field(default_factory=lambda: os.cpu_count() or 4)
 

data_provider.py

@@ -1,10 +1,10 @@
 """
-ForexBot2 — Data Provider (Twelve Data, USD/JPY only)
+ForexBot2 — Data Provider (Twelve Data SDK, USD/JPY only)
-=======================================================
+==========================================================
-Fetches and validates OHLCV from Twelve Data. No synthetic fallback —
+Fetches and validates OHLCV via the official `twelvedata` Python SDK.
-if the API fails the caller learns about it. Adds a parquet on-disk
+No synthetic fallback — if the API fails the caller learns about it.
-cache so the backtest worker pool reuses one download instead of
+Adds a parquet on-disk cache so the backtest worker pool reuses one
-hammering Twelve Data 27 times during a parameter sweep.
+download instead of hammering Twelve Data during a parameter sweep.
 """
 
 import logging
@@ -14,7 +14,7 @@ from collections import deque
 from typing import Dict, Optional
 
 import pandas as pd
-import requests
+from twelvedata import TDClient
 
 from config import PlatformConfig, Timeframe, USDJPY, DISPLAY, PIP_VALUE
 
@@ -27,9 +27,7 @@ class DataIntegrityError(Exception):
 
 
 class ForexDataProvider:
-    """Twelve Data REST client + on-disk parquet cache."""
+    """Twelve Data SDK client + on-disk parquet cache."""
-
-    TD_BASE_URL = "https://api.twelvedata.com"
 
     TD_INTERVALS = {
         Timeframe.M1:  "1min",
@@ -66,10 +64,9 @@ class ForexDataProvider:
         self._cache: Dict[str, pd.DataFrame] = {}
         self._cache_timestamps: Dict[str, float] = {}
         self._request_times: deque = deque()
-        self._session = requests.Session()
+        self._td: Optional[TDClient] = (
-        self._session.headers.update({
+            TDClient(apikey=self._api_key) if self._api_key else None
-            "User-Agent": "Mozilla/5.0 (compatible; ForexIntel/1.0)"
+        )
-        })
 
     @staticmethod
     def _to_td_symbol(symbol: str) -> str:
@@ -230,7 +227,7 @@ class ForexDataProvider:
         start_date: Optional[str] = None,
         end_date: Optional[str] = None,
     ) -> pd.DataFrame:
-        if not self._api_key:
+        if self._td is None:
             raise DataIntegrityError(
                 "No Twelve Data API key. Set TWELVE_DATA_API_KEY env var "
                 "or config.twelve_data_api_key."
@@ -241,50 +238,44 @@ class ForexDataProvider:
 
         self._throttle(credits=1)
 
-        params = {
+        kwargs = {
             "symbol": td_symbol,
             "interval": interval,
-            "outputsize": outputsize,
+            "outputsize": min(outputsize, 5000),
-            "apikey": self._api_key,
             "timezone": "UTC",
-            "format": "json",
         }
         if start_date:
-            params["start_date"] = start_date
+            kwargs["start_date"] = start_date
         if end_date:
-            params["end_date"] = end_date
+            kwargs["end_date"] = end_date
 
-        resp = self._session.get(
+        try:
-            f"{self.TD_BASE_URL}/time_series",
+            df = self._td.time_series(**kwargs).as_pandas()
-            params=params,
+        except Exception as e:
-            timeout=30,
-        )
-        resp.raise_for_status()
-        data = resp.json()
-
-        if data.get("status") == "error":
             raise DataIntegrityError(
-                f"Twelve Data error for {td_symbol}: {data.get('message', 'unknown')}"
+                f"Twelve Data fetch failed for {td_symbol}: {e}"
-            )
+            ) from e
 
-        values = data.get("values")
+        if df is None or df.empty:
-        if not values:
             raise DataIntegrityError(f"Empty response from Twelve Data for {td_symbol}")
 
-        records = [
+        # SDK returns lowercase columns; engine expects capitalised.
-            {
+        df = df.rename(columns={
-                "Datetime": pd.Timestamp(v["datetime"], tz="UTC"),
+            "open": "Open", "high": "High", "low": "Low",
-                "Open":   float(v["open"]),
+            "close": "Close", "volume": "Volume",
-                "High":   float(v["high"]),
+        })
-                "Low":    float(v["low"]),
+        for col in ("Open", "High", "Low", "Close", "Volume"):
-                "Close":  float(v["close"]),
+            if col in df.columns:
-                "Volume": float(v.get("volume") or 0),
+                df[col] = pd.to_numeric(df[col], errors="coerce")
-            }
+        if "Volume" not in df.columns:
-            for v in reversed(values)
+            df["Volume"] = 0.0
-        ]
 
-        df = pd.DataFrame(records).set_index("Datetime")
+        # SDK returns newest-first; engine + indicators want oldest-first.
+        df = df.sort_index()
+        if df.index.tz is None:
+            df.index = df.index.tz_localize("UTC")
         df.index.name = "Datetime"
+
         logger.debug(f"Fetched {len(df)} bars for {td_symbol} @ {interval}")
         return df
 

main.py

@@ -333,6 +333,18 @@ Examples:
                         help="Gate signals on higher-TF ADX trend confirmation")
     parser.add_argument("--regime-min-adx", type=float, default=30.0,
                         help="Min ADX for regime filter (default 30, requires --regime-filter)")
+    parser.add_argument("--session-filter", action="store_true",
+                        help="Only trade during a UTC hour window (default 07-16, London + NY)")
+    parser.add_argument("--session-start", type=int, default=7,
+                        help="Session start hour UTC, inclusive (default 7)")
+    parser.add_argument("--session-end", type=int, default=16,
+                        help="Session end hour UTC, exclusive (default 16)")
+    parser.add_argument("--auto-session-filter", action="store_true",
+                        help="Auto-toggle session filter when recent trades show adverse mean pnl%%")
+    parser.add_argument("--auto-session-lookback", type=int, default=10,
+                        help="Lookback in closed trades for auto-session detection (default 10)")
+    parser.add_argument("--auto-session-threshold-pct", type=float, default=-0.3,
+                        help="Mean pnl%% threshold below which auto-session activates (default -0.3)")
     parser.add_argument("--ablate", action="store_true",
                         help="Run leave-one-out indicator ablation sweep (parallel)")
 
@@ -368,6 +380,12 @@ def build_config(args) -> PlatformConfig:
     config.backtest.optimize = args.optimize
     config.backtest.regime_filter = args.regime_filter
     config.backtest.regime_filter_min_adx = args.regime_min_adx
+    config.backtest.session_filter = args.session_filter
+    config.backtest.session_start_hour = args.session_start
+    config.backtest.session_end_hour = args.session_end
+    config.backtest.auto_session_filter = args.auto_session_filter
+    config.backtest.auto_session_lookback_trades = args.auto_session_lookback
+    config.backtest.auto_session_threshold_pct = args.auto_session_threshold_pct
 
     return config
 

metrics.py

@@ -30,6 +30,15 @@ class Trade:
     grade: str                # signal grade at entry
     score_at_entry: float
     confidence_at_entry: float
+    # Diagnostic fields captured at entry — used to mine for loss-pattern
+    # filters when a window underperforms.
+    entry_hour: int = -1            # UTC hour 0-23
+    entry_dow: int = -1             # 0=Mon, 6=Sun
+    adx_at_entry: float = 0.0
+    atr_pct_at_entry: float = 0.0
+    market_regime: str = ""         # trending_up / trending_down / ranging
+    volatility_regime: str = ""     # low / medium / high
+    mtf_agreement: bool = False
 
 
 @dataclass

requirements.txt

@@ -3,3 +3,4 @@ pandas>=2.0
 requests>=2.28
 pyarrow>=14.0
 tqdm>=4.66
+twelvedata>=1.2.0

signal_aggregator.py

@@ -130,8 +130,33 @@ class SignalAggregator:
         if bt.regime_filter:
             self._apply_regime_filter_live(report, dfs)
 
+        if bt.session_filter:
+            self._apply_session_filter_live(report)
+
         return report
 
+    def _apply_session_filter_live(self, report: SignalReport) -> None:
+        """Skip signals outside the configured UTC trading window."""
+        bt = self.config.backtest
+        now_h = datetime.now(timezone.utc).hour
+        if not (bt.session_start_hour <= now_h < bt.session_end_hour):
+            report.warnings.append(
+                f"SESSION FILTER: {now_h:02}:00 UTC outside "
+                f"{bt.session_start_hour:02}:00-{bt.session_end_hour:02}:00 — skipping"
+            )
+            report.final_signal = SignalStrength.NEUTRAL
+            report.signal_quality = "F"
+            report.action_recommendation = (
+                f"NO TRADE — Outside session "
+                f"({bt.session_start_hour:02}-{bt.session_end_hour:02} UTC; now {now_h:02})"
+            )
+            report.trade_setup = None
+        else:
+            report.warnings.append(
+                f"Session: {now_h:02}:00 UTC inside trading window "
+                f"{bt.session_start_hour:02}-{bt.session_end_hour:02}"
+            )
+
     def _apply_regime_filter_live(
         self, report: SignalReport, dfs: Dict[Timeframe, pd.DataFrame]
     ) -> None: