From 7dbcbc6c82bb8a0aa992a4b0671163b60afa86bc Mon Sep 17 00:00:00 2001 From: Harwinder Sekhon Date: Tue, 4 Aug 2026 20:21:34 -0400 Subject: [PATCH] audit: NY-anchored weekend calendar (all 3 copies) + FB2_EMA_LO crash-proofing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner decisions from the 2026-08-04 audit report: - spread-in-exit-triggers: DECLINED (books stay comparable; ledger in CLAUDE.md records the measured ~4-6% optimism). - weekend clock: FIXED. The market week is Fri 17:00 -> Sun 18:00 New York time, not Fri/Sun 22:00 UTC — during US DST the Oanda feed stops at 21:00 UTC (measured: 159/162 summer weekends). All three weekend implementations (providers/base, signal_aggregator, gpu_backtest calendar filter) now share the NY-anchored rule; a new test sweeps them hourly across both DST transitions and asserts agreement. No NTP needed — the host clock was fine, the constant was wrong. Ends the weekly false-"STALE DATA / feed lagging" hours (summer Fri 21-22 UTC, winter Sun 22-23 UTC). - FB2_EMA_LO (delegated): kept as a research knob but crash-proofed — an unparseable value degrades to the tuned default 0.5 with a warning instead of killing every scan. Documented in CLAUDE.md. Book gates: M15 12mo pinned book byte-identical (+78.7111R, 397 trades). M5 12mo drops exactly ONE trade (556 -> 555, +97.56 -> +98.64R): a phantom whose signal fired on the final summer-Friday bar (decision 21:00 UTC = NY 17:00 market close) and was "filled" at Sunday's reopen 49 hours later, -1.08R — an entry live trading can never take. The backtest no longer contains it; live == backtest parity improved. M15 unaffected because hour 21 is in its blocked set. Suite: 341/341 green. Claude-Session: https://claude.ai/code/session_01UnZkzeFcTu9agFg8BDh2tn --- CLAUDE.md | 10 +++---- backtest/gpu_backtest.py | 19 +++++++++---- providers/base.py | 34 +++++++++++++++++------ signal_aggregator.py | 35 ++++++++++++++++++------ test_all.py | 59 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 28 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 90c0509..6f36c7a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -74,12 +74,12 @@ Signal currency everywhere is `SignalStrength` (−2..+2). DataFrames are UTC tz 8. Don't add config fields nothing reads; don't disable indicators in production paths (`disabled_indicators` is for experiments only). 9. EUR/USD was removed 2026-07-16 after a no-edge verdict (cost-to-volatility; last commit with EUR support: `7d3fd95`). Do not re-add without new data. BTC/ETH work lives on unmerged branches only. -## Known open items from the 2026-08-04 QA audit (owner decisions pending) +## 2026-08-04 QA audit — owner decisions ledger -- **Weekend constant vs Oanda reality**: `_is_forex_weekend` hardcodes Fri 22:00→Sun 22:00 UTC, but Oanda's week is Fri 17:00→Sun 18:00 New York time — during US DST the feed stops at 21:00 UTC (159/162 summer weekends measured) and winter reopen is Sun 23:00. Result: two false-"STALE DATA/feed lagging" hours weekly (summer Fri 21–22, winter Sun 22–23). Fix = NY-anchored calendar in all three copies; books provably unaffected (no bars exist in those hours) but run a book gate anyway. -- **Spread in exit-trigger detection (rule 6)**: backtest stop triggers use raw mid High/Low; bid/ask-adjusted triggers would change 15/397 trades (−3.55R) at $0.30 and 30/397 (−4.24R) at $0.65 on the pinned M15 12mo book — a ~4–6% systematic optimism. Fixing invalidates all book comparability: conscious re-pin decision only. -- **H4 seeding divergence**: live H4 fetches only 380 bars; the backtest sees unbounded history — ~2% of sampled H4 bars flip an indicator signal (composite Δ up to 12.3). M5/M15/H1 are clean at their real fetch depths. H4 is already "unproven for auto-execution". -- `FB2_EMA_LO` env var is an undocumented knob inside the scoring path (`signal_aggregator.py` EMA200-proximity penalty floor); an unparseable value crashes every analysis. Remove or config-ify only with rule-1 revalidation. +- **Weekend calendar: FIXED (owner-approved 2026-08-04).** All three `_is_forex_weekend` copies are NY-anchored (Fri 17:00 → Sun 18:00 ET; during US DST that is Fri 21:00 UTC — measured on 5yr of Oanda bars). Kills the weekly false-"STALE DATA/feed lagging" hours. Book impact, fully accounted: M15 byte-identical; M5 12mo dropped exactly ONE trade — a phantom whose signal fired on the final summer-Friday bar (close 21:00 UTC) and "filled" at Sunday's reopen 49 h later (−1.08R, live-impossible). M5 12mo reference at HEAD: 555 trades, +98.64R PF≈1.50 @ $0.65 (was 556/+97.56/1.48). A Module 21 test sweeps all three copies hourly across both DST transitions. +- **Spread in exit-trigger detection (rule 6): owner DECLINED 2026-08-04.** Backtest stop triggers stay on raw mid High/Low; the measured optimism (~4–6% of total R: −3.55R @ $0.30, −4.24R @ $0.65 on the M15 12mo book) is accepted and books stay comparable. Re-open only as a conscious re-pin campaign. +- **`FB2_EMA_LO`: HARDENED 2026-08-04 (delegated).** Research override for the EMA200-proximity band, default = tuned 0.5; an unparseable value now degrades to the default with a warning instead of crashing every scan. Documented here — it is a knob that exists. +- **H4 seeding divergence: parked.** Live H4 fetches only 380 bars vs the backtest's unbounded history — ~2% of sampled H4 bars flip an indicator signal (composite Δ up to 12.3). M5/M15/H1 clean at their real fetch depths. Fold into any future H4 campaign; H4 remains "unproven for auto-execution". ## Current state notes (2026-07-30) diff --git a/backtest/gpu_backtest.py b/backtest/gpu_backtest.py index 4f0eb8b..6982027 100644 --- a/backtest/gpu_backtest.py +++ b/backtest/gpu_backtest.py @@ -40,10 +40,13 @@ from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Dict, List, Optional, Tuple +from zoneinfo import ZoneInfo import numpy as np import pandas as pd +_NY_TZ = ZoneInfo("America/New_York") + # Ensure project root on sys.path so this script can be invoked from anywhere. PROJECT_ROOT = Path(__file__).resolve().parent.parent if str(PROJECT_ROOT) not in sys.path: @@ -1450,18 +1453,22 @@ class BacktestConfig: def _is_xau_blocked_by_calendar(ts: pd.Timestamp, cfg: PlatformConfig) -> Optional[str]: """Apply XAU weekend + allowed-weekday filter matching live `analyze_pair`. - Forex market closed: Friday 22:00 UTC -> Sunday 22:00 UTC (Sydney open). - Must match `_is_forex_weekend` in signal_aggregator.py exactly. + Market closed Friday 17:00 -> Sunday 18:00 NEW YORK time (= Fri + 21:00 UTC in US summer, 22:00 in winter — matches measured Oanda + feed behavior; 2026-08-04 audit I43). Must match `_is_forex_weekend` + in signal_aggregator.py exactly (rule 4; Module 21 agreement test). + Note the weekday gates below stay on UTC weekday — they mirror the + live gates, which evaluate UTC weekdays. """ xau = cfg.xau_strategy if xau.weekend_filter: - wd = ts.weekday() - hour = ts.hour - if wd == 4 and hour >= 22: + ny = ts.astimezone(_NY_TZ) + wd = ny.weekday() + if wd == 4 and ny.hour >= 17: return "WEEKEND_FILTER" if wd == 5: return "WEEKEND_FILTER" - if wd == 6 and hour < 22: + if wd == 6 and ny.hour < 18: return "WEEKEND_FILTER" if xau.allowed_weekdays and ts.weekday() not in xau.allowed_weekdays: return "DAY_FILTER" diff --git a/providers/base.py b/providers/base.py index 183ee5e..1d9de5f 100644 --- a/providers/base.py +++ b/providers/base.py @@ -24,6 +24,7 @@ import time from abc import ABC, abstractmethod from datetime import datetime, timedelta from typing import AsyncIterator, Callable, Dict, Optional, Sequence, Tuple +from zoneinfo import ZoneInfo import pandas as pd @@ -31,19 +32,36 @@ from config import PAIR_INFO, PlatformConfig, Timeframe, XAUUSD logger = logging.getLogger(__name__) +# The gold/FX market week is defined in New York time (see +# _is_forex_weekend); a fixed UTC hour is wrong for half the year. +_NY_TZ = ZoneInfo("America/New_York") + def _is_forex_weekend(now=None) -> bool: - """Forex market closed: Friday 22:00 UTC → Sunday 22:00 UTC.""" + """Gold/FX market closed for the weekend. + + The market week is anchored to NEW YORK time — Friday 17:00 ET → + Sunday 18:00 ET — not a fixed UTC hour. During US DST the feed + actually stops at 21:00 UTC (measured on 5 years of Oanda bars: + 159/162 summer Fridays; winter close 22:00, winter reopen Sunday + 23:00 UTC). The old hardcoded Fri 22:00→Sun 22:00 UTC rule produced + two false-"STALE DATA/feed lagging" hours every week (2026-08-04 + audit I43, fixed per owner decision). + + Kept as three deliberate copies (here, signal_aggregator.py, + gpu_backtest._is_xau_blocked_by_calendar) — a Module 21 test asserts + they agree hour-by-hour across DST transitions. + """ if now is None: now = pd.Timestamp.now(tz="UTC") - wd = now.weekday() - hour = now.hour - if wd == 4 and hour >= 22: - return True - if wd == 5: - return True - if wd == 6 and hour < 22: + ny = now.astimezone(_NY_TZ) + wd = ny.weekday() + if wd == 4: # Friday + return ny.hour >= 17 + if wd == 5: # Saturday return True + if wd == 6: # Sunday + return ny.hour < 18 return False diff --git a/signal_aggregator.py b/signal_aggregator.py index 4bf9bba..e21a676 100755 --- a/signal_aggregator.py +++ b/signal_aggregator.py @@ -10,9 +10,12 @@ import os from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Dict, List, Optional +from zoneinfo import ZoneInfo import pandas as pd +_NY_TZ = ZoneInfo("America/New_York") + from chart_patterns import detect_chart_pattern from config import PlatformConfig, SignalStrength, Timeframe from providers import BaseDataProvider, make_data_provider @@ -23,15 +26,19 @@ logger = logging.getLogger(__name__) def _is_forex_weekend(now_utc: datetime) -> bool: - """Forex market closed: Friday 22:00 UTC → Sunday 22:00 UTC (Sydney open).""" - wd = now_utc.weekday() - hour = now_utc.hour - if wd == 4 and hour >= 22: # Friday after 22:00 - return True - if wd == 5: # All Saturday - return True - if wd == 6 and hour < 22: # Sunday before 22:00 + """Gold/FX market closed for the weekend: Friday 17:00 → Sunday 18:00 + NEW YORK time (= Fri 21:00 UTC in US summer, 22:00 in winter — + matches measured Oanda feed behavior; 2026-08-04 audit I43). + Deliberate copy of providers.base._is_forex_weekend — a Module 21 + test asserts the copies agree across DST transitions.""" + ny = now_utc.astimezone(_NY_TZ) + wd = ny.weekday() + if wd == 4: # Friday + return ny.hour >= 17 + if wd == 5: # Saturday return True + if wd == 6: # Sunday + return ny.hour < 18 return False @@ -1253,7 +1260,17 @@ class SignalAggregator: ema200 = primary.key_levels.get("ema_trend") if ema200 and ema200 > 0 and report.current_price > 0: ema_dist_pct = abs(report.current_price - ema200) / ema200 * 100.0 - _lo = float(os.environ.get("FB2_EMA_LO", "0.5")) + # FB2_EMA_LO: research override for the EMA200-proximity + # penalty band; the tuned default is 0.5. An unparseable + # value must degrade to the default, never crash the scan + # (2026-08-04 audit; hardened per owner delegation). + try: + _lo = float(os.environ.get("FB2_EMA_LO", "0.5")) + except (TypeError, ValueError): + logger.warning( + "FB2_EMA_LO=%r is not a number — using tuned " + "default 0.5", os.environ.get("FB2_EMA_LO")) + _lo = 0.5 if ema_dist_pct < _lo: report.warnings.append(f"NEAR EMA200 ({ema_dist_pct:.2f}%) — chop risk") penalties.append(("ema200_near", 5)) diff --git a/test_all.py b/test_all.py index cb3fad4..0a3dc23 100644 --- a/test_all.py +++ b/test_all.py @@ -6397,6 +6397,65 @@ def _(): assert [p for p, _ in _gb._apply_cooldown(cands, 1)] == [10, 12, 15, 21, 24] +@test("weekend calendar is NY-anchored: summer close 21:00 UTC, winter reopen 23:00") +def _(): + # 2026-08-04 audit I43, fixed per owner decision: the market week is + # Fri 17:00 -> Sun 18:00 NEW YORK time (measured on 5yr of Oanda + # bars), not a fixed UTC hour. + import providers.base as pbase + T = lambda s: pd.Timestamp(s, tz="UTC") + # US DST 2026: Mar 8 – Nov 1. + assert pbase._is_forex_weekend(T("2026-07-31 21:30")) is True, \ + "summer Friday 21:30 UTC = 17:30 ET → market CLOSED (old code said open)" + assert pbase._is_forex_weekend(T("2026-07-31 20:30")) is False + assert pbase._is_forex_weekend(T("2026-01-02 21:30")) is False, \ + "winter Friday 21:30 UTC = 16:30 ET → still open" + assert pbase._is_forex_weekend(T("2026-01-02 22:30")) is True + assert pbase._is_forex_weekend(T("2026-01-04 22:30")) is True, \ + "winter Sunday 22:30 UTC = 17:30 ET → still closed (old code said open)" + assert pbase._is_forex_weekend(T("2026-01-04 23:30")) is False + assert pbase._is_forex_weekend(T("2026-08-02 21:30")) is True # summer Sun 17:30 ET + assert pbase._is_forex_weekend(T("2026-08-02 22:30")) is False # summer Sun 18:30 ET + assert pbase._is_forex_weekend(T("2026-08-01 03:00")) is True # Saturday + + +@test("weekend calendar: all three copies agree hourly across DST transitions") +def _(): + import providers.base as pbase + import signal_aggregator as sa_mod + import gpu_backtest as _gb + cfg = PlatformConfig() + cfg.xau_strategy.weekend_filter = True + cfg.xau_strategy.allowed_weekdays = [] # isolate the weekend gate + # Sweep spring-forward (2026-03-06..11) and fall-back (2026-10-30..11-04). + for start in ("2026-03-06", "2026-10-30"): + for ts in pd.date_range(start, periods=5 * 24, freq="1h", tz="UTC"): + a = pbase._is_forex_weekend(ts) + b = sa_mod._is_forex_weekend(ts) + c = _gb._is_xau_blocked_by_calendar(ts, cfg) == "WEEKEND_FILTER" + assert a == b == c, f"{ts}: base={a} aggregator={b} backtest={c}" + + +@test("FB2_EMA_LO: garbage value degrades to tuned default, scan survives") +def _(): + os.environ["FB2_EMA_LO"] = "not-a-number" + try: + config = PlatformConfig() + config.primary_timeframe = Timeframe.H1 + config.confirmation_timeframes = [Timeframe.M15, Timeframe.H4] + agg = SignalAggregator(config) + dfs = { + Timeframe.H1: make_ohlcv(300, seed=42), + Timeframe.M15: make_ohlcv(300, seed=43), + Timeframe.H4: make_ohlcv(300, seed=44), + } + report = agg.analyze_from_dataframes("XAU/USD", dfs, use_news=False) + assert isinstance(report, SignalReport), \ + "analysis must survive an unparseable FB2_EMA_LO" + finally: + os.environ.pop("FB2_EMA_LO", None) + + # ══════════════════════════════════════════════════════════════════════ # SUMMARY # ══════════════════════════════════════════════════════════════════════