From de36b104d9c9f136d815193571ba49505bfbdf57 Mon Sep 17 00:00:00 2001 From: Harwinder Sekhon Date: Mon, 7 Sep 2026 11:07:29 -0400 Subject: [PATCH] providers: keep real news bars (snap-back bad-print rule), Oanda retry ladder, scan_kind=failed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2026-09-07 signal-log inspection follow-up — three fixes that do not depend on sample size. - providers/base.py: the signal-side 10σ outlier filter is now a bad-print filter. A >10σ close-to-close bar is dropped only when the NEXT close snaps back ≥80% of the way (OUTLIER_SIGMA / OUTLIER_REVERT_FRACTION); persisting moves and the newest bar are kept. The unconditional rule deleted ~36 real M5 news bars/yr in the backtester (whole-frame σ) and a different ~24/yr live (fetch-window σ) — e.g. the 2026-09-04 12:30 UTC NFP bar vanished from M5+M15 and the 12:35 scan was a byte-identical repeat of 12:30. 3-yr Oanda drops are now M5 2 / M15 1 / H1 0. - providers/oanda.py: transient 401/5xx/network faults retry along RETRY_DELAYS_S=(2,5,10)s under RETRY_BUDGET_S=45s. 11 boundary scans in two days had lost their primary frame at xx:00:22–:26 UTC; one 2 s retry could not outlast the top-of-hour blip. - signal_aggregator/_log_signal: a scan whose primary fetch failed logs scan_kind='failed' (price 0, no signal); analyze_signal_log.parity_report skips it like 'blocked'. - test_all.py: 466 tests (+6): retry ladder walks all delays, 5xx blip recovers on the third attempt, retry budget stops the ladder, persisting/newest/partially-retraced 10σ bars kept, failed-scan row kind, parity skip. - CLAUDE.md: cleaning rule, retry ladder, failed scan kind, dated current-state note. Book impact at $0.65 (rule 1 still passes everywhere, both halves positive): 12mo M5 557/+112.21/PF1.56 → 560/+108.81/1.54 M15 397/+73.93/1.53 → 398/+74.61/1.53 H1 identical 3yr M5 857/+161.3/1.53 → 870/+153.6/1.50 M15 1086/+148.7/1.37 → 1092/+151.9/1.38 H1 255/+70.5/1.98 → 253/+70.0/1.98 H4 113/+22.7/1.63 → 114/+21.7/1.59 Only trades on the previously-deleted event bars move; the pinned M5/M15 CSVs are no longer byte-reproducible and should be regenerated at the next re-pin. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01NDnBxJ2y8GRUM6ZNFJs1hZ --- CLAUDE.md | 5 +- analyze_signal_log.py | 2 +- providers/base.py | 42 ++++++++--- providers/oanda.py | 51 +++++++++---- signal_aggregator.py | 12 ++- signal_log.py | 6 +- test_all.py | 166 +++++++++++++++++++++++++++++++++++++++++- 7 files changed, 248 insertions(+), 36 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 2c67cc8..78d50a6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -61,11 +61,11 @@ Signal currency everywhere is `SignalStrength` (−2..+2). DataFrames are UTC tz ### Layers -- **`providers/`** — one class per provider subclassing `BaseDataProvider` (`providers/base.py`), which owns everything the pipeline depends on: TTL+boundary cache, forming-candle drop, freshness check, validate/clean, paginated `fetch_range` (feeds the backtester from any provider), optional tick streaming. Subclasses implement only `_fetch_raw` + symbol/interval maps. Registry + `make_data_provider(config, name=None)` in `providers/__init__.py`; config.json `data_providers.active` selects the live candle feed and the optional `data_providers.streaming` scalar selects a different provider for the tick stream. A test enforces that subclasses don't override shared base methods (`supports_streaming`/`iter_ticks` are the sanctioned streaming override points). Tiingo streams mid-price ticks (`wss://api.tiingo.com/fx`) — do not re-add spread display from its free FX feed (two-tier indicative quotes, removed 2026-07-23); the no-spread rule extends to the WS stream. +- **`providers/`** — one class per provider subclassing `BaseDataProvider` (`providers/base.py`), which owns everything the pipeline depends on: TTL+boundary cache, forming-candle drop, freshness check, validate/clean, paginated `fetch_range` (feeds the backtester from any provider), optional tick streaming. Subclasses implement only `_fetch_raw` + symbol/interval maps. **Cleaning rule (2026-09-07):** the signal-side outlier filter is a bad-print filter, not a volatility filter — a >10σ close-to-close bar is dropped only when the NEXT close snaps back ≥80% of the way (`OUTLIER_SIGMA`/`OUTLIER_REVERT_FRACTION` on the base class); persisting moves (NFP/CPI/Sunday-gap bars) and the newest bar are always kept. The previous unconditional 10σ rule deleted ~36 real M5 bars/yr in the backtester and a *different* ~24/yr live (window-dependent σ), hiding e.g. the 2026-09-04 12:30 UTC NFP bar from ATR/exhaustion math. Oanda fetches retry 401/5xx/network faults along `RETRY_DELAYS_S` (2/5/10 s) under a 45 s `RETRY_BUDGET_S`; a scan that still loses its primary frame is logged with `scan_kind='failed'`. Registry + `make_data_provider(config, name=None)` in `providers/__init__.py`; config.json `data_providers.active` selects the live candle feed and the optional `data_providers.streaming` scalar selects a different provider for the tick stream. A test enforces that subclasses don't override shared base methods (`supports_streaming`/`iter_ticks` are the sanctioned streaming override points). Tiingo streams mid-price ticks (`wss://api.tiingo.com/fx`) — do not re-add spread display from its free FX feed (two-tier indicative quotes, removed 2026-07-23); the no-spread rule extends to the WS stream. - **`engine.py`** — `TechnicalEngine`: 9 scored indicators (ema_trend, rsi, macd, bollinger, adx, roc, keltner, divergence, candlestick) + ATR as context. Regime classification (trending_up/down, `trending` = mixed/direction-UNKNOWN, transitioning, ranging) drives per-indicator weight multipliers and regime-aware RSI reinterpretation. Compute order matters (ATR first; MACD always computed even when not a pillar — divergence needs the histogram). Private series ride on the result as `_underscore` attrs; scorers/formatters skip anything `_`-prefixed or weight-0. - **`signal_aggregator.py`** — MTF agreement, diminishing-decay penalty system, exhaustion hard block, quality grading, then an ordered chain of post-grade gates (session, XAU SELL quality/vol/hour, regime blocks, M5 noise gates), then three setup generators: XAU ATR (A+/A), generic swing-based, plus the M5 pullback (`XAU_PB`) and chart-pattern (`XAU_CP`, `chart_patterns.py`) entry paths. The PB/CP generators run AFTER the gate chain and force quality to "B"; the CP path deliberately accepts grade-F composites and re-checks only session/day/big-candle (via warning prefixes) plus its own SELL/vol/RSI gates — it does NOT re-check exhaustion or the M5 min-score/MTF gates. **This is validated behavior, not a bug** (2026-08-04 audit: 226/586 trades in the shipped M5 12mo book are XAU_CP, +38.9R, 104 of them exhaustion-flagged and net-positive) — changing it is a rule-1 revalidation event. `TradeSetup.strategy` selects trailing params; trail params are stamped on the setup from the *producing* config so per-TF overrides reach formatters. - **`pair_signals/`** — per-pair config factories. **Always use the factory (`produce_signal` / `make_config`), never bare `PlatformConfig()`** — the factory applies the config.json overlay first, then the validated per-TF deltas (M5 hour-9 block, M15 trail arm 0.70, M15 SELL-hour block, M5 SELL-in-transitioning block, H1 all-shorts block (2026-09-03), etc.; evidence in the docstring). A test enforces factory ≡ inline-config equivalence. Confirmation map: M5→[M15,H1], M15→[M5,H1], H1→[M15,H4], H4→[H1,D1]. -- **`signal_log.py` / `advisory_outcome.py`** — SQLite signal-history logger (stdlib only). Hooked into `SignalAggregator.analyze_pair` only (the live entry point — the backtester never writes here, preserving parity), so every live scan from every surface (Telegram `telegram`, CLI `cli`, dashboard `dashboard`) lands one row in `logs/signals.db`: signal/grade/price, MTF + trade-setup + reasons JSON, full per-TF indicator snapshots, the last 14 closed OHLCV candles per fetched TF, and the news at scan time (event risk + calendar events + scored headlines, re-fetched via the NewsEngine class-level caches). Config.json `signal_log` block (`enabled`, `path`, `log_blocked_scans`; env `SIGNAL_LOG_ENABLED`/`SIGNAL_LOG_PATH`/`SIGNAL_LOG_BLOCKED_SCANS`). **Fail-safe by contract** — a logging failure can never break a scan. Panel fields arrive post-hoc: `log_signal` stashes the rowid and `agent_panel.attach_panel` writes verdict + creation/cache provenance + payload hash onto the same row. **Advisory outcomes (2026-09):** EVERY row with a trade setup is seeded at log time (not only panel rows); same-setup re-logs within 60 min from any surface (`setup_key` = tf|direction|entry|stop|strategy) are `DUPLICATE` rows pointing at the canonical row via `advisory_event_id`, so sum `advisory_r_multiple` over `RESOLVED` rows only. The versioned model advances from later stored closed-candle snapshots (next-open reanchoring, intrabar peak with close-only ratchet, SL-first, scheduled weekend/21:00-UTC closures allowed, 300-bar timeout, $0.30/$0.65 views); when a snapshot no longer overlaps the outcome's last bar it rebuilds the stream from every stored snapshot of that TF before giving up. It is a cleaned-mid evidence model, not execution P&L. **Post-gate truth columns:** `gate_reason` (the warning that removed the setup), `actionable` (grade ≥ C AND setup — the `action` text is written BEFORE the gate chain and can say "STRONG SELL — High-quality setup" on a D row), `cleaned_warnings_json` (what the XAU_CP acceptance stripped), `bar_range_median` (= the misnamed `spread` column: median High-Low, NOT a broker spread), `scan_kind` (`blocked` weekend/day early returns carry no news/candles), `config_fingerprint` (gate config hash — surfaces config drift between processes). **News is content-addressed:** `headlines_json`/`events_json` are interned into `news_snapshots` (they were 73% of a 667 MB DB); read through the `signals_with_news` view or `signal_log.read_news`. Readers must open with `mode=ro`, never `immutable=1` (blind to WAL rows). **Weight-research columns (2026-09):** `bar_ts` (the primary bar LABEL the scan closed on — same string as `indicators_json`, indexed with `timeframe`), `engine_fingerprint` (hash of the SCORING config: indicator params, thresholds, the regime weight table in force, TFs — `SignalAggregator.engine_config_fingerprint`, the gate fingerprint's twin), `code_version` (short git SHA), `penalty_tags_json`/`penalty_total`/`momentum_fading`/`exhaustion_count`, and the post-hoc `watch_level`/`watch_kinds`/`watch_line` written by `signal_log.update_watch` from `violent_watch.attach_watch` (level and kinds are stored even when the display line is off). Every `indicators_json` entry also carries `eff_weight` (base weight × regime multiplier = the weight the composite actually used; `engine.regime_weight_table` is the single source of truth) and `wscore` (its contribution) — null for skipped indicators (weight 0, `_`-prefixed); **divergence and candlestick entries exist only when they fire, so absence is "no vote", not a zero vote**. **Table `scan_outcomes`:** model-free forward moves per scan (`fwd_1h/4h/24h` in $ and ATRs, `mfe_24h_atr`/`mae_24h_atr`, `burst_1h` = a ≥3×ATR bar within the hour, `bars_used`, `complete`), measured from the row's own last closed bar on the stored candle stream; a horizon stays NULL until it has elapsed, and `complete` needs the full 24 h with no gap over 3 bars beyond the scheduled weekend/17:00-NY closures (`advisory_outcome._has_continuity`). `SignalLogger.label_scan_outcomes` runs opportunistically at the end of `log()` (at most once per 60 s per process, 150 rows and ~2 s per pass, idempotent — a row is rewritten only when its numbers change), so no daemon is needed; `scripts/label_scan_outcomes.py` is the backlog driver (`--dry-run` counts read-only). Readers must dedup by the FIRST row per (`timeframe`, `bar_ts`) — several surfaces log the same bar. Tools: `analyze_signal_log.py` (`--stored-candles` DB-only replay with event/opportunity/one-position books + persona scorecard, `--news` news-sentiment lead/lag + extremes + outcome buckets, `--parity` telegram-vs-dashboard divergence report, `--scan-outcomes` forward-move + hit-rate table by TF×signal/grade with a NEUTRAL baseline and burst rate by watch level), `scripts/backfill_signal_outcomes.py` (seed/dedup/advance legacy rows), `scripts/label_scan_outcomes.py` (label/relabel scan outcomes), `scripts/migrate_signal_log_news.py` (intern legacy blobs; `--vacuum` with both writers stopped). +- **`signal_log.py` / `advisory_outcome.py`** — SQLite signal-history logger (stdlib only). Hooked into `SignalAggregator.analyze_pair` only (the live entry point — the backtester never writes here, preserving parity), so every live scan from every surface (Telegram `telegram`, CLI `cli`, dashboard `dashboard`) lands one row in `logs/signals.db`: signal/grade/price, MTF + trade-setup + reasons JSON, full per-TF indicator snapshots, the last 14 closed OHLCV candles per fetched TF, and the news at scan time (event risk + calendar events + scored headlines, re-fetched via the NewsEngine class-level caches). Config.json `signal_log` block (`enabled`, `path`, `log_blocked_scans`; env `SIGNAL_LOG_ENABLED`/`SIGNAL_LOG_PATH`/`SIGNAL_LOG_BLOCKED_SCANS`). **Fail-safe by contract** — a logging failure can never break a scan. Panel fields arrive post-hoc: `log_signal` stashes the rowid and `agent_panel.attach_panel` writes verdict + creation/cache provenance + payload hash onto the same row. **Advisory outcomes (2026-09):** EVERY row with a trade setup is seeded at log time (not only panel rows); same-setup re-logs within 60 min from any surface (`setup_key` = tf|direction|entry|stop|strategy) are `DUPLICATE` rows pointing at the canonical row via `advisory_event_id`, so sum `advisory_r_multiple` over `RESOLVED` rows only. The versioned model advances from later stored closed-candle snapshots (next-open reanchoring, intrabar peak with close-only ratchet, SL-first, scheduled weekend/21:00-UTC closures allowed, 300-bar timeout, $0.30/$0.65 views); when a snapshot no longer overlaps the outcome's last bar it rebuilds the stream from every stored snapshot of that TF before giving up. It is a cleaned-mid evidence model, not execution P&L. **Post-gate truth columns:** `gate_reason` (the warning that removed the setup), `actionable` (grade ≥ C AND setup — the `action` text is written BEFORE the gate chain and can say "STRONG SELL — High-quality setup" on a D row), `cleaned_warnings_json` (what the XAU_CP acceptance stripped), `bar_range_median` (= the misnamed `spread` column: median High-Low, NOT a broker spread), `scan_kind` (`blocked` weekend/day early returns carry no news/candles; `failed` = the primary fetch failed — price 0, no signal — readers skip it like `blocked`), `config_fingerprint` (gate config hash — surfaces config drift between processes). **News is content-addressed:** `headlines_json`/`events_json` are interned into `news_snapshots` (they were 73% of a 667 MB DB); read through the `signals_with_news` view or `signal_log.read_news`. Readers must open with `mode=ro`, never `immutable=1` (blind to WAL rows). **Weight-research columns (2026-09):** `bar_ts` (the primary bar LABEL the scan closed on — same string as `indicators_json`, indexed with `timeframe`), `engine_fingerprint` (hash of the SCORING config: indicator params, thresholds, the regime weight table in force, TFs — `SignalAggregator.engine_config_fingerprint`, the gate fingerprint's twin), `code_version` (short git SHA), `penalty_tags_json`/`penalty_total`/`momentum_fading`/`exhaustion_count`, and the post-hoc `watch_level`/`watch_kinds`/`watch_line` written by `signal_log.update_watch` from `violent_watch.attach_watch` (level and kinds are stored even when the display line is off). Every `indicators_json` entry also carries `eff_weight` (base weight × regime multiplier = the weight the composite actually used; `engine.regime_weight_table` is the single source of truth) and `wscore` (its contribution) — null for skipped indicators (weight 0, `_`-prefixed); **divergence and candlestick entries exist only when they fire, so absence is "no vote", not a zero vote**. **Table `scan_outcomes`:** model-free forward moves per scan (`fwd_1h/4h/24h` in $ and ATRs, `mfe_24h_atr`/`mae_24h_atr`, `burst_1h` = a ≥3×ATR bar within the hour, `bars_used`, `complete`), measured from the row's own last closed bar on the stored candle stream; a horizon stays NULL until it has elapsed, and `complete` needs the full 24 h with no gap over 3 bars beyond the scheduled weekend/17:00-NY closures (`advisory_outcome._has_continuity`). `SignalLogger.label_scan_outcomes` runs opportunistically at the end of `log()` (at most once per 60 s per process, 150 rows and ~2 s per pass, idempotent — a row is rewritten only when its numbers change), so no daemon is needed; `scripts/label_scan_outcomes.py` is the backlog driver (`--dry-run` counts read-only). Readers must dedup by the FIRST row per (`timeframe`, `bar_ts`) — several surfaces log the same bar. Tools: `analyze_signal_log.py` (`--stored-candles` DB-only replay with event/opportunity/one-position books + persona scorecard, `--news` news-sentiment lead/lag + extremes + outcome buckets, `--parity` telegram-vs-dashboard divergence report, `--scan-outcomes` forward-move + hit-rate table by TF×signal/grade with a NEUTRAL baseline and burst rate by watch level), `scripts/backfill_signal_outcomes.py` (seed/dedup/advance legacy rows), `scripts/label_scan_outcomes.py` (label/relabel scan outcomes), `scripts/migrate_signal_log_news.py` (intern legacy blobs; `--vacuum` with both writers stopped). - **`news_engine.py`** — Forex Factory calendar (MyFXBook RSS fallback) + RSS keyword sentiment with gold-inverse keywords. Class-level caches (fresh engine per scan). Optional DeepSeek enrichment via `llm_client.py` (requests-only, no SDK): batched headline sentiment overriding keyword scores, gold-direction on release surprises, and a market-summary blurb (`SignalReport.news_summary`) — **display/warning-only, keyword fallback on any failure, inert without a key**; prompts are static module constants (DeepSeek prefix-cache pricing) and requests send `thinking: disabled` (V4-Flash reasoning otherwise blows the timeout). By XAU-tuned default, event risk and sentiment **warn but do not gate signals** (`event_risk_caps_signals=False`, `sentiment_affects_signals=False`) — the LLM does not change that. **Headline pool (2026-09-04):** only headlines ≤ 4 h old with a parsable publish time (`headline_max_age_seconds=14400`, undated items dropped), and `get_fundamental_context` drops headlines whose every relevant score is inside ±0.1 (`_is_neutral_headline`, `NEUTRAL_HEADLINE_THRESHOLD`) before the per-currency mean — so `news_sentiment` from that date is a mean over directional headlines only (bolder values; log comparisons must split at the deploy date). The market summary prompt writes for someone who has never traded (no acronyms), sends headlines newest-first with the platform's own gold direction, and caps at 400 chars. **Verdict 2026-09-04** (`backtest/research/news_sentiment_value_2026-09-04.md`): `news_sentiment` is a trailing-24h echo of price (r≈0.67 with the past day, ≈0.03 with the next; flips 44 h after tops / 16 h after bottoms) — both gating flags stay False; re-check with `analyze_signal_log.py --stored-candles --news`. - **`agent_panel.py`** — ai-hedge-fund-style advisory panel: 4 DeepSeek persona calls (macro/technical/risk/contrarian, concurrent on a panel-private pool) + 1 synthesizer producing an **ADVISORY-ONLY** consensus (`AGREE/NEUTRAL/OPPOSE` + confidence + rationale) written onto `SignalReport.panel_*` fields. Attached **post-hoc** by consumers (`telegram_bot` alert path, `main --panel`, `dashboard_publisher`) — never runs inside `analyze_from_dataframes` (backtest parity, test-locked) and no gate reads its fields. Persona payloads are role-filtered: macro gets high-impact events plus only code-scored directional news; technical gets strategy/pattern, primary indicators, MTF, levels, and ATR-normalized candles; risk gets setup strategy, strategy cost assumption, trailing semantics, volatility/events/warnings (never the median candle range as spread); contrarian gets directional news, setup, timestamped latest-closed price, indicators, reasons, and warnings. Spend is bounded by exact cache, full-evidence-identity TTL reuse, and `max_panels_per_day`; changed evidence never inherits a direction-only cached verdict (test-locked — which also means TTL reuse effectively never fires live because price/candles change every bar; the daily cap is the real spend bound). Verdict provenance logs creation time, `MISS`/`EXACT_HIT`/`TTL_REUSE`, and a SHA-256 payload hash. Prompts forbid outside knowledge and outputs remain short. **OPPOSE guard (2026-09):** the 3-week signal-log scorecard showed the contrarian/risk/technical personas oppose almost every trend entry (chair said OPPOSE 73% of the time; opposed trades made +30R, PF 2.6) while the macro persona was the only informative voice (macro=support → 88% win). `_apply_oppose_guard` therefore downgrades a chair OPPOSE to NEUTRAL unless macro opposes, or technical opposes while macro is not supportive; the synthesizer prompt states the same rule. Re-score with `analyze_signal_log.py --stored-candles` (persona scorecard) before changing it again. **Verdict source (2026-09-03, owner decision):** all four personas still run and are logged (`panel_takes_json`), but `agent_panel.verdict_source` defaults to `"macro"` — the alert verdict is the macro persona's stance (support/neutral/oppose → AGREE/NEUTRAL/OPPOSE) and the synthesizer call is skipped (4 LLM calls per panel instead of 5); `"chair"` restores the four-voice synthesizer + OPPOSE guard. Re-score personas monthly with `analyze_signal_log.py --stored-candles`; weights are not to be tuned before ~300 graded outcomes with a held-out half. - **`market_outlook.py`** — hourly DeepSeek market brief for XAU/USD, using ATR-normalized M15/H1/H4 candles, visible indicator states, levels, news, and recent signal history (read `mode=ro`). It returns a written **ADVISORY-ONLY** bias/setup-forming assessment and persists it independently in `outlooks` within `logs/signals.db` together with the news, indicator/regime/level, and candle evidence sent to the model. It may be sent post-scan by Telegram or requested with `main.py --outlook`; it never imports or runs inside the signal path. Hourly dedup is by UTC hour BUCKET (the old `now − last < 3600` rule dropped every other hour because the H1 loop fires at boundary+10 s+scan time) plus its own UTC-day cap (default 12); every silent early return logs `market outlook: skipped (...)` and an empty LLM reply logs a warning — a dark feature must be visible in `logs/telegram.log`. Provider/news failures leave a safe partial brief while LLM/database failures degrade to no outlook. @@ -103,4 +103,5 @@ Signal currency everywhere is `SignalStrength` (−2..+2). DataFrames are UTC tz - 2026-09-03 signal-log campaign shipped two new per-TF gates (`backtest/research/signal_log_campaign_2026-09.md`): M5 `sell_blocked_regimes=["transitioning"]` and H1 `sell_min_quality=[]` (all H1 shorts blocked). Current-HEAD M5 canonical book is now `backtest/oanda_m5_12mo_065_nopb_20260903.csv` (557 / +112.21R / PF 1.56, halves +55.9/+56.4); H1 reference `backtest/oanda_h1_12mo_065_20260903.csv` (91 / +22.2R / PF 1.79; 5yr +49.1R PF 1.46; OOS +16.7R PF 1.74). - 2026-09-06 stake-sizing campaign (`backtest/research/sizing_campaign_2026-09.md`): a walk-forward "second opinion" stake multiplier (meta-labelling on 65 decision-time features, 3-year Oanda books at $0.65) FAILED the pre-registered rule on M5 (clean null) and M15 (lift inside the label-shuffle null, explained by BUY-vs-SELL). Nothing that changes numbers shipped. Behaviour-neutral tooling remains: `position_sizing.build_features` + `--export-features` (default export stays 63 columns), `SignalReport.penalty_tags/penalty_total/momentum_fading`, `TradeSetup.stake_multiplier` (None; reserved) (the campaign's research scripts were removed 2026-09-06; git history has them). Stage-B facts: M5 stretch measures are POSITIVE in both halves (do not re-propose exhaustion sizing on M5); M15 exhaustion tag is weakly negative (≈ +3R/yr, noise); bursts cluster at 12–15 UTC and 01 UTC bar-close and do NOT hurt entries. - Auto-execution (oanda_daemon.py / oanda_executor.py) was removed 2026-09-03 — the owner is not auto-trading; last commit containing it is 31002cb. Book-parity rules that lived in its docstring (no broker TP, 5-bar cooldown, once-per-bar trail) remain the backtester's model. +- 2026-09-07 signal-log inspection (fresh 2-day `logs/signals.db` import: NFP Friday + Monday; 5 setups, all BUY, 4 stopped out at local tops — two days is not evidence). Three sample-size-independent fixes shipped: (1) the signal-side 10σ outlier filter now drops only bars whose NEXT close snaps back ≥80% (it had been deleting ~36 real M5 news bars/yr in the backtester and a different ~24/yr live, e.g. the 2026-09-04 12:30 UTC NFP bar; 3-yr drops are now M5 2 / M15 1 / H1 0). Book impact at $0.65, 12mo: M5 557/+112.21R/PF 1.56 → 560/+108.81R/PF 1.54 (halves +55.6/+53.2), M15 397/+73.93/1.53 → 398/+74.61/1.53 (max DD 10.8→9.1R), H1 byte-identical; 3-yr (2023-09-05→2026-09-05): M5 857/+161.3/1.53 → 870/+153.6/1.50, M15 1086/+148.7/1.37 → 1092/+151.9/1.38, H1 255/+70.5/1.98 → 253/+70.0/1.98, H4 113/+22.7/1.63 → 114/+21.7/1.59 — only trades on the flagged event days move (M5 gives back a few Jan–Mar 2026 crash-period winners); the pinned M5/M15 CSVs are therefore no longer byte-reproducible (regenerate with the parity recipe when re-pinning). (2) Oanda candle fetches retry along a 2/5/10 s ladder under a 45 s budget (11 boundary scans in two days had lost their primary frame). (3) A scan that still loses its primary frame logs `scan_kind='failed'` and readers skip it. The live dashboard publisher was found dead (0 rows Friday, stopped Mon 02:20 UTC): install `deploy/gold-dashboard.service` on the live server. - 2026-09-06 stale-code cleanup (behaviour-neutral; suite green, M5/H1 12-month books byte-identical after every signal-side commit): `main.py --timeframe` now accepts M5/M15/H1/H4 and takes its confirmation TFs from the pair factory (the CLI's private table had paired M5 with M1+M15); `PlatformConfig.pip_value` is gone (`PAIR_INFO` is the source); removed as dead or superseded: `_get_affected_pairs`/`event_impact_on_pairs`, `SignalLogger.advance_with_bars`/`path`/`news_for`, `reset_llm_client`, the Telegram scan-summary/keyboard helpers and duplicate confirmation map, the 2026-07 Oanda-vs-TD research pager+study, `burst_profile.py`, `hour_analysis.py`, `scripts/compare_oanda_tiingo.py`, the never-implemented LLM-provider plan and the merged config.json spec. Second pass the same day (owner directive: keep only what the system needs): every research script and data file under `backtest/research/` (book_report, book_diff, the burst tools, sizing_model, commit_ladder, llm_news_eval + golden CSV, ff_calendar_scraper, calendar_impact, watch_rule_eval, the Dukascopy spread study + CSVs, the panel dry-test JSONL), the two completed study scripts in `scripts/`, the architecture diagram + generator and the data-source test plan were deleted; the campaign `.md` reports stay as the decision ledger. Git history before `953c5ad` has all of it. Third pass (owner: "remove dead functions as well"): the write-only pb/cp rejection counters, the unreachable `_TP_QUALITY_SCALE` B/C entries, the engine's unused DataFrame argument and M1 arm, the providers' generic-symbol guesses (unknown symbols now raise), the sub-$10 price format, the `=X` pair parsing and the unread `minutes_until`/`is_high_impact` news.json keys are gone. diff --git a/analyze_signal_log.py b/analyze_signal_log.py index 678a5ce..fc09fc6 100644 --- a/analyze_signal_log.py +++ b/analyze_signal_log.py @@ -545,7 +545,7 @@ def parity_report(rows: Sequence[LogRow]) -> dict: source (over-scanning symptom), and the divergence list.""" groups: Dict[Tuple[str, str], List[LogRow]] = {} for r in rows: - if not r.bar_ts or r.scan_kind == "blocked": + if not r.bar_ts or r.scan_kind in ("blocked", "failed"): continue groups.setdefault((r.timeframe, str(r.bar_ts)), []).append(r) per_bar: Dict[Tuple[str, str], List[int]] = {} diff --git a/providers/base.py b/providers/base.py index 1d9de5f..fb105bb 100644 --- a/providers/base.py +++ b/providers/base.py @@ -100,6 +100,14 @@ class BaseDataProvider(ABC): # boundary is missing the just-closed bar (feed latency). FRESH_FETCH_RETRY_SECONDS = 3.0 + # Signal-side bad-print filter (see _validate_and_clean): a bar is a + # candidate when its close-to-close return sits more than + # OUTLIER_SIGMA standard deviations from the frame mean, and is + # dropped only when the NEXT close snaps back at least + # OUTLIER_REVERT_FRACTION of the way to the pre-spike close. + OUTLIER_SIGMA = 10.0 + OUTLIER_REVERT_FRACTION = 0.8 + def __init__(self, config: PlatformConfig): self.config = config self._cache: Dict[str, pd.DataFrame] = {} @@ -498,20 +506,32 @@ class BaseDataProvider(ABC): returns = returns.replace([float("inf"), float("-inf")], float("nan")) std_ret = returns.std() if std_ret > 0: + # A 10σ close-to-close move is NOT bad data by default. On + # three years of Oanda XAU bars every such bar was a real + # news/reopen move (NFP, CPI, Sunday gap — 2026-09-07 audit: + # 155 flagged M5 bars, 7 snapped back), and dropping them hid + # the most violent bar of the day from ATR/exhaustion math — + # while the backtester (whole-frame σ) dropped a DIFFERENT + # set than live (fetch-window σ). A bad print has the one + # signature a real move lacks: the NEXT close returns to the + # pre-spike level. Drop a candidate only on that observed + # snap-back; the newest bar (no successor yet) is kept as + # real, and the snap-back bar itself is never a candidate + # (its residual against the pre-spike close is small). dev = returns - returns.mean() - outlier_mask = dev.abs() > 10 * std_ret - # A spike bar makes the NEXT bar's return look extreme too (it is - # measured against the spike). If the previous bar is flagged and - # this bar reverses direction, it is the legitimate snap-back — - # keep it, or every spike would punch a two-bar hole in the data. - recovery = ( - outlier_mask - & outlier_mask.shift(1, fill_value=False) - & (dev * dev.shift(1) < 0) + candidate = dev.abs() > self.OUTLIER_SIGMA * std_ret + close = df["Close"] + spike = close - close.shift(1) + residual = close.shift(-1) - close.shift(1) + snapped_back = residual.notna() & ( + residual.abs() + <= (1.0 - self.OUTLIER_REVERT_FRACTION) * spike.abs() ) - outlier_mask = outlier_mask & ~recovery + outlier_mask = candidate & snapped_back if outlier_mask.any(): - logger.warning(f"{symbol}: Removing {outlier_mask.sum()} extreme outlier bars") + logger.warning( + f"{symbol}: Removing {int(outlier_mask.sum())} bad-print " + f"bar(s) (>{self.OUTLIER_SIGMA:.0f}σ move that snapped back)") df = df[~outlier_mask] min_required = max( diff --git a/providers/oanda.py b/providers/oanda.py index e8b9d1d..b640490 100644 --- a/providers/oanda.py +++ b/providers/oanda.py @@ -32,7 +32,11 @@ class OandaDataProvider(BaseDataProvider): name = "oanda" DATA_ERROR = OandaDataError - RETRY_DELAY_S = 2.0 + # Transient-fault retry ladder (see _fetch_oanda_candles): sleeps + # between attempts, plus a wall-clock cap on the whole sequence. + RETRY_DELAYS_S = (2.0, 5.0, 10.0) + RETRY_BUDGET_S = 45.0 + RETRY_DELAY_S = RETRY_DELAYS_S[0] # first delay; kept for callers/tests OANDA_GRANULARITY = { Timeframe.M1: "M1", @@ -128,34 +132,53 @@ class OandaDataProvider(BaseDataProvider): else: params["count"] = min(count, self.MAX_BARS_PER_REQUEST) - # One retry absorbs Oanda's sporadic spurious 401s (~15-20/day on - # the practice API — the same token succeeds on adjacent requests) - # plus transient 5xx/network faults. A real credential problem - # fails both attempts and raises exactly as before. Other 4xx - # (bad request, 429) are never retried. + # Retry ladder for transient faults: Oanda's sporadic spurious + # 401s (~15-20/day on the practice API — the same token succeeds + # on adjacent requests), 5xx and network errors. Other 4xx (bad + # request, 429) are never retried; a real credential problem + # fails every attempt and raises as before. The 2026-09-07 + # signal-log audit found 11 boundary scans in two days that lost + # their primary frame at xx:00:22-:26 UTC: a single 2 s retry + # could not outlast a 10-15 s provider blip at the top of the + # hour, and a scan without its primary frame logs a price-0 + # NEUTRAL row. RETRY_BUDGET_S bounds the whole sequence so a scan + # (three fetches) still finishes well inside an M5 bar. last_err: Optional[OandaDataError] = None - for attempt in range(2): + resp = None + attempts = 1 + len(self.RETRY_DELAYS_S) + started = time.monotonic() + for attempt in range(attempts): if attempt: - time.sleep(self.RETRY_DELAY_S) + delay = self.RETRY_DELAYS_S[attempt - 1] + if time.monotonic() - started + delay > self.RETRY_BUDGET_S: + logger.warning( + f"Oanda fetch for {instrument}: retry budget " + f"({self.RETRY_BUDGET_S:.0f}s) exhausted after " + f"{attempt} attempt(s) — giving up") + break + time.sleep(delay) try: resp = self._session.get(url, params=params, timeout=30) except requests.RequestException as e: last_err = OandaDataError( f"Oanda candle fetch failed for {instrument}: {e}") last_err.__cause__ = e + resp = None + logger.warning( + f"Oanda fetch for {instrument} attempt " + f"{attempt + 1}/{attempts} failed: {e}") continue if resp.status_code == 401 or resp.status_code >= 500: last_err = OandaDataError( f"Oanda candle fetch failed for {instrument}: " f"HTTP {resp.status_code}: {resp.text[:300]}") - if attempt == 0: - logger.warning( - f"Oanda fetch for {instrument} got HTTP " - f"{resp.status_code}; retrying in " - f"{self.RETRY_DELAY_S}s") + logger.warning( + f"Oanda fetch for {instrument} attempt " + f"{attempt + 1}/{attempts} got HTTP {resp.status_code}") + resp = None continue break - else: + if resp is None: raise last_err if resp.status_code != 200: diff --git a/signal_aggregator.py b/signal_aggregator.py index 414a490..e98db43 100755 --- a/signal_aggregator.py +++ b/signal_aggregator.py @@ -305,7 +305,10 @@ class SignalAggregator: matches the context the signal just used; if it fails, the row is still written, just without the news JSON blobs. ``blocked`` marks weekend/day early returns: no news fetch, no snapshots — - a ~1 KB liveness row (or nothing, per signal_log.log_blocked_scans).""" + a ~1 KB liveness row (or nothing, per signal_log.log_blocked_scans). + A scan whose PRIMARY fetch failed is logged as ``scan_kind`` + ``failed`` (price 0, no signal): liveness evidence that readers + skip like ``blocked``, so it never pollutes NEUTRAL statistics.""" context = None if not blocked: try: @@ -313,7 +316,12 @@ class SignalAggregator: except Exception: logger.debug("signal log: news context unavailable", exc_info=True) - extra = {"scan_kind": "blocked" if blocked else "scan"} + kind = "blocked" if blocked else "scan" + if not blocked and dataframes is not None: + primary = dataframes.get(self.config.primary_timeframe) + if primary is None or primary.empty: + kind = "failed" + extra = {"scan_kind": kind} for key, fingerprint in ( ("config_fingerprint", self.gate_config_fingerprint), ("engine_fingerprint", self.engine_config_fingerprint)): diff --git a/signal_log.py b/signal_log.py index 171e2a9..183de23 100644 --- a/signal_log.py +++ b/signal_log.py @@ -53,8 +53,10 @@ or news_engine) so this module stays at the bottom of the import graph. XAU_CP acceptance removed from ``warnings_json``), ``bar_range_median`` (the value historically stored as ``spread`` — it is the median High-Low of the primary frame, NOT a broker spread), ``scan_kind`` - (``scan`` / ``blocked`` for weekend/day early returns, which carry no - news or candles) and ``config_fingerprint`` (hash of the gate config the + (``scan``; ``blocked`` for weekend/day early returns, which carry no + news or candles; ``failed`` when the primary frame could not be + fetched — price 0, no signal — readers skip it like ``blocked``) and + ``config_fingerprint`` (hash of the gate config the producing process ran with — surfaces config drift between processes). 2026-09 weight-research additions: diff --git a/test_all.py b/test_all.py index 401b93e..9a9835a 100644 --- a/test_all.py +++ b/test_all.py @@ -2384,6 +2384,55 @@ def _(): "snap-back bar after the spike must be kept (two-bar hole regression)" +@test("Outlier filter keeps a >10σ bar whose move PERSISTS (real news bar)") +def _(): + # 2026-09-07 audit: the 12:30 UTC NFP bar (-1.7%, ~23σ) was deleted + # from the signal-side M5/M15 data, hiding the day's most violent bar + # from ATR/exhaustion math. Three years of Oanda XAU bars: 155 such + # M5 bars, 7 snapped back. A level shift that holds is real price. + provider = TwelveDataProvider(PlatformConfig()) + df = make_ohlcv(n=300, seed=3) + k = 150 + jump_ts = df.index[k] + shift = df["Close"].iloc[k - 1] * 0.05 # ~+5% step on a ~0.1%-σ series + df.loc[df.index[k]:, ["Open", "High", "Low", "Close"]] += shift + cleaned = provider._validate_and_clean(df, "XAU/USD", Timeframe.H1) + assert jump_ts in cleaned.index, \ + "a 10σ move that the next close confirms must be KEPT" + assert len(cleaned) == len(df), "no bar may be dropped for a real move" + + +@test("Outlier filter keeps the NEWEST >10σ bar (no successor to judge it)") +def _(): + # Live at 12:35 on a release day the crash bar IS the last closed + # bar. Dropping it would make the scan run on pre-release data (the + # 2026-09-04 12:35 scan was a byte-identical repeat of 12:30). With + # no next close available the bar is treated as real; a true bad + # print on the newest bar is caught on the following scan. + provider = TwelveDataProvider(PlatformConfig()) + df = make_ohlcv(n=300, seed=3) + last_ts = df.index[-1] + df.loc[last_ts, ["Open", "High", "Low", "Close"]] = df["Close"].iloc[-2] * 1.05 + cleaned = provider._validate_and_clean(df, "XAU/USD", Timeframe.H1) + assert last_ts in cleaned.index, "newest 10σ bar must be kept" + + +@test("Outlier filter: partial snap-back (below the revert fraction) is a real move") +def _(): + provider = TwelveDataProvider(PlatformConfig()) + df = make_ohlcv(n=300, seed=3) + k = 150 + base = df["Close"].iloc[k - 1] + spike_ts = df.index[k] + # +5% spike, next close gives back only half — a violent real move, + # not a bad print (revert fraction 0.5 < OUTLIER_REVERT_FRACTION 0.8). + df.loc[spike_ts, ["Open", "High", "Low", "Close"]] = base * 1.05 + df.loc[df.index[k + 1]:, ["Open", "High", "Low", "Close"]] += base * 0.025 + cleaned = provider._validate_and_clean(df, "XAU/USD", Timeframe.H1) + assert spike_ts in cleaned.index, "half-retraced spike is real price" + assert provider.OUTLIER_REVERT_FRACTION == 0.8 + + @test("Cache: pre-boundary snapshot refetched; boundary-complete snapshot served") def _(): def bars_ending(last): @@ -3558,20 +3607,26 @@ def _(): f"Expected one retry sleep, got {slept}" -@test("providers.oanda: persistent 401 raises after retry; 4xx never retried") +@test("providers.oanda: persistent 401 walks the whole retry ladder then raises; 4xx never retried") def _(): orig_sleep = providers.oanda.time.sleep - providers.oanda.time.sleep = lambda s: None + slept = [] + providers.oanda.time.sleep = lambda s: slept.append(s) try: + ladder = providers.oanda.OandaDataProvider.RETRY_DELAYS_S provider = _make_oanda_provider([ - _oanda_resp(401, text='{"errorMessage":"Insufficient authorization"}'), - _oanda_resp(401, text='{"errorMessage":"Insufficient authorization"}'), + _oanda_resp(401, text='{"errorMessage":"Insufficient authorization"}') + for _ in range(1 + len(ladder)) ]) try: provider._fetch_oanda_candles("XAU/USD", Timeframe.H1, 300) assert False, "Persistent 401 should raise OandaDataError" except providers.oanda.OandaDataError as e: assert "401" in str(e) + assert slept == list(ladder), \ + f"every ladder delay must be slept before giving up, got {slept}" + assert ladder == (2.0, 5.0, 10.0), \ + "ladder is sized to outlast a 10-15 s top-of-hour provider blip" # Non-auth 4xx must fail immediately — only one response queued, # so a retry attempt would IndexError on the empty queue. @@ -3585,6 +3640,55 @@ def _(): providers.oanda.time.sleep = orig_sleep +@test("providers.oanda: a 5xx blip that clears on the third attempt recovers") +def _(): + # The 2026-09-07 signal-log audit: 11 boundary scans lost their + # primary frame because one 2 s retry could not outlast a ~12 s + # provider blip. Two failures then success must recover, having + # slept exactly the first two ladder steps. + orig_sleep = providers.oanda.time.sleep + slept = [] + providers.oanda.time.sleep = lambda s: slept.append(s) + try: + provider = _make_oanda_provider([ + _oanda_resp(502, text="bad gateway"), + _oanda_resp(503, text="unavailable"), + _oanda_resp(200, candles=[_OANDA_OK_CANDLE]), + ]) + df = provider._fetch_oanda_candles("XAU/USD", Timeframe.H1, 300) + finally: + providers.oanda.time.sleep = orig_sleep + assert len(df) == 1, "third attempt must return the frame" + assert slept == list( + providers.oanda.OandaDataProvider.RETRY_DELAYS_S[:2]), slept + + +@test("providers.oanda: retry budget stops the ladder before a delay would overrun it") +def _(): + # A scan fetches three frames; the whole ladder must stay bounded so + # an M5 scan finishes inside its bar. With the clock jumped past the + # budget after the first failure, no further attempt (and no sleep) + # may happen — the queue holds only ONE response, so a second + # attempt would IndexError. + orig_sleep = providers.oanda.time.sleep + orig_mono = providers.oanda.time.monotonic + slept = [] + ticks = iter([0.0, 10_000.0, 10_000.0, 10_000.0]) + providers.oanda.time.sleep = lambda s: slept.append(s) + providers.oanda.time.monotonic = lambda: next(ticks) + try: + provider = _make_oanda_provider([_oanda_resp(503, text="unavailable")]) + try: + provider._fetch_oanda_candles("XAU/USD", Timeframe.H1, 300) + assert False, "budget exhaustion must raise the last error" + except providers.oanda.OandaDataError as e: + assert "503" in str(e) + finally: + providers.oanda.time.sleep = orig_sleep + providers.oanda.time.monotonic = orig_mono + assert slept == [], "no sleep once the budget cannot fit the next delay" + + @test("providers.oanda: instrument name mapping") def _(): provider = providers.oanda.OandaDataProvider.__new__( @@ -7772,6 +7876,48 @@ def _(): os.remove(path) +@test("signal_log: a scan whose PRIMARY fetch fails is logged as scan_kind 'failed'") +def _(): + # 2026-09-07 audit: 11 rows in two days carried price 0 / score 0 / + # NEUTRAL F because the top-of-hour Oanda fetch failed; logged as + # ordinary scans they polluted NEUTRAL statistics. They are liveness + # evidence and readers skip them like ``blocked``. + import signal_log as slog + fd, path = tempfile.mkstemp(suffix=".db") + os.close(fd) + os.remove(path) + try: + os.environ["SIGNAL_LOG_PATH"] = path + os.environ.pop("SIGNAL_LOG_ENABLED", None) + cfg = PlatformConfig() + cfg.xau_strategy.weekend_filter = False # deterministic on any day + cfg.xau_strategy.allowed_weekdays = [] + agg = SignalAggregator(cfg) + agg._get_news_engine = lambda: _NoNews() + + def _boom(*a, **k): + raise RuntimeError("HTTP 503 at the boundary") + agg.data_provider.fetch_ohlcv = _boom + report = agg.analyze_pair(symbol="XAU/USD", log_source="test") + assert "Primary timeframe data unavailable" in report.warnings + import sqlite3 + conn = sqlite3.connect(path) + row = conn.execute( + "SELECT scan_kind, gate_reason, signal, quality FROM signals" + ).fetchone() + conn.close() + assert row is not None, "the failed scan must still leave a liveness row" + assert row[0] == "failed", f"scan_kind must be 'failed', got {row[0]!r}" + assert row[1] == "Primary timeframe data unavailable" + assert row[2] == "NEUTRAL" and row[3] == "F" + finally: + os.environ.pop("SIGNAL_LOG_PATH", None) + os.environ.pop("SIGNAL_LOG_ENABLED", None) + slog.reset_loggers_for_tests() + if os.path.exists(path): + os.remove(path) + + @test("signal_log parity: analyze_from_dataframes never logs") def _(): import signal_log as slog @@ -7985,6 +8131,18 @@ def _(): assert "config fingerprint" in d["diffs"] and "action" in d["diffs"] assert rep["rows_per_bar"]["telegram/15min"] == 1.5 assert rep["rows_per_bar"]["dashboard/15min"] == 1.0 + # A ``failed`` row (primary fetch lost, price 0) is liveness evidence, + # not a scan: it must not count as a source on its bar. + import dataclasses + failed = dataclasses.replace( + _asl_row(6, t0, bar_ts="2026-08-19T12:30:00+00:00", source="cli", + quality="F", setup=False, price=0.0, + action="NO TRADE — No clear signal"), + scan_kind="failed") + rep2 = asl.parity_report(rows + [failed]) + assert rep2["bars_with_2_sources"] == 2 + assert "cli/15min" not in rep2["rows_per_bar"] + assert len(rep2["divergences"]) == 1 @test("analyze_signal_log: load_rows sees WAL-resident rows and never writes")