From 42025da0b0ed75244652f38a54fd0d7a2cf49596 Mon Sep 17 00:00:00 2001 From: Harwinder Sekhon Date: Sun, 27 Sep 2026 21:02:42 -0400 Subject: [PATCH] Local news model: ML-NEWS XAU transformer scores headlines (news_model.py, mode primary) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The owner's DeBERTa-v3-small two-head ONNX model (golden set 72/78, ~72% agreement with DeepSeek on unseen days) replaces DeepSeek for headline scoring. DeepSeek keeps event direction, the summary and the dashboard endpoint. - news_model.py: vendored ML-NEWS runtime (lazy onnxruntime/tokenizers, so the bot starts without them) + load-once, never-raising singleton. - news_engine: _enrich_headline_scores mode switch (off/fallback/primary), _apply_model_sentiment (gold head only: xau_score + XAU entry, never USD; not-relevant shows 0.0; never touches _llm_sent_cache), _audit_with_deepseek (archive-only labels, deepseek_audit_per_day). - Owner decision: primary mode writes nothing to logs/newsdata.db except audit copies — the model's output is not training data. - app_config: news_model section (path/threads/mode/deepseek_audit_per_day) + NEWS_MODEL_MODE env override; validated at boot. - news_archive: model rows carry teacher_model xau_transformer@. - deploy/README-news-model.md (ops for bot + dashboard daemon), unit gets a commented NEWS_MODEL_MODE=off line; requirements + onnxruntime, tokenizers. - test_all.py Module 38 (15 tests, one against the real bundle). Suite 581. - CLAUDE.md, spec status note, handoff marked implemented. Books unaffected: news never enters signals; no signal file touched. --- CLAUDE.md | 27 +- NEWS_MODEL_HANDOFF_2026-09-27.md | 81 +++ app_config.py | 88 ++- config.json.example | 6 + deploy/README-dashboard.md | 4 +- deploy/README-news-model.md | 185 +++++ deploy/gold-dashboard.service | 4 + .../news_model_integration_spec_2026-09-27.md | 535 +++++++++++++++ news_archive.py | 20 +- news_engine.py | 203 +++++- news_model.py | 203 ++++++ requirements.txt | 4 + test_all.py | 640 +++++++++++++++++- 13 files changed, 1966 insertions(+), 34 deletions(-) create mode 100644 NEWS_MODEL_HANDOFF_2026-09-27.md create mode 100644 deploy/README-news-model.md create mode 100644 docs/news_model_integration_spec_2026-09-27.md create mode 100644 news_model.py diff --git a/CLAUDE.md b/CLAUDE.md index a7eb981..3acd699 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -23,7 +23,7 @@ cp config.json.example config.json # then fill i venv/bin/pip install cupy-cuda12x "nvidia-cuda-runtime-cu12==12.9.*" # optional, dev box: backtester GPU backend # (CuPy from pip needs that header wheel to compile kernels) venv/bin/python test_all.py # QA suite, no network; prints its own count - # (566 tests / 37 modules as of 2026-09-27) + # (581 tests / 38 modules as of 2026-09-27) venv/bin/python main.py --pair XAUUSD --timeframe H1 --detail # one signal report in the terminal # --json full dump, --panel adds the advisory LLM panel, # --timeframe M5 | M15 | H1 | H4 @@ -106,8 +106,9 @@ direction with the two confirmation timeframes. Agreement needs 2 of 3. Disagree penalty and blocks the top grades. **6. News context (warning only).** `news_engine.py` reads the Forex Factory calendar (MyFXBook -RSS as fallback) and four RSS feeds, scores headlines for gold, and can ask DeepSeek to improve -the scores. This produces warnings and a text summary. **By design it never changes a signal.** +RSS as fallback) and four RSS feeds and scores headlines for gold: the keyword scorer first, then +the local XAU transformer (`news_model.py`, mode `primary`) or DeepSeek (modes `off` / `fallback`) +overwrites those scores. This produces warnings and a text summary. **By design it never changes a signal.** The backtester runs with news off, so the live decision must be purely technical too. **7. Penalties, then the signal and the grade.** When the timeframes agree, confidence gains 10 @@ -270,11 +271,12 @@ Arrows show who imports whom (generated from the code on 2026-09-15). | File | Role | |---|---| -| `news_engine.py` | Calendar + RSS headline sentiment with gold-specific keywords; class-level caches; optional DeepSeek enrichment; display and warnings only | +| `news_engine.py` | Calendar + RSS headline sentiment with gold-specific keywords; class-level caches; headline scores upgraded by the local XAU model (`news_model.py`, mode `primary` live since 2026-09-27) or by DeepSeek (modes `off` / `fallback`, `_enrich_headline_scores`); DeepSeek keeps event direction and the summary; display and warnings only | | `llm_client.py` | Thin `requests` client for DeepSeek (`deepseek-v4-flash`, `thinking: disabled`, one attempt, UTC-day request cap). Every caller has a non-LLM fallback | -| `news_archive.py` | Durable headline corpus in `logs/newsdata.db` (one row per distinct headline, upgrade-only scores, provenance of the scorer). Its own database on purpose: a `signals.db` rotation in 2026-09 destroyed a 3-week corpus of 4,054 headlines and `logs/` is gitignored. Hooked at the end of `news_engine._fetch_news_sentiment_raw`, before the ±0.1 neutral-headline filter, so neutral headlines are kept for training; do not move the hook. Since 2026-09-17 the corpus is complete for a gold scorer: `xau_score` holds DeepSeek's gold score for every LLM-scored row (it used to survive only for gold-keyword headlines, so 930 of the first 1,546 rows had no gold label; older rows read NULL), headlines naming no tracked currency are scored and archived too (`currencies_json` `[]`; they are never returned to the platform), and every new headline goes to DeepSeek in the cycle it arrives, 25 per call | +| `news_archive.py` | Durable headline corpus in `logs/newsdata.db` (one row per distinct headline, upgrade-only scores, provenance of the scorer). Its own database on purpose: a `signals.db` rotation in 2026-09 destroyed a 3-week corpus of 4,054 headlines and `logs/` is gitignored. Hooked at the end of `news_engine._fetch_news_sentiment_raw`, before the ±0.1 neutral-headline filter, so neutral headlines are kept for training; do not move the hook. Since 2026-09-17 the corpus is complete for a gold scorer: `xau_score` holds DeepSeek's gold score for every LLM-scored row (it used to survive only for gold-keyword headlines, so 930 of the first 1,546 rows had no gold label; older rows read NULL), headlines naming no tracked currency are scored and archived too (`currencies_json` `[]`; they are never returned to the platform), and every new headline goes to DeepSeek in the cycle it arrives, 25 per call. **Frozen in `news_model.mode` `primary` (live since 2026-09-27)**: the local model's output is not training data, so nothing is archived except the optional DeepSeek audit copies (`deepseek_audit_per_day`, shipped 0); `off` / `fallback` archive as before, a `fallback` model row carrying `teacher_model` `xau_transformer@` | | `agent_panel.py` | Advisory persona panel (macro by default; technical / risk / contrarian and the chair available through `agent_panel.personas` for research). Attached post-hoc. v2 (2026-09-17, `PANEL_PROMPT_VERSION`): every persona sees a code-computed `context` block (prior move in ATR, distance from the 24 h extreme, candle size, stop/target in ATR, the class base rate from `agent_panel_base_rates.json`, event timing against the expected hold) and returns `p_win` plus one reason each way; stance is derived symmetrically (≥ 60 support, ≤ 40 oppose); the verdict is the equal-weight mean (quorum 3, `verdict_source: "mean"`, default). `macro` (stance verbatim) and `chair` (synthesizer + `_apply_oppose_guard`) remain for comparison. The prompt version is logged and hashed. Persona weights are not to be tuned before about 300 graded outcomes with a held-out half | | `violent_watch.py` | Display-only violent-move warning and heads-up. Attached post-hoc | +| `news_model.py` | The local headline scorer (2026-09-27): vendored CPU runtime of the owner's ML-NEWS XAU transformer (DeBERTa-v3-small, two heads: moves gold? / up or down?, ONNX; `onnxruntime` + `tokenizers` imported lazily inside `load_model`, so the bot starts without them) plus a load-once, never-raising singleton (`get_news_model`, `teacher_tag`, `warm_up_in_background`). Gold head only: `news_engine._apply_model_sentiment` writes `xau_score` and XAU's per-currency entry, never USD. Bundle (280 MB) lives outside the tree at `news_model.path`. Spec and deviations: `docs/news_model_integration_spec_2026-09-27.md`; ops: `deploy/README-news-model.md` | ### Logging and evaluation @@ -311,7 +313,7 @@ Arrows show who imports whom (generated from the code on 2026-09-15). | `app_config.py` | Loads `config.json` once per process; env var → config.json → refuse; validates keys | | `config.json.example` | The schema of `config.json` (which is gitignored) | | `deploy/README-dashboard.md`, `deploy/gold-dashboard.service` | Live-server checklist and systemd unit for the dashboard daemon | -| `test_all.py` | The QA suite (36 modules; Module 36 is the GPU backend parity module, Module 37 the CUDA decision-stage parity module) | +| `test_all.py` | The QA suite (38 modules; Module 36 is the GPU backend parity module, Module 37 the CUDA decision-stage parity module, Module 38 the local news model) | --- @@ -374,6 +376,7 @@ gates live in `config.py` and the pair factory, never in `config.json`. Blocks ( | `violent_watch` | `enabled`, `alert_line`, `headsup`, `max_headsup_per_day`, `headsup_min_level` (env `VIOLENT_WATCH_ENABLED`). Off by default | | `signal_log` | `enabled`, `path`, `log_blocked_scans` (env `SIGNAL_LOG_ENABLED`, `SIGNAL_LOG_PATH`, `SIGNAL_LOG_BLOCKED_SCANS`) | | `news_archive` | `enabled` (default true), `path` (env `NEWS_ARCHIVE_ENABLED`, `NEWS_ARCHIVE_PATH`) | +| `news_model` | `path` (bundle directory, required when mode ≠ off), `threads` (default 4), `mode` (`off` default = DeepSeek/keyword path; `fallback` = model only where DeepSeek failed; `primary` = model scores every headline, no DeepSeek headline calls), `deepseek_audit_per_day` (default 0; primary only, archive-only DeepSeek labels). Env `NEWS_MODEL_MODE` overrides `mode` per process (the dashboard unit). Live: `primary` on both processes | | `log_level` | Logging level | A legacy top-level `market_outlook` block is tolerated and ignored with one warning (the feature @@ -389,7 +392,9 @@ day, 30 kept), `logs/dashboard.log`. Read the databases with `mode=ro`, never `i ## 6. Testing - `venv/bin/python test_all.py` runs everything with mocked network. Expect - `RESULTS: 566 passed, 0 failed, 566 total` (2026-09-27). On a box with a visible CUDA device + `RESULTS: 581 passed, 0 failed, 581 total` (2026-09-27). Module 38's last test loads the + real news-model bundle when `NEWS_MODEL_DIR` (or the configured `news_model.path`) exists and + prints SKIP otherwise. On a box with a visible CUDA device Module 36 fails if the GPU backend falls back (the reason is in the message); with `FB2_BACKEND=cpu` or no CUDA the GPU-path tests print a notice and the NumPy twins are tested. Module 37 (the CUDA decision stage) adds about 85 s on the GPU because its references replay @@ -568,6 +573,13 @@ MD5s. Scorecards from it are in-book statistics, never decisions. Interior gaps in history are logged, never fatal, for the same reason. - Existing databases still contain an empty-going-forward `outlooks` table from the removed market-outlook feature. Nothing reads it. +- The local news model has a gold head only. `_apply_model_sentiment` may write `xau_score`, XAU's + per-currency entry and, for a headline naming no other currency, the overall score; USD and every + other currency keep the keyword value (the pair sentiment is XAU − USD). It never touches + `_llm_sent_cache`, and a not-relevant headline displays 0.0 while `xau_score` keeps the raw + number. In `primary` mode nothing is archived but the DeepSeek audit copies, and the dashboard + process never archives in any mode (its DB writes are off), so a retrain needs + `deepseek_audit_per_day > 0`, a spell in `fallback`, or a labelling run in ML-NEWS. - The news score is a trailing echo of the last day's price move (r ≈ 0.67 with the past day, ≈ 0.03 with the next). Both news gating flags stay off. - The level-breach rescan and the candle loop both skip the settlement-pause window @@ -632,6 +644,7 @@ One line per validated change or verdict, newest first. Full evidence in the nam | Date | Decision | Where the evidence is | |---|---|---| +| 2026-09-27 | Local news scorer, round two: the owner's ML-NEWS model is now a DeBERTa-v3-small two-head transformer (gold-relevant? then bullish/bearish), distilled from 17,245 DeepSeek gold labels (Sep 8–22 archive + a 2019–2026 Reuters/CNBC/MarketWatch crawl labelled by DeepSeek, 560 calls). Golden set 72/78 (DeepSeek 78, keywords 24, the 09-17 TF-IDF 39), agreement with DeepSeek on unseen live days ~72% (ceiling 90%), 19 ms/headline on one CPU thread as int8-embedding ONNX; passed the four promotion criteria fixed on 09-22. OWNER DECISIONS: it is to score headlines here in mode `primary` with NO DeepSeek for headline scoring (DeepSeek stays for event direction, summaries, the dashboard endpoint); model files live OUTSIDE this tree at a configured path (280 MB, never in git); shadow mode declined (the archive already allows offline replay). IMPLEMENTED the same day (branch `sql-integration`): `news_model.py` (vendored runtime + never-raising singleton, lazy `onnxruntime` / `tokenizers`), `news_model` config section + `NEWS_MODEL_MODE` env, `news_engine._enrich_headline_scores` / `_apply_model_sentiment` (gold-only rules A–F) / `_audit_with_deepseek`, archive provenance for `model` rows, `deploy/README-news-model.md`, Module 38 (15 tests, one against the real bundle: bullish / neutral labels, 7 ms per headline on the dev box). Two further owner decisions during implementation: **`primary` mode stops writing `newsdata.db`** (only the DeepSeek audit sample, shipped 0, is archived) and **both processes load the model**. Books unaffected (news never enters signals); no signal file touched, no parity event. ML-NEWS `news_transformer.py` received the same two `load_model` edits (commit `9496cd9` there) | `docs/news_model_integration_spec_2026-09-27.md`; `~/Development/ML-NEWS/eval/RESULTS_2026-09-27.md` | | 2026-09-27 | Single "conviction" number vs the two-number grade (owner question: one number that is easier to understand and to decide on). Pre-registered; switch `signals.grading_mode` (`pair` default = the shipped rule, byte-identical on both scripts, all five pins reproduced; `min` = min(\|score\|, 0.6 × conf); `product` = \|score\| × conf / 100; thresholds from the control's corner points, no tuning; mirrored in K2, twin MD5-equal to the reference on both new paths). FAILED on M5 and M15: K-MIN moves 43–54 trades for −1.2R / −2.9R (a, b, e / a, b, d); K-PROD −29R on M15 with PF 1.29 and W3 DD 28.9R, neutral on M5. H1 / H4 K-PROD +11.9R / +17.0R on W3 by taking 40–100 extra low-score trades (H1 PF 1.98 → 1.75, DD doubles): a hypothesis, not evidence. Nothing ships; the knob stays research-only. Code review (high) before the runs: 6 findings fixed, 1 recorded. SHIPPED the display half of the question (owner request the same day): alerts lead with the grade and print score / confidence as words (`main.plain_words`, same line on Telegram, `/status`, CLI and the dashboard card); books untouched. Suite 566 | `backtest/research/conviction_campaign_2026-09-27.md` | | 2026-09-26 | Quant review + live log 2026-09-04 → 09-25 (100 setups replayed on the scans' stored candles): stop-hunt claim REFUTED (71% of stop-outs ran ≥ 1R past the stop, same as the books); −25.1R of the −37.8R raw live book were consecutive-bar re-entries the backtester's cooldown never takes; the cooldown view sits at the 3rd–5th percentile of the books' three-week windows. SHIPPED: 5-bar alert cooldown; alert wording (trail first as the tested exit, TP1 a guide with its R, planned RR, validated-book line per class); M15 grade-A SELL block (`sell_min_quality` `A+, B`, PASSED a–f: W2 +154.15R/813 PF 1.53 vs +146.14/931 PF 1.43, OOS +68.5 vs +60.67, W3 DD +8%); 1R printed-target floor in every generator after the gates (`tp_print_floor_rr`, owner rule "TP1 at least 1:1"; label-only re-pin of all four timeframes). FAILED: M15 `min_rr_ratio` 1.0 (trade-identical, −2.0R); a pre-gate target floor (−24.6R). OWNER'S CALL: M5 `tp_floor_rr` 1.0 adds +48.9R on W2 with both halves up but max DD 10.1 → 15.9R (ladder 0.6–1.0 non-monotone); not shipped. Telegram / dashboard AI block rewritten for a new trader (owner rule): one plain headline against the usual odds, the chance "in 100", one reason for and one against, no verdict word or confidence figure; persona prompts ban trading jargon (`PANEL_PROMPT_VERSION` v2.1). Two delivery bugs from the DB checks fixed: the macro-source panel verdict never stored its probability (all v2 rows NULL); the level-breach rescan re-logged the 20:45 M15 bar three times every evening after the settlement pause (guard added to `_handle_breach`). Indicator grid and volume / daily-direction screens (Part C) not run | `backtest/research/rr_and_sell_campaign_2026-09-26.md` | | 2026-09-18 | Full-GPU research twin SHIPPED: `backtest/cudaBasedBacktesting.py` + `backtest/cuda_signal_kernels.py` (owner decisions: research only, `gpu_backtest.py` stays the base reference, every research flag supported, the one exemption from instructions item 6 recorded there). Proof: the five reference MD5s, four 6-month acceptance books (2026-03-18 → 2026-09-18, M5 and M15, $0.65 and $0.30), the M15 OOS `--export-features` CSV and `--record --record-bars` tables, and ten research-flag variants, all byte-identical to the reference script; Module 37 (16 tests) pins every stage against the live code. Books take 0.8–2.4 s instead of 13–115 s. Rule added: any signal change is a parity event for the twin | `backtest/research/cuda_backtester_2026-09-18.md` | diff --git a/NEWS_MODEL_HANDOFF_2026-09-27.md b/NEWS_MODEL_HANDOFF_2026-09-27.md new file mode 100644 index 0000000..f82862e --- /dev/null +++ b/NEWS_MODEL_HANDOFF_2026-09-27.md @@ -0,0 +1,81 @@ +# Handoff: the local news model and ForexBot2 (2026-09-27) + +**Status: IMPLEMENTED later the same day** (`news_model.py`, `news_model` config section, Module 38, +`deploy/README-news-model.md`). Two decisions were added during implementation: `primary` mode writes +nothing to `logs/newsdata.db` except the DeepSeek audit sample, and both processes load the model. The +authoritative record is the status note at the top of `docs/news_model_integration_spec_2026-09-27.md` +and the CLAUDE.md ledger row; this file is kept as the historical handoff and can be deleted. + +Read this before touching headline scoring. It records what exists, what the owner decided, and what +was left to build at the time of writing. + +## What exists + +The owner's news model lives in a separate project, `~/Development/ML-NEWS` +(git `ssh://git@192.168.0.11:222/harry/News-ML-Decoder.git`). On 2026-09-27 it became a +DeBERTa-v3-small two-head transformer, exported to ONNX for CPU: + +- Question 1: does this headline move the gold price? Question 2: if so, up or down? +- Trained to copy DeepSeek's gold score on 17,245 headlines: the Sep 8–22 `newsdata.db` archive plus a + 2019–2026 Reuters / CNBC / MarketWatch crawl that DeepSeek labelled (560 calls). +- Golden set 72/78. DeepSeek 78, the keyword scorer 24, the 2026-09-17 TF-IDF model 39. +- Agreement with DeepSeek on unseen live-feed days about 72% (DeepSeek agrees with itself about 90%). +- 19 ms per headline on one CPU thread, 9 ms on four. Needs only `onnxruntime`, `tokenizers`, `numpy`. +- Bundle: `ML-NEWS/models/xau_transformer/{model.onnx (271 MB), tokenizer.json, meta.json}`. +- Full results: `~/Development/ML-NEWS/eval/RESULTS_2026-09-27.md`. + +It passed the four promotion criteria fixed in advance on 2026-09-22 (golden ≥ 55/78, distractors ≥ 6/8, +unseen-days agreement ≥ 62.8%, latency ≤ 20 ms). This supersedes the 2026-09-17 verdict "a fallback, +not a replacement" in the CLAUDE.md ledger. + +## What the owner decided (2026-09-27) + +1. **Use it as the primary headline scorer, with no DeepSeek calls for headline scoring.** DeepSeek + stays for everything else: calendar-event direction (`_llm_event_direction`), the news summary, the + dashboard's on-demand summary endpoint. +2. **Model files live outside the ForexBot2 tree**, for example `/home/Forex/models/xau_transformer`, + copied by hand, path set in `config.json`. Never in git. ForexBot2 keeps deploying by `git pull`. +3. **No shadow mode.** The archive already stores every headline with its score, so offline replay + covers the comparison. +4. Accepted consequence: with DeepSeek off for headlines, the archive stops receiving fresh DeepSeek + labels, which is what ML-NEWS retrains on. An optional config knob (`deepseek_audit_per_day`, + default 0) can label a few headlines a day for the archive only. + +## What is left to build + +Everything. The specification is `docs/news_model_integration_spec_2026-09-27.md` in this repo +(identical to `ML-NEWS/docs/FOREXBOT2_INTEGRATION_SPEC.md`). In one screen: + +| File | Change | +|---|---| +| `requirements.txt` | add `onnxruntime`, `tokenizers` (no "torch"/"transformers" anywhere in the file; a test greps for them) | +| `news_model.py` (new) | vendored copy of ML-NEWS `news_transformer.py` + a load-once singleton that never raises; onnxruntime imported lazily | +| `app_config.py` | optional `news_model` section: `path`, `threads`, `mode` (`off`/`fallback`/`primary`), `deepseek_audit_per_day`; `NEWS_MODEL_MODE` env override | +| `news_engine.py` | `_apply_model_sentiment` + a mode switch at the call site (~line 945); model score goes into `xau_score` and the XAU entry only, never into USD | +| `news_archive.py` | store `teacher_model` for `model` rows (the `model` rank already exists) | +| `test_all.py` | 11 tests listed in the spec | +| `deploy/README-news-model.md` (new) | ops for both the bot and the dashboard daemon | +| `config.json` (live) | the new section | + +Rules that must survive any implementation: + +- The model has a gold head only. **Never write its score into a non-XAU currency**: + `get_pair_sentiment` computes `XAU_mean − USD_mean` and the signal aggregator acts on it. +- A headline the model calls not-relevant shows 0.0, whatever the raw score; the raw score still goes + to `xau_score` for training data. +- Never write model results into `_llm_sent_cache`; that cache decides what is re-sent to DeepSeek. +- If the model fails to load, `primary` falls back to the DeepSeek path for that cycle, not to keywords. +- Both the bot and `dashboard_publisher.py` build a `NewsEngine`; each process that loads the model + holds about 0.6 GB. Decide per process with the env override. +- News never enters signals. CLAUDE.md §7 applies unchanged; no book is affected by any of this. + +## How to verify when it is built + +`venv/bin/python test_all.py` green; one news cycle in `primary` mode archives only `score_source='model'` +rows with `xau_score` set and zero DeepSeek headline calls; renaming the model directory away still +starts both processes and yields `llm` rows; `mode="off"` reproduces today's rows exactly. + +## Re-verify before editing + +Every file:line number in the spec was checked on 2026-09-27 against `6af1237` and drifts with each +commit. Check them first. diff --git a/app_config.py b/app_config.py index 363bdaf..55a916e 100644 --- a/app_config.py +++ b/app_config.py @@ -90,7 +90,7 @@ _KNOWN_TOP = {"telegram", "data_providers", "scanning", "pairs", "log_level", # config.json files still carry the block and must keep # loading; load_app_config logs one warning when it is there. "market_outlook", - "violent_watch", "signal_log", "news_archive"} + "violent_watch", "signal_log", "news_archive", "news_model"} _KNOWN_OANDA = { # "account_id" is TOLERATED but ignored: it belonged to the auto- # execution daemon removed 2026-09-03 (last commit 31002cb). The candle @@ -111,6 +111,8 @@ _KNOWN_VIOLENT_WATCH = {"enabled", "alert_line", "headsup", "max_headsup_per_day", "headsup_min_level"} _KNOWN_SIGNAL_LOG = {"enabled", "path", "log_blocked_scans"} _KNOWN_NEWS_ARCHIVE = {"enabled", "path"} +_KNOWN_NEWS_MODEL = {"path", "threads", "mode", "deepseek_audit_per_day"} +_NEWS_MODEL_MODES = ("off", "fallback", "primary") _KNOWN_TELEGRAM = {"token", "chat_id", "channel_id", "alert_min_quality"} _KNOWN_SCANNING = {"pairs", "timeframes"} _KNOWN_PAIR_KEYS = {"blocked_hours_utc"} @@ -150,6 +152,7 @@ class AppConfig: default_factory=lambda: dict(_VIOLENT_WATCH_DEFAULTS)) signal_log_settings: Dict[str, Any] = field(default_factory=dict) news_archive_settings: Dict[str, Any] = field(default_factory=dict) + news_model_settings: Dict[str, Any] = field(default_factory=dict) log_level: str = "INFO" @@ -405,6 +408,30 @@ def load_app_config(path: str = CONFIG_PATH) -> AppConfig: "config.json: news_archive.path must be a non-empty string") app.news_archive_settings = dict(narch) + nm = raw.get("news_model", {}) + _check_keys(nm, _KNOWN_NEWS_MODEL, '"news_model"') + if "mode" in nm and nm["mode"] not in _NEWS_MODEL_MODES: + raise AppConfigError( + 'config.json: news_model.mode must be "off", "fallback" or "primary"') + if "threads" in nm and (not isinstance(nm["threads"], int) + or isinstance(nm["threads"], bool) + or nm["threads"] < 1): + raise AppConfigError( + "config.json: news_model.threads must be a positive integer") + if "deepseek_audit_per_day" in nm and ( + not isinstance(nm["deepseek_audit_per_day"], int) + or isinstance(nm["deepseek_audit_per_day"], bool) + or nm["deepseek_audit_per_day"] < 0): + raise AppConfigError( + "config.json: news_model.deepseek_audit_per_day must be a " + "non-negative integer") + if nm.get("mode", "off") != "off" and ( + not isinstance(nm.get("path"), str) or not nm.get("path")): + raise AppConfigError( + "config.json: news_model.path is required when news_model.mode " + "is not off") + app.news_model_settings = dict(nm) + level = raw.get("log_level", app.log_level) if level not in _VALID_LOG_LEVELS: raise AppConfigError( @@ -569,6 +596,65 @@ def signal_log_path(app: Optional[AppConfig] = None) -> str: return path +# ── Local news model (env → config.json → default) ─────────────────── +# +# The owner's ML-NEWS XAU transformer scores headlines locally +# (news_model.py). "off" is today's DeepSeek/keyword path; "fallback" runs +# the model only on rows DeepSeek left keyword-scored; "primary" scores +# every headline with the model and never calls DeepSeek for headlines +# (except the optional audit sample). NEWS_MODEL_MODE lets one process +# (the dashboard daemon's systemd unit) pick its own mode: each process +# that loads the model holds ~0.6 GB. + +def news_model_mode(app: Optional[AppConfig] = None) -> str: + env = os.environ.get("NEWS_MODEL_MODE") + if env: + mode = env.strip().lower() + if mode in _NEWS_MODEL_MODES: + return mode + logger.warning( + 'NEWS_MODEL_MODE=%r is not one of %s; using config.json', + env, _NEWS_MODEL_MODES) + app = app or get_app_config() + mode = app.news_model_settings.get("mode", "off") + return mode if mode in _NEWS_MODEL_MODES else "off" + + +def news_model_path(app: Optional[AppConfig] = None) -> Optional[str]: + """Absolute path to the bundle directory (model.onnx, tokenizer.json, + meta.json), or None when not configured. A relative value resolves + against the repo root like news_archive_path.""" + app = app or get_app_config() + path = app.news_model_settings.get("path") + if not isinstance(path, str) or not path: + return None + if not os.path.isabs(path): + path = os.path.join(os.path.dirname(os.path.abspath(__file__)), path) + return path + + +def news_model_threads(app: Optional[AppConfig] = None) -> int: + """ONNX intra-op threads (measured 2026-09-27: 1 → 19 ms per headline, + 4 → 9 ms, 16 → 13 ms).""" + app = app or get_app_config() + try: + return max(1, int(app.news_model_settings.get("threads", 4))) + except (TypeError, ValueError): + return 4 + + +def news_model_audit_per_day(app: Optional[AppConfig] = None) -> int: + """primary mode only: headlines per UTC day also sent to DeepSeek so + the archive keeps receiving fresh training labels. Archive-only, never + displayed. 0 = no DeepSeek headline calls at all (owner decision + 2026-09-27).""" + app = app or get_app_config() + try: + return max(0, int(app.news_model_settings.get("deepseek_audit_per_day", 0))) + except (TypeError, ValueError): + return 0 + + # ── News-archive resolution helpers (env → config.json → default) ──── def news_archive_enabled(app: Optional[AppConfig] = None) -> bool: diff --git a/config.json.example b/config.json.example index 0612e29..a831a20 100644 --- a/config.json.example +++ b/config.json.example @@ -65,5 +65,11 @@ "enabled": true, "path": "logs/newsdata.db" }, + "news_model": { + "path": "/home/Forex/models/xau_transformer", + "threads": 4, + "mode": "off", + "deepseek_audit_per_day": 0 + }, "log_level": "INFO" } diff --git a/deploy/README-dashboard.md b/deploy/README-dashboard.md index 6fc37a9..7ff76a1 100644 --- a/deploy/README-dashboard.md +++ b/deploy/README-dashboard.md @@ -19,7 +19,9 @@ git pull venv/bin/python test_all.py # optional sanity: suite must be green ``` -No new Python dependencies are required. +Two optional Python packages (`onnxruntime`, `tokenizers`) arrive with the local news model; +`venv/bin/pip install -r requirements.txt` after the pull. See `deploy/README-news-model.md`, +which also says when to keep this daemon on DeepSeek (`NEWS_MODEL_MODE=off` in the unit). ## 2. Make the web folder writable diff --git a/deploy/README-news-model.md b/deploy/README-news-model.md new file mode 100644 index 0000000..6bc9df6 --- /dev/null +++ b/deploy/README-news-model.md @@ -0,0 +1,185 @@ +# Deploying the local news model (live server) + +Since 2026-09-27 ForexBot2 can score news headlines with the owner's ML-NEWS gold transformer +instead of DeepSeek. The model runs on the CPU inside the bot process (and inside the dashboard +daemon), needs two extra Python packages and a 280 MB model folder that lives **outside** the git +tree. DeepSeek keeps its other jobs: the direction of a calendar surprise, the news summary and the +dashboard's on-demand summary endpoint. + +What the model is, in one paragraph: a DeBERTa-v3-small transformer with two heads (does this +headline move gold? if so, up or down?) exported to ONNX, trained to copy DeepSeek's gold score on +17,245 headlines. On the 78-headline golden set it scores 72 (DeepSeek 78, the old keyword scorer +24); on unseen live days it agrees with DeepSeek about 72% of the time (DeepSeek agrees with itself +about 90%). It scores one headline in about 10–20 ms on one CPU thread. It is **gold only**: every +other currency's headline sentiment stays with the keyword scorer in `primary` mode. + +Owner decisions this deployment implements: + +- Mode `primary`: the model scores every headline; **no DeepSeek calls for headline scoring**. +- **The news archive stops growing.** In `primary` mode the model's own output is not archived (it is + not training data). Only the optional DeepSeek audit sample (`deepseek_audit_per_day`, shipped 0) + writes rows. `logs/newsdata.db` stays as it is; nothing is migrated or deleted. +- Both processes (bot and dashboard daemon) load the model so Telegram and the dashboard show the same + headline scores. Each ONNX session holds about 0.6 GB of RAM. + +## 0. Check memory + +```bash +free -m +``` + +Each process that loads the model needs about 0.7 GB of headroom; two processes about 1.5 GB. If the +box cannot spare that, keep the model on the bot only: uncomment the `Environment=NEWS_MODEL_MODE=off` +line in `deploy/gold-dashboard.service` (step 5). The dashboard then keeps scoring its headlines with +DeepSeek and its scores may differ from the bot's. The dashboard process never writes the archive in +any mode, so this choice does not affect training data. + +## 1. Code and dependencies + +```bash +cd /home/Forex/ForexBot2 +git pull +venv/bin/pip install -r requirements.txt # adds onnxruntime and tokenizers +``` + +`requirements.txt` must never contain the training stack (no torch, no transformers); the test suite +checks that. The live box runs Python 3.9, where pip resolves onnxruntime 1.19.2 and tokenizers +0.22.2; the bundle of 2026-09-27 scores identically there and on the dev box (1.30.0 / 0.23.2). + +## 2. Copy the model files (outside git) + +On the machine that holds ML-NEWS (`~/Development/ML-NEWS/models/xau_transformer/`), copy exactly +three files. Skip the `torch/` and `torch_final/` subfolders; they are training checkpoints. + +```bash +ssh Forex@LIVE 'mkdir -p /home/Forex/models/xau_transformer' +rsync -av --progress \ + ~/Development/ML-NEWS/models/xau_transformer/{model.onnx,tokenizer.json,meta.json} \ + Forex@LIVE:/home/Forex/models/xau_transformer/ +``` + +Verify on the live box: + +```bash +sha256sum /home/Forex/models/xau_transformer/model.onnx +# bundle of 2026-09-27: 9e7a64d3ae1df335059ce829cafcf6e4544b85a4f72cc88923c2e42c48e975a0 +sha256sum /home/Forex/models/xau_transformer/tokenizer.json +# bundle of 2026-09-27: 045f93f80eb9ad18e559647563fb827dea59d93258fdb8b922de82bc53666d16 +python3 -c "import json; print(json.load(open('/home/Forex/models/xau_transformer/meta.json'))['trained_at'])" +# 2026-09-27T23:15:30.403442+00:00 +``` + +The files should be owned by (or readable by) the account the bot runs as. Read-only is enough. + +## 3. Configure + +Add one block to `/home/Forex/ForexBot2/config.json` (any position; `config.json.example` shows it). +Leave the `deepseek` block as it is. + +```json +"news_model": { + "path": "/home/Forex/models/xau_transformer", + "threads": 2, + "mode": "primary", + "deepseek_audit_per_day": 0 +} +``` + +(`threads` is clamped to the core count; the live box has 2 cores, hence 2. Measured there: load +0.8 s, 16 ms per headline, 633 MB resident for the session.) + +| Key | Meaning | +|---|---| +| `path` | folder with `model.onnx`, `tokenizer.json`, `meta.json`; required when `mode` is not `off` | +| `threads` | ONNX threads per process. Measured: 1 thread 19 ms per headline, 4 threads 9 ms, 16 threads 13 ms. 4 is right for a 4–8 core box | +| `mode` | `off` = today's DeepSeek/keyword path, nothing loads. `fallback` = DeepSeek first, the model only for headlines DeepSeek failed on. `primary` = the model scores everything, DeepSeek never sees a headline (except the audit sample) | +| `deepseek_audit_per_day` | `primary` only: headlines per UTC day also sent to DeepSeek and archived as `llm` rows (one call of up to 25 per cycle at most). They are training labels for the next ML-NEWS retrain; the bot never displays them. 0 = no DeepSeek headline calls at all | + +The environment variable `NEWS_MODEL_MODE=off|fallback|primary` overrides `mode` for one process. It +exists so the dashboard daemon's systemd unit can pick its own mode. + +A config error (unknown key, bad mode, missing path) stops the bot at boot with a clear message, like +every other `config.json` mistake. + +## 4. Smoke test without starting the bot + +```bash +cd /home/Forex/ForexBot2 +venv/bin/python -c " +import news_model as m +b = m.get_news_model() +print(m.score_news(b, 'Gold jumps as dollar slides')) +print(m.score_news(b, 'Red Lobster endless shrimp losses'))" +``` + +Expect a log line `news model loaded from /home/Forex/models/xau_transformer (trained_at 2026-09-27…, +threshold 0.2, 4 threads)`, then `label: bullish` for the first headline and `label: neutral` for the +second. If instead you see `news model unavailable, DeepSeek/keyword path stays: …`, the message names +the cause (missing file, missing package, bad JSON). The bot would still start in that state and +would score with DeepSeek for the whole process lifetime; fix the cause and restart. + +Then run the suite. With the model folder configured, the integration test loads the real bundle: + +```bash +venv/bin/python test_all.py # expect RESULTS: … passed, 0 failed +``` + +## 5. Restart both processes + +Bot: restart it the way it is run on this box (nohup / screen / its unit). Dashboard: + +```bash +sudo systemctl restart gold-dashboard +``` + +If the dashboard must not load the model (step 0), first uncomment the +`Environment=NEWS_MODEL_MODE=off` line in `deploy/gold-dashboard.service`, then +`sudo cp deploy/gold-dashboard.service /etc/systemd/system/ && sudo systemctl daemon-reload` before +the restart. + +## 6. What to check afterwards + +- **Logs** (`logs/telegram.log`, `logs/dashboard.log`): exactly one `news model loaded` line per + process, no `news model unavailable` line. If the level is DEBUG you also see + `news archive: primary mode, 0 DeepSeek audit rows stored, N model/keyword rows not archived` every + news cycle. +- **`top`**: each process about 0.6 GB more resident memory than before; CPU near 0% between scans + (thread spinning is disabled in the runtime). +- **DeepSeek spend**: the daily request count no longer moves with the headline count; only the event + direction, the summary and the panel use it. +- **Archive frozen** (the intended state with audit 0): + + ```bash + sqlite3 -readonly logs/newsdata.db "SELECT MAX(first_seen), COUNT(*) FROM headlines;" + ``` + + The newest `first_seen` stops advancing after the restart. With `deepseek_audit_per_day` > 0 you + see at most that many new rows a day, all `score_source = 'llm'`. +- **Telegram / dashboard**: headline scores still appear; the wording and layout are unchanged, only + the numbers now come from the model. A headline the model calls irrelevant to gold shows as neutral. + +## 7. Rollback + +Set `"mode": "fallback"` (DeepSeek first, the model only fills gaps and the archive grows again) or +`"mode": "off"` (exactly the pre-2026-09-27 behaviour) in `config.json`, or export +`NEWS_MODEL_MODE=off` for one process, and restart. No data migration in either direction. + +## 8. Retraining cadence + +When ML-NEWS ships a new bundle, copy the three files over the old ones (step 2), check the new +`trained_at`, and restart both processes. Rows written in `fallback` mode carry +`teacher_model = xau_transformer@`, so the archive shows which bundle scored them. + +With `deepseek_audit_per_day` at 0 the archive receives no new DeepSeek labels, so the next retrain +needs one of: a positive audit value for a few weeks, a spell in `fallback` mode, or a separate +labelling run in ML-NEWS on freshly crawled headlines (`eval/eval_archive.py --scorer transformer` on a +copy of the live archive gives the model-versus-DeepSeek agreement on whatever labelled rows exist). + +## Known limits + +- The model copies DeepSeek's opinion; it has not learned from price moves. DeepSeek itself is only + about 90% consistent with itself, and the model agrees with it about 72% of the time. +- Gold only. In `primary` mode every other currency's headline sentiment is the keyword scorer (24 of + 78 on the golden set), which is what those currencies got whenever DeepSeek was unavailable before. +- News never enters signals (CLAUDE.md section 7). No trade, grade or backtest book changes with any of + this; the model only changes the headline lines people read. diff --git a/deploy/gold-dashboard.service b/deploy/gold-dashboard.service index 320f023..5601e3b 100644 --- a/deploy/gold-dashboard.service +++ b/deploy/gold-dashboard.service @@ -15,6 +15,10 @@ Type=simple User=Forex WorkingDirectory=/home/Forex/ForexBot2 Environment=DASHBOARD_OUT_DIR=/var/www/trade +# The local news model (deploy/README-news-model.md) loads in this process +# too (~0.6 GB RSS) so the dashboard shows the bot's headline scores. +# Uncomment to keep this daemon on DeepSeek when memory is short: +#Environment=NEWS_MODEL_MODE=off ExecStart=/home/Forex/ForexBot2/venv/bin/python dashboard_publisher.py Restart=always RestartSec=30 diff --git a/docs/news_model_integration_spec_2026-09-27.md b/docs/news_model_integration_spec_2026-09-27.md new file mode 100644 index 0000000..1eefa5f --- /dev/null +++ b/docs/news_model_integration_spec_2026-09-27.md @@ -0,0 +1,535 @@ +# Specification: run the ML-NEWS XAU transformer inside ForexBot2 + +Status: **IMPLEMENTED 2026-09-27** (ForexBot2 branch `sql-integration`, on top of `6af1237`; suite +Module 38). The text below is the specification as written before the implementation; file:line +numbers refer to `6af1237` and have drifted. Where the implementation differs, this list wins: + +- **`primary` mode does not archive the cycle's rows.** Owner decision during implementation: once the + local model scores, `logs/newsdata.db` stops growing, because the model's own output is not training + data. Only the DeepSeek audit copies (`deepseek_audit_per_day`, shipped 0) are written. `off` and + `fallback` archive as before (a `fallback` model row carries `model_teacher`). §5.4 and §9 read + accordingly. +- `_audit_with_deepseek` **returns** the `llm` copies; there is no `_pending_audit_rows` attribute. + The mode switch of §5.2 lives in `NewsEngine._enrich_headline_scores(scored, mode)`, which returns + those copies, so the call site and the tests share one function. +- `config.json.example` ships `"mode": "off"`; the live server sets `primary` + (`deploy/README-news-model.md`). +- §8 step 0 was wrong: the dashboard daemon never writes the archive in any mode + (`set_db_writes_for_process(False)`), so its DeepSeek rows are not "free audit labels". Both + processes load the model (owner decision); `NEWS_MODEL_MODE=off` stays as the memory fallback. +- `warm_up_in_background()` is implemented (module-level once flag; a `NewsEngine` is built per scan). +- `news_model.get_news_model()` also treats a missing `path` as a load failure (one warning). +- `requirements.txt` ranges: `onnxruntime>=1.19,<2`, `tokenizers>=0.22,<1`. The live box runs + Python 3.9, where only 1.19.2 / 0.22.2 install; verified on 2026-09-28 to score the bundle + identically to the dev box's 1.30.0 / 0.23.2 (golden pair, four probe headlines). +- ML-NEWS `news_transformer.py` received the §3.4 edits in the same session; the vendored copy differs + only in `load_model` having no default directory. + +Repos: +- **ML-NEWS** `~/Development/ML-NEWS` (git `ssh://git@192.168.0.11:222/harry/News-ML-Decoder.git`). + Produces the model. Nothing in ForexBot2 imports from it. +- **ForexBot2** `~/Development/ForexBot2`, live at `/home/Forex/ForexBot2`, deployed by `git pull`. + Consumes the model. + +--- + +## 1. Context and goal + +### 1.1 What the model is +`ML-NEWS/models/xau_transformer/` holds a fine-tuned `microsoft/deberta-v3-small` with two heads, +exported to ONNX: + +| File | Size | Purpose | +|---|---|---| +| `model.onnx` | 271 MB | encoder + heads; int8 embedding table, fp32 compute | +| `tokenizer.json` | 8 MB | HF `tokenizers` fast tokenizer | +| `meta.json` | 4 KB | thresholds, `trained_at`, `relevance_threshold`, `export_checks`, training stats | + +Inputs: one headline, ≤ 64 tokens. Outputs per headline (from `news_transformer.score_batch`): + +```python +{"headline": str, "p_relevant": float, "p_bullish": float, + "relevant": bool, # p_relevant >= meta["relevance_threshold"] (0.2 for the shipped bundle) + "direction": "bullish"|"bearish", + "label": "bullish"|"bearish"|"neutral", # neutral when not relevant + "score": float} # p_relevant * (2*p_bullish - 1), in [-1, 1] +``` + +`score` is on the same scale as DeepSeek's `xau` score that ForexBot2 already stores in +`xau_score`. The model was trained to copy DeepSeek (`deepseek-v4-flash`, prompt sha1 `d32e04c6…`, the +same prompt as `news_engine._LLM_SENTIMENT_PROMPT_SHA1`) on 17,245 headlines. Quality on +2026-09-27: golden set 72/78 (DeepSeek 78, keyword scorer 24), agreement with DeepSeek on unseen live +days ~72%, CPU latency 19 ms/headline on 1 thread, 9 ms on 4. Full numbers: `ML-NEWS/eval/RESULTS_2026-09-27.md`. + +**The model scores gold only.** It has no head for USD, EUR or anything else. This drives most of +the rules in §5. + +### 1.2 How ForexBot2 scores headlines today +`news_engine.py`, inside `NewsEngine._fetch_news_sentiment_raw` (called via +`_cached_fetch("_headlines_cache", …)` at :822, which holds `NewsEngine._cache_lock`, :629-647): + +1. RSS items become `NewsSentiment` objects (:891-910) with the **keyword scorer** filling + `sentiment_score` and `sentiment_by_currency[cur]` for each currency in `currencies_mentioned`, + `score_source="keyword"`, `xau_score=None`. +2. `scored = headlines + unmatched` (:944); `self._apply_llm_sentiment(scored)` (:945, in try/except). + `_apply_llm_sentiment` (:1032-1065) sends headlines not in `NewsEngine._llm_sent_cache` to DeepSeek + in batches of `_LLM_BATCH_CAP = 25` via `self._llm_headline_scores(texts)` (:1009) and, on success, + overwrites `sentiment_score`, `sentiment_by_currency` (XAU gets the gold score, every other currency + the overall score), sets `xau_score`, `score_source="llm"`. A failed batch stops sending; those + headlines stay keyword-scored and **stay out of the cache**, so they are re-sent next window. +3. `news_archive.archive_headlines(scored, teacher_model=…, teacher_prompt_sha1=…)` (:958-964). + The archive UPSERT (news_archive.py:203-260) keeps the higher-ranked source: + `_SOURCE_RANK = {"unknown": 0, "keyword": 1, "model": 2, "llm": 3}` (:107). `"model"` is already + reserved for "a trained local artifact" (news_engine.py:223-227). `build_row` (:323-354) stores + `teacher_model`/`teacher_prompt_sha1` only when `source == "llm"`. +4. Downstream consumers of the scores: `get_fundamental_context` (drops |score| < + `NEUTRAL_HEADLINE_THRESHOLD = 0.1`, :256), `get_pair_sentiment` (:553-557, computes + `XAU_mean − USD_mean`), `signal_aggregator.py:1677/1723` (acts on that when + `sentiment_affects_signals`), `agent_panel.py`, the dashboard. + +Two processes run this code: the bot (`venv/bin/python telegram_bot.py`, CLAUDE.md:30) and the +dashboard daemon (`dashboard_publisher.py:401-403` builds its own `NewsEngine`, `:503` calls +`get_fundamental_context()`; unit `deploy/gold-dashboard.service`). They share `config.json` and the +archive DB. + +### 1.3 Decisions taken by the owner (2026-09-27) +- **Mode `primary`, no DeepSeek for headline scoring.** The model scores every headline. DeepSeek stays + for everything else it does (`_llm_event_direction` for calendar surprises, the news summary, the + dashboard's on-demand `news_summary.php`). +- **Model files live outside the ForexBot2 tree**, path in `config.json`. +- Accepted consequence: the archive stops receiving fresh DeepSeek labels, so the ML-NEWS weekly + retrain and the "replay and compare" check lose their data source. Mitigation kept optional: + `deepseek_audit_per_day` (§4) labels a few headlines a day for the archive only. Default 0. + +### 1.4 Non-goals +- Removing the DeepSeek client or its non-headline uses. +- Scoring non-gold currencies with the model. +- Changing ML-NEWS training. (The one ML-NEWS change is §3.4, a runtime option.) +- Training from price moves. + +--- + +## 2. Target layout on the live server + +``` +/home/Forex/ForexBot2/ # git pull as today + news_model.py # NEW, vendored runtime (§3) + requirements.txt # + onnxruntime, tokenizers (§3.1) + app_config.py, news_engine.py, news_archive.py, test_all.py # edited (§4, §5, §6, §7) + config.json # + "news_model" section (§4) + deploy/README-news-model.md # NEW ops checklist (§8) + deploy/gold-dashboard.service # maybe + Environment=NEWS_MODEL_MODE=off (§8) +/home/Forex/models/xau_transformer/ # NEW, outside git, copied by hand + model.onnx tokenizer.json meta.json +``` + +Memory: each process that loads the model holds ~0.5–0.7 GB RSS for the ONNX session. Two processes +→ ~1.5 GB. `free -m` before deciding whether the dashboard daemon also runs it (§8 step 0). + +--- + +## 3. `news_model.py` (new, ForexBot2) + +### 3.1 Dependencies +Add to `requirements.txt`: +``` +onnxruntime>=1.30,<2 +tokenizers>=0.23,<1 +``` +Pin to the versions that verified the bundle in ML-NEWS `.venv` (check `pip show` there before +writing). `tokenizer.json` compatibility across older `tokenizers` releases is not guaranteed. +**Do not write the words "torch" or "transformers" anywhere in requirements.txt**, not even in a +comment: `test_all.py:11437-11462` greps the file for them and fails. + +### 3.2 Source +Copy `ML-NEWS/news_transformer.py` → `ForexBot2/news_model.py`. Keep verbatim: `clean_headline`, +`_sigmoid`, `load_model`, `score_batch`, `score_news`, constants `MAX_LENGTH = 64`, +`RELEVANT_MIN = 0.25`, `RELEVANT_MAX_IRRELEVANT = 0.10`. Drop the CLI `main` or keep it, either is fine. +Add a docstring line: `Vendored from ML-NEWS news_transformer.py at commit ; bundle trained_at +.` + +**Imports.** `onnxruntime` and `tokenizers` must stay imported *inside* `load_model` (as in +news_transformer.py:58-59). Module top imports only `numpy`, `json`, `logging`, `os`, `pathlib`, +`threading`. Reason: `news_engine.py` will `import news_model` at its top; if onnxruntime is missing on +a box, the bot must still start. + +### 3.3 Singleton +```python +_lock = threading.Lock() +_bundle: Optional[dict] = None +_failed: bool = False # tried and failed; don't retry, don't re-log + +def get_news_model() -> Optional[dict]: + """The loaded bundle, or None. Loads once per process. Never raises.""" + global _bundle, _failed + if _bundle is not None or _failed: + return _bundle + with _lock: + if _bundle is not None or _failed: + return _bundle + try: + import app_config + path = app_config.news_model_path() + threads = app_config.news_model_threads() + b = load_model(path, threads=threads) + logger.info("news model loaded from %s (trained_at %s, threshold %s, %d threads)", + path, b["meta"].get("trained_at"), b["meta"].get("relevance_threshold"), threads) + _bundle = b + except Exception as e: # ImportError, FileNotFoundError, bad JSON, ORT errors + logger.warning("news model unavailable, DeepSeek/keyword path stays: %s: %s", + type(e).__name__, e) + _failed = True + return _bundle + +def reset_for_tests() -> None: # clears _bundle/_failed; used by test_all.py +``` + +Optional: `warm_up_in_background()` that calls `get_news_model()` in a daemon thread at +`NewsEngine.__init__`, so the first news cycle isn't blocked by the ~3–5 s session creation under +`_cache_lock`. Nice to have, not required. + +### 3.4 Two changes inside `load_model` (also apply to ML-NEWS `news_transformer.py` so the copies stay identical) +```python +options = ort.SessionOptions() +if threads: + options.intra_op_num_threads = max(1, min(int(threads), os.cpu_count() or 1)) + options.inter_op_num_threads = 1 +# ORT intra-op threads spin-wait after each run by default; in a mostly idle +# bot process that is 4 cores at 100% between cycles. +options.add_session_config_entry("session.intra_op.allow_spinning", "0") +``` + +### 3.5 Contract `score_batch(bundle, headlines: list[str]) -> list[dict]` +Unchanged from ML-NEWS. Empty or whitespace-only headline raises `ValueError` from `clean_headline`; +callers pass already-non-empty `NewsSentiment.headline`, but §5 still wraps per-batch in try/except. +Batch size: `_apply_model_sentiment` sends all of a cycle's headlines in one call (typically 30–150); +`score_batch` pads to the longest in the batch, ≤ 64 tokens, so one call of 150 is ~1 s on 4 threads. + +--- + +## 4. Configuration + +### 4.1 `config.json` section (add to `config.json.example` too) +```json +"news_model": { + "path": "/home/Forex/models/xau_transformer", + "threads": 4, + "mode": "primary", + "deepseek_audit_per_day": 0 +} +``` + +| Key | Type | Default if absent | Meaning | +|---|---|---|---| +| `path` | str | required when mode ≠ off | directory with the three bundle files | +| `threads` | int ≥ 1 | 4 | ORT intra-op threads (measured: 1→19 ms, 4→9 ms, 16→13 ms) | +| `mode` | `"off"`/`"fallback"`/`"primary"` | `"off"` | see §5.2 | +| `deepseek_audit_per_day` | int ≥ 0 | 0 | primary mode only: headlines/day to also send to DeepSeek, archive-only | + +Whole section absent → `mode="off"` → today's behaviour exactly. + +Environment override: `NEWS_MODEL_MODE=off|fallback|primary` wins over config.json. Purpose: the +dashboard daemon's systemd unit can turn the model off for that process only. Pattern to copy: +`NEWS_ARCHIVE_ENABLED` in `app_config.py:574-583`. + +### 4.2 `app_config.py` edits (follow the `news_archive` section, :397-406, accessors :574/:588) +1. `_KNOWN_TOP` (:86-92): add `"news_model"`. **Required**; `_check_keys(raw, _KNOWN_TOP, "top level")` + at :181 makes boot fail on an unknown top-level key. +2. `_KNOWN_NEWS_MODEL = {"path", "threads", "mode", "deepseek_audit_per_day"}` next to + `_KNOWN_NEWS_ARCHIVE` (:113). +3. `AppConfig` dataclass (~:147): `news_model_settings: Dict[str, Any] = field(default_factory=dict)`. +4. In `load_app_config`, after the `news_archive` block: + ```python + nm = raw.get("news_model", {}) + _check_keys(nm, _KNOWN_NEWS_MODEL, '"news_model"') + if "mode" in nm and nm["mode"] not in ("off", "fallback", "primary"): + raise AppConfigError('config.json: news_model.mode must be "off", "fallback" or "primary"') + if "threads" in nm and (not isinstance(nm["threads"], int) or isinstance(nm["threads"], bool) or nm["threads"] < 1): + raise AppConfigError("config.json: news_model.threads must be a positive integer") + if "deepseek_audit_per_day" in nm and (not isinstance(nm["deepseek_audit_per_day"], int) or nm["deepseek_audit_per_day"] < 0): + raise AppConfigError("config.json: news_model.deepseek_audit_per_day must be a non-negative integer") + if nm.get("mode", "off") != "off" and not isinstance(nm.get("path"), str): + raise AppConfigError("config.json: news_model.path is required when mode is not off") + app.news_model_settings = dict(nm) + ``` +5. Accessors: + ```python + def news_model_mode() -> str: # env NEWS_MODEL_MODE overrides; default "off" + def news_model_path() -> Optional[str] + def news_model_threads() -> int # default 4 + def news_model_audit_per_day() -> int # default 0 + ``` +`NewsConfig` in `config.py:169` is **not** touched: tests construct it bare as `NewsConfig()`, and it +holds feed/threshold settings, not paths. `news_engine.py` reads the new settings through `app_config` +the same way it reaches DeepSeek through `get_llm_client()`. + +--- + +## 5. `news_engine.py` changes + +### 5.1 New method `_apply_model_sentiment` +Place next to `_apply_llm_sentiment` (:1032). Runs under `_cache_lock` like it. + +```python +def _apply_model_sentiment(self, headlines: List[NewsSentiment]) -> int: + """Score with the local XAU transformer every headline that is still + keyword-scored. Returns the number scored. Never raises. + + The model has a gold head only. It writes xau_score always, XAU's + per-currency score when XAU is mentioned, and the overall score only + when the headline names no other currency. USD/EUR/... entries keep + their keyword score: get_pair_sentiment subtracts USD from XAU and + must not see the same number on both sides. + """ + bundle = news_model.get_news_model() + if bundle is None or not headlines: + return 0 + todo = [h for h in headlines if h.score_source == "keyword"] + if not todo: + return 0 + try: + results = news_model.score_batch(bundle, [h.headline for h in todo]) + except Exception as e: + logger.debug(f"news model scoring failed, keyword scores stand: {e}") + return 0 + for h, r in zip(todo, results): + raw = float(r["score"]) # archive value, unmodified + shown = raw if r["relevant"] else 0.0 # display value, see rule D + h.xau_score = raw + if "XAU" in h.currencies_mentioned: + h.sentiment_by_currency["XAU"] = shown + if set(h.currencies_mentioned) <= {"XAU"}: # [] or ["XAU"] + h.sentiment_score = shown + h.score_source = "model" + return len(todo) +``` + +Rules, with the reason each exists: + +- **A. Never write the model score into a non-XAU currency.** `get_pair_sentiment` (:553-557) forms + `XAU_mean − USD_mean`; `signal_aggregator.py:1677/1723` acts on it. The DeepSeek path puts its + *overall* score into USD; the model has no overall score, so USD keeps the keyword number. +- **B. `sentiment_score` from the model only when the headline names no other currency.** The archive's + `score` column is `sentiment_score`, and the ML-NEWS replay eval reads it; for a gold-only headline it + should be the model's number, for "Fed hikes, dollar jumps" it stays keyword so the row is honest + about what the model did and didn't score. `score_source="model"` therefore means "XAU score is the + model's" — amend the dataclass comment at :223-227 to say so. +- **C. `xau_score` is always the raw signed score.** That column exists for training data + (:229-237); keep it unfiltered. +- **D. Display fields get 0.0 when `relevant` is False.** A headline the model calls not-relevant can + still carry |score| up to ~0.2 from a confident direction head; `NEUTRAL_HEADLINE_THRESHOLD = 0.1` + (:256) would let 0.15 through as "leaning bearish". Zeroing the shown value makes "neutral" mean + neutral. (0.1 happens to equal the model's irrelevant band `RELEVANT_MAX_IRRELEVANT`, so the two + notions agree.) +- **E. Do not touch `NewsEngine._llm_sent_cache`.** That cache decides what is re-sent to DeepSeek. In + `fallback` mode a headline that failed DeepSeek must stay uncached so it is retried next window and + its archive row upgrades `model` → `llm` through the rank rule (news_archive.py:203-260). +- **F. Only `score_source == "keyword"` rows are touched.** Never overwrite an `llm` row in memory. + +### 5.2 Call site (:944-947) +Replace +```python +scored = headlines + unmatched +try: + self._apply_llm_sentiment(scored) +except Exception as e: + logger.debug(f"LLM sentiment failed, keyword scores stand: {e}") +``` +with +```python +scored = headlines + unmatched +mode = app_config.news_model_mode() +try: + if mode == "primary": + n = self._apply_model_sentiment(scored) + if n == 0 and news_model.get_news_model() is None: + # model unavailable this process: degrade to DeepSeek, not to keywords + self._apply_llm_sentiment(scored) + else: + self._audit_with_deepseek(scored) # no-op when audit_per_day == 0 + elif mode == "fallback": + self._apply_llm_sentiment(scored) + self._apply_model_sentiment(scored) # only what DeepSeek left keyword-scored + else: + self._apply_llm_sentiment(scored) +except Exception as e: + logger.debug(f"headline sentiment enrichment failed, keyword scores stand: {e}") +``` +`import news_model` and `import app_config` at the module top of news_engine.py (news_model's top +imports only numpy, §3.2). + +Mode semantics: + +| mode | DeepSeek for headlines | model | when a step fails | +|---|---|---|---| +| `off` | as today | never | keyword scores stand (today) | +| `fallback` | first, as today | on rows DeepSeek left keyword-scored | keyword scores stand | +| `primary` | never (except audit) | on all rows | model missing → DeepSeek path for the cycle; model raises mid-batch → keyword | + +### 5.3 Audit sample (primary mode, optional feature, default off) +```python +_audit_day: str = "" # class attrs, like the caches +_audit_sent_today: int = 0 + +def _audit_with_deepseek(self, headlines): + cap = app_config.news_model_audit_per_day() + if cap <= 0 or not get_llm_client().available: + return + today = datetime.now(timezone.utc).date().isoformat() + if NewsEngine._audit_day != today: + NewsEngine._audit_day, NewsEngine._audit_sent_today = today, 0 + room = cap - NewsEngine._audit_sent_today + if room <= 0: + return + cand = [h for h in headlines if h.score_source == "model" + and hashlib.sha1(h.headline.encode()).hexdigest() not in NewsEngine._llm_sent_cache] + batch = cand[:min(room, self._LLM_BATCH_CAP)] + if not batch: + return + scores = self._llm_headline_scores([h.headline for h in batch]) + if scores is None: + return + NewsEngine._audit_sent_today += len(batch) + copies = [] + for h, (overall, xau) in zip(batch, scores): + NewsEngine._llm_sent_cache[hashlib.sha1(h.headline.encode()).hexdigest()] = (overall, xau) + c = copy.copy(h) # the displayed object keeps the model score + c.sentiment_score = overall + c.sentiment_by_currency = {cur: (xau if cur == "XAU" else overall) for cur in h.currencies_mentioned} + c.xau_score = xau + c.score_source = "llm" + copies.append(c) + self._pending_audit_rows = copies # archived by the call site right after `scored` +``` +The call site then archives `scored` first and `self._pending_audit_rows` second (same +`teacher_model`/`teacher_prompt_sha1` as today's DeepSeek rows); the rank rule upgrades those rows +to `llm`, the in-memory objects the bot displays keep the model's numbers. One batch per cycle at +most, ≤ `cap` per UTC day. This is the only place DeepSeek is called for headlines in primary mode. + +### 5.4 Archive call (:958-964) +Pass, in addition to today's kwargs: +```python +model_teacher=news_model.teacher_tag() # "xau_transformer@" or None +``` +where `teacher_tag()` reads the loaded bundle's meta (None if not loaded). See §6. + +### 5.5 Dataclass comment +`NewsSentiment.score_source` comment (:223-227): add `"model" = the local XAU transformer scored +xau_score (and XAU's per-currency score); other currencies remain keyword-scored.` + +--- + +## 6. `news_archive.py` changes + +No schema change. `_SOURCE_RANK` (:107) and `_STORED_RANK` (:203-206) already rank `model` = 2. + +- `archive_headlines(headlines, *, teacher_model=None, teacher_prompt_sha1=None, origin="live", + model_teacher=None)` — new kwarg, passed through to `build_row`. +- `build_row` (:323-354): + ```python + "teacher_model": (teacher_model if source == "llm" + else model_teacher if source == "model" else None), + "teacher_prompt_sha1": (teacher_prompt_sha1 if source in ("llm", "model") else None), + ``` + For a model row `teacher_prompt_sha1` is the DeepSeek prompt the model was distilled from (the same + constant the bot already passes), which is the truthful provenance. +- Effect: a retrain is visible in the data (`teacher_model` changes), and the UPSERT's + `COALESCE(excluded.teacher_model, headlines.teacher_model)` (:253-260) only applies when the new + rank ≥ stored rank, so an `llm` row's teacher is never replaced by a model tag. + +--- + +## 7. Tests (`test_all.py`) + +The suite is a plain script: `@test(name)` (:51-64) runs each function at definition time and +tallies; the file ends with `sys.exit(1 if FAIL > 0 else 0)`. Run: `venv/bin/python test_all.py`. +Stubbing patterns in use: instance-level `ne._llm_headline_scores = lambda texts: …` with +`lc._client = _StubLLM({...})` and `finally` cleanup (:5196-5215); module-level swap +`saved = _ne_mod.get_llm_client; _ne_mod.get_llm_client = stub; … finally restore` (:11391-11424). + +Add, using the same patterns and a fake bundle (`news_model.score_batch` swapped on the module): + +1. **model scores keyword rows only.** Two `NewsSentiment`s, one `score_source="llm"`; after + `_apply_model_sentiment`, the llm one is byte-identical, the keyword one has `score_source="model"`, + `xau_score` set, `sentiment_by_currency["XAU"]` = score, `["USD"]` unchanged (rule A). +2. **overall score rule (B).** `currencies_mentioned=["XAU"]` → `sentiment_score` = model score; + `["USD","XAU"]` → `sentiment_score` unchanged; `[]` → model score. +3. **relevance zeroing (D).** stub returns `relevant=False, score=-0.18` → `sentiment_by_currency["XAU"] + == 0.0`, `xau_score == -0.18`. +4. **cache untouched (E).** `_llm_sent_cache` empty before and after. +5. **failure isolation.** stub raises → all rows still `keyword`, method returns 0, no exception. + `news_model.get_news_model()` returning None → returns 0. +6. **primary degrades to DeepSeek.** `get_news_model` → None, `_llm_headline_scores` stubbed → rows + end up `llm`. +7. **audit sample.** `deepseek_audit_per_day=3`, 10 model rows → exactly 3 sent, displayed objects + still `model`, `_pending_audit_rows` has 3 `llm` copies; second call same day sends 0. +8. **config.** unknown key in `news_model` → `AppConfigError`; `mode="banana"` → error; section + absent → `news_model_mode() == "off"`; `NEWS_MODEL_MODE=off` env beats `"mode": "primary"`. +9. **archive.** `build_row` of a `model` headline with `model_teacher="xau_transformer@T"` stores it; + an `llm` row stores the DeepSeek teacher; existing rank tests still pass. +10. **import hygiene.** update the docstring of the test at :11437-11462 (it says numpy-only); add + an assertion that `requirements.txt` lists `onnxruntime` and `tokenizers`; keep its "no torch / + transformers" checks. +11. **integration, skipped when the bundle is absent** (`os.environ.get("NEWS_MODEL_DIR")` or the + configured path missing → print SKIP): load, `score_news(b, "Fed cuts rates, signals more + easing")["label"] == "bullish"`, `score_news(b, "Red Lobster endless shrimp losses")["label"] == + "neutral"`, latency of 20 calls < 1 s. + +`test_all.py:3675-3690` loads `config.json.example` for pair output only; adding the section there is +safe. + +--- + +## 8. Ops checklist → `deploy/README-news-model.md` (new) + +Covers **both** processes: the bot (`telegram_bot.py`) and the dashboard daemon +(`deploy/gold-dashboard.service`). + +0. `free -m`. Each model-loading process needs ~0.7 GB headroom. If only the bot should score with + the model, add `Environment=NEWS_MODEL_MODE=off` to `deploy/gold-dashboard.service` (the dashboard + then keeps using DeepSeek for its headlines; those `llm` rows land in the shared archive and act + as free audit labels). +1. `cd /home/Forex/ForexBot2 && git pull && venv/bin/pip install -r requirements.txt`. +2. `mkdir -p /home/Forex/models/xau_transformer` and copy `model.onnx`, `tokenizer.json`, `meta.json` + from the ML-NEWS machine (rsync, ~280 MB). Verify `sha256sum model.onnx` matches the source. + Owner: the account the bot runs as; read-only is enough. +3. Add the `news_model` section to `config.json` (§4.1) with `"mode": "primary"`. Leave the + `deepseek` section unchanged. +4. Smoke test without starting the bot: + ``` + venv/bin/python -c "import news_model as m; b=m.get_news_model(); print(m.score_news(b,'Gold jumps as dollar slides'))" + ``` + Expect `label: bullish`, and the "news model loaded" log line. +5. `venv/bin/python test_all.py` — green (integration test runs because the dir exists). +6. Restart the bot; restart the dashboard unit. Check: + - log: one "news model loaded" line per process, no "news model unavailable"; + - `top`: RSS ~+0.6 GB per process, idle CPU ~0% between scans (spinning disabled); + - archive: new rows have `score_source='model'`, `xau_score` set, `teacher_model` + `xau_transformer@…`; DeepSeek daily counter for headlines stays at 0 (audit off). +7. Rollback: set `"mode": "fallback"` or `"off"` (or the env var), restart. No data migration needed. +8. Retrain cadence: when ML-NEWS ships a new bundle, copy the three files over the old ones and + restart both processes. `meta.json.trained_at` changes, so the archive shows which model scored which + rows. If `deepseek_audit_per_day` is set, ML-NEWS's `eval/eval_archive.py --scorer transformer` on a + copy of the live archive gives the model-vs-DeepSeek agreement on those rows. + +--- + +## 9. Acceptance criteria (end-to-end) + +- With `mode="primary"` and DeepSeek enabled, one live news cycle archives only `model` rows (plus + ≤ `deepseek_audit_per_day` `llm` rows), and the cycle's added latency is < 1 s for ~100 headlines. +- With the model directory renamed away, both processes start, log one warning each, and the cycle's + rows are `llm` (DeepSeek path), not `keyword`. +- With `onnxruntime` uninstalled in a scratch venv, `python -c "import news_engine"` succeeds. +- A headline naming USD and XAU ends with `sentiment_by_currency["USD"]` equal to the keyword value + and `["XAU"]` equal to the model value. +- `mode="off"` reproduces today's archive rows exactly (`llm`/`keyword`, same teacher fields). +- `venv/bin/python test_all.py` exits 0. + +## 10. Known limits to state in the README +- The model copies DeepSeek's opinion (72% agreement on unseen live days, 72/78 golden); it does not + learn from price. DeepSeek itself is ~90% self-consistent. +- Gold only. Every other currency's sentiment remains the ~31%-accurate keyword scorer in primary + mode, which is what those currencies get today whenever DeepSeek is unavailable. +- With audit off, no new DeepSeek labels accrue; the ML-NEWS retrain loop needs either + `deepseek_audit_per_day > 0`, the dashboard left on DeepSeek, or a separate labelling run. diff --git a/news_archive.py b/news_archive.py index f195b7f..f7edf4f 100644 --- a/news_archive.py +++ b/news_archive.py @@ -324,9 +324,17 @@ def build_row(headline: Any, *, teacher_model: Optional[str] = None, teacher_prompt_sha1: Optional[str] = None, origin: str = "live", first_seen: Optional[str] = None, - last_seen: Optional[str] = None) -> Optional[Dict[str, Any]]: + last_seen: Optional[str] = None, + model_teacher: Optional[str] = None) -> Optional[Dict[str, Any]]: """Flatten one ``NewsSentiment`` (or an equivalent object) into a row. + Provenance: an ``llm`` row stores ``teacher_model`` (the DeepSeek + model name); a ``model`` row stores ``model_teacher`` (the local + scorer's tag, ``xau_transformer@``, news_model.teacher_tag). + Both carry ``teacher_prompt_sha1``: for a model row it is the DeepSeek + prompt the model was distilled from, which is the truthful lineage. + Keyword rows carry neither. + Returns None when the object carries no usable headline text.""" text = getattr(headline, "headline", None) if not isinstance(text, str) or not text.strip(): @@ -349,9 +357,10 @@ def build_row(headline: Any, *, teacher_model: Optional[str] = None, getattr(headline, "currencies_mentioned", None)), "keywords_json": _json_or_none(getattr(headline, "keywords", None)), "score_source": source, - "teacher_model": teacher_model if source == "llm" else None, + "teacher_model": (teacher_model if source == "llm" + else model_teacher if source == "model" else None), "teacher_prompt_sha1": ( - teacher_prompt_sha1 if source == "llm" else None), + teacher_prompt_sha1 if source in ("llm", "model") else None), "origin": origin, "xau_score": _finite_score(getattr(headline, "xau_score", None)), } @@ -364,7 +373,8 @@ _archives_lock = threading.Lock() def archive_headlines(headlines: Any, *, teacher_model: Optional[str] = None, teacher_prompt_sha1: Optional[str] = None, - origin: str = "live") -> None: + origin: str = "live", + model_teacher: Optional[str] = None) -> None: """Persist a batch of scored headlines if the news_archive config block enables it. NEVER raises — a logging failure must not break a scan.""" @@ -377,7 +387,7 @@ def archive_headlines(headlines: Any, *, teacher_model: Optional[str] = None, for headline in headlines: row = build_row(headline, teacher_model=teacher_model, teacher_prompt_sha1=teacher_prompt_sha1, - origin=origin) + origin=origin, model_teacher=model_teacher) if row is not None: rows.append(row) if not rows: diff --git a/news_engine.py b/news_engine.py index aa344e4..86d68ad 100755 --- a/news_engine.py +++ b/news_engine.py @@ -8,6 +8,7 @@ Key principle: High-impact news events override technical signals. A perfect technical setup can be destroyed by an unexpected NFP print. """ +import copy import hashlib import html import logging @@ -23,7 +24,9 @@ from typing import Dict, List, Optional, Tuple import requests +import app_config import news_archive +import news_model from config import NewsConfig from llm_client import get_llm_client @@ -222,18 +225,23 @@ class NewsSentiment: publisher: Optional[str] = None # Google News text, else None # Which scorer produced sentiment_score / sentiment_by_currency: # "keyword" (the built-in scorer, ~31% on the 2026-08-06 golden set), - # "llm" (DeepSeek, 78/78 on the same set) or "model" (a trained local - # artifact). Without it the two are indistinguishable in the archive - # and a model trained on the mix inherits the keyword scorer's errors. + # "llm" (DeepSeek, 78/78 on the same set) or "model" (the local XAU + # transformer, news_model.py, 72/78; since 2026-09-27). "model" means + # the model scored xau_score and XAU's per-currency entry; every + # other currency's entry, and sentiment_score when the headline names + # another currency, remain keyword-scored — the model has a gold head + # only. Without the tag the scorers are indistinguishable in the + # archive and a model trained on the mix inherits the keyword errors. score_source: str = "keyword" - # DeepSeek's gold-specific score for THIS headline, whatever currencies - # it names. sentiment_by_currency["XAU"] carries it only when the - # headline matched a gold keyword; for a pure Fed / dollar / yields - # headline it used to be discarded, which left 930 of the first 1,546 - # archived DeepSeek rows without a gold label (eval 2026-09-17). - # Nothing displayed or averaged reads it — the archive stores it as - # training data for a local scorer. None until DeepSeek has scored - # the headline (the keyword scorer never sets it). + # The gold-specific score for THIS headline (DeepSeek's, or the local + # model's raw signed score), whatever currencies it names. + # sentiment_by_currency["XAU"] carries it only when the headline + # matched a gold keyword; for a pure Fed / dollar / yields headline it + # used to be discarded, which left 930 of the first 1,546 archived + # DeepSeek rows without a gold label (eval 2026-09-17). Nothing + # displayed or averaged reads it — the archive stores it as training + # data. None until DeepSeek or the model has scored the headline (the + # keyword scorer never sets it). xau_score: Optional[float] = None @@ -384,6 +392,12 @@ class NewsEngine: #: client's max_tokens (~10 tokens per headline, 2000 max). _LLM_BATCH_CAP = 25 + # primary-mode DeepSeek audit sample (news_model.deepseek_audit_per_day): + # how many model-scored headlines went to DeepSeek today, archive-only. + # Written under _cache_lock like _llm_sent_cache. + _audit_day: str = "" + _audit_sent_today: int = 0 + # Per-source fetch health for status displays (telegram /status). # name → {"ok": ts|None, "err": ts|None, "detail": str, "extra": str}. # All writers run under _cache_lock (fetches happen inside @@ -398,6 +412,14 @@ class NewsEngine: self._session.headers.update({ "User-Agent": "Mozilla/5.0 (compatible; ForexIntel/1.0)" }) + # Create the local scorer's ONNX session off the scan thread so + # the first news cycle does not pay 3-5 s under _cache_lock. + # Idempotent per process (a NewsEngine is built for every scan). + try: + if app_config.news_model_mode() != "off": + news_model.warm_up_in_background() + except Exception as e: # never let the warm-up cost a scan + logger.debug(f"news model warm-up skipped: {e}") def get_fundamental_context( self, currencies: Optional[List[str]] = None @@ -941,10 +963,13 @@ class NewsEngine: # currency-matched headlines so display, per-currency means, the # panel and the summary see exactly what they saw before. scored = headlines + unmatched + mode = app_config.news_model_mode() + audit_rows: List[NewsSentiment] = [] try: - self._apply_llm_sentiment(scored) + audit_rows = self._enrich_headline_scores(scored, mode) except Exception as e: - logger.debug(f"LLM sentiment failed, keyword scores stand: {e}") + logger.debug(f"headline sentiment enrichment failed, keyword " + f"scores stand: {e}") # Archive BEFORE any downstream filtering. get_fundamental_context # drops every headline inside ±NEUTRAL_HEADLINE_THRESHOLD, so a @@ -955,16 +980,65 @@ class NewsEngine: # arguments are evaluated BEFORE that function's own try/except, so # a raising get_llm_client() would escape it. Archiving is a side # effect — it must never cost us the headlines we just fetched. + # + # In "primary" mode the local model is the scorer and its output + # is not training data, so the cycle's rows are NOT archived + # (owner decision 2026-09-27: stop writing newsdata.db once the + # model scores). Only the DeepSeek audit copies, the one source of + # fresh labels, are stored; with deepseek_audit_per_day 0 the + # archive is frozen. "off" and "fallback" archive as before. try: - news_archive.archive_headlines( - scored, - teacher_model=getattr(get_llm_client(), "_model", None), - teacher_prompt_sha1=_LLM_SENTIMENT_PROMPT_SHA1) + teacher = getattr(get_llm_client(), "_model", None) + if mode == "primary": + if audit_rows: + news_archive.archive_headlines( + audit_rows, teacher_model=teacher, + teacher_prompt_sha1=_LLM_SENTIMENT_PROMPT_SHA1) + logger.debug( + f"news archive: primary mode, {len(audit_rows)} DeepSeek " + f"audit rows stored, {len(scored)} model/keyword rows " + f"not archived") + else: + news_archive.archive_headlines( + scored, teacher_model=teacher, + teacher_prompt_sha1=_LLM_SENTIMENT_PROMPT_SHA1, + model_teacher=news_model.teacher_tag()) except Exception as e: logger.debug(f"news archive skipped, headlines unaffected: {e}") return headlines + def _enrich_headline_scores( + self, scored: List[NewsSentiment], mode: str + ) -> List[NewsSentiment]: + """Upgrade keyword scores on `scored` per the news_model mode. + Returns the DeepSeek audit copies to archive (primary mode only, + usually empty). Runs under _cache_lock like the scorers it calls. + + | mode | DeepSeek for headlines | local model | + |----------|------------------------|----------------------------| + | off | every new headline | never | + | fallback | first, as before | rows DeepSeek left keyword | + | primary | audit sample only | every row | + + primary with the model unavailable in this process (missing + files, onnxruntime not installed) degrades to the DeepSeek path + for the cycle, never straight to keywords; a model failure mid- + batch leaves keyword scores standing (see _apply_model_sentiment). + """ + if mode == "primary": + n = self._apply_model_sentiment(scored) + if n == 0 and news_model.get_news_model() is None: + self._apply_llm_sentiment(scored) + return [] + return self._audit_with_deepseek(scored) + if mode == "fallback": + self._apply_llm_sentiment(scored) + self._apply_model_sentiment(scored) + return [] + self._apply_llm_sentiment(scored) + return [] + def _analyze_headline_sentiment( self, headline: str, for_currency: Optional[str] = None ) -> float: @@ -1074,6 +1148,101 @@ class NewsEngine: h.xau_score = xau h.score_source = "llm" + # ── Local XAU transformer (news_model.py, since 2026-09-27) ────── + + def _apply_model_sentiment(self, headlines: List[NewsSentiment]) -> int: + """Score with the local XAU transformer every headline that is + still keyword-scored. Returns the number scored. Never raises. + + The model has a gold head only. It writes xau_score always, XAU's + per-currency score when XAU is mentioned, and the overall score + only when the headline names no other currency. USD/EUR/... + entries keep their keyword score: get_pair_sentiment subtracts + USD from XAU and must not see the same number on both sides. + + Rules (spec 2026-09-27): + A. never write the model score into a non-XAU currency; + B. sentiment_score from the model only for [] or ["XAU"] — the + archive's `score` column must be honest about what the model + did and did not score; + C. xau_score is the raw signed score (training data, unfiltered); + D. displayed values are 0.0 when the model says not relevant — a + confident direction head can still give |score| ~0.2, which + NEUTRAL_HEADLINE_THRESHOLD (0.1) would show as "leaning"; + E. NewsEngine._llm_sent_cache is never touched — it decides what + is re-sent to DeepSeek (fallback mode retries, audit sample); + F. only score_source == "keyword" rows change; an llm row stands. + """ + bundle = news_model.get_news_model() + if bundle is None or not headlines: + return 0 + todo = [h for h in headlines if h.score_source == "keyword"] + if not todo: + return 0 + try: + results = news_model.score_batch(bundle, [h.headline for h in todo]) + if len(results) != len(todo): + raise ValueError( + f"{len(results)} results for {len(todo)} headlines") + except Exception as e: + logger.debug(f"news model scoring failed, keyword scores stand: {e}") + return 0 + for h, r in zip(todo, results): + raw = float(r["score"]) # archive value + shown = raw if r["relevant"] else 0.0 # display value + h.xau_score = raw + if "XAU" in h.currencies_mentioned: + h.sentiment_by_currency["XAU"] = shown + if set(h.currencies_mentioned) <= {"XAU"}: # [] or ["XAU"] + h.sentiment_score = shown + h.score_source = "model" + return len(todo) + + def _audit_with_deepseek( + self, headlines: List[NewsSentiment] + ) -> List[NewsSentiment]: + """primary mode: send up to news_model.deepseek_audit_per_day + model-scored headlines per UTC day to DeepSeek and return `llm` + COPIES for the archive. The displayed objects keep the model's + numbers. One batch per cycle at most; the pairs go into + _llm_sent_cache so a headline is never audited twice. This is the + only place DeepSeek is called for headlines in primary mode; with + the cap at 0 (the shipped value) it returns [] without a call.""" + cap = app_config.news_model_audit_per_day() + if cap <= 0 or not headlines or not get_llm_client().available: + return [] + today = datetime.now(timezone.utc).date().isoformat() + if NewsEngine._audit_day != today: + NewsEngine._audit_day, NewsEngine._audit_sent_today = today, 0 + room = cap - NewsEngine._audit_sent_today + if room <= 0: + return [] + cache = NewsEngine._llm_sent_cache + cand = [h for h in headlines if h.score_source == "model" + and hashlib.sha1(h.headline.encode()).hexdigest() not in cache] + batch = cand[:min(room, self._LLM_BATCH_CAP)] + if not batch: + return [] + scores = self._llm_headline_scores([h.headline for h in batch]) + if scores is None: + return [] + NewsEngine._audit_sent_today += len(batch) + copies = [] + for h, (overall, xau) in zip(batch, scores): + cache[hashlib.sha1(h.headline.encode()).hexdigest()] = (overall, xau) + c = copy.copy(h) + c.sentiment_score = overall + c.sentiment_by_currency = { + cur: (xau if cur == "XAU" else overall) + for cur in h.currencies_mentioned + } + c.xau_score = xau + c.score_source = "llm" + copies.append(c) + while len(cache) > self._LLM_SENT_CACHE_MAX: + cache.pop(next(iter(cache))) + return copies + def _llm_event_direction(self, event: EconomicEvent) -> Optional[str]: """Gold direction for a released surprise, e.g. "bearish: strong jobs lift yields". None → caller keeps the numeric description. diff --git a/news_model.py b/news_model.py new file mode 100644 index 0000000..dac0fa2 --- /dev/null +++ b/news_model.py @@ -0,0 +1,203 @@ +"""Local XAU headline scorer: the owner's ML-NEWS transformer on CPU. + +Vendored from ML-NEWS news_transformer.py at commit afa26dc; bundle +trained_at 2026-09-27T23:15:30.403442+00:00. Keep the scoring functions +identical to that file (the one difference: ``load_model`` has no default +directory here, the path always comes from config.json); ML-NEWS trains +and exports, ForexBot2 only scores. + +A fine-tuned DeBERTa-v3-small encoder with two heads, exported to ONNX +with an int8 embedding table. Scoring needs only onnxruntime, tokenizers +and numpy. For each headline: + +- ``p_relevant``: probability the news moves the gold price (trained on + DeepSeek |xau| >= 0.25 vs <= 0.10) +- ``p_bullish``: probability that, if it does, gold goes up +- ``relevant``: ``p_relevant >= relevance_threshold`` (from meta.json) +- ``label``: ``bullish`` / ``bearish``, or ``neutral`` when not relevant +- ``score``: ``p_relevant * (2 * p_bullish - 1)`` in [-1, 1], the same + scale as DeepSeek's gold score the archive stores in ``xau_score`` + +The model has a GOLD head only. news_engine writes its score into +``xau_score`` and the XAU per-currency entry, never into USD or any other +currency (get_pair_sentiment subtracts USD from XAU). + +Import contract: ``onnxruntime`` and ``tokenizers`` are imported INSIDE +``load_model`` so ``import news_model`` (and therefore ``import +news_engine``) succeeds on a box without them; ``get_news_model`` then +returns None once, with one warning, and the DeepSeek path stays. + +Configuration comes from app_config (``news_model`` section, env +``NEWS_MODEL_MODE``): ``news_model_path()``, ``news_model_threads()``. +""" + +from __future__ import annotations + +import html +import json +import logging +import os +import threading +from pathlib import Path +from typing import Any, Iterable, Optional + +import numpy as np + +logger = logging.getLogger(__name__) + +MAX_LENGTH = 64 +#: DeepSeek |xau| at or above this is "moves gold"; at or below +#: RELEVANT_MAX_IRRELEVANT it is "does not"; the band between carries no +#: relevance label in training. +RELEVANT_MIN = 0.25 +RELEVANT_MAX_IRRELEVANT = 0.10 + + +def clean_headline(headline: str) -> str: + """The exact text normalisation used in training and scoring.""" + text = " ".join(html.unescape(str(headline or "")).split()) + if not text: + raise ValueError("headline is empty") + return text + + +def _sigmoid(x: np.ndarray) -> np.ndarray: + return 1.0 / (1.0 + np.exp(-x)) + + +def load_model(model_dir: str | Path, + threads: int | None = None) -> dict[str, Any]: + """Load the ONNX model, tokenizer and metadata. Only load trusted files.""" + import onnxruntime as ort + from tokenizers import Tokenizer + + model_dir = Path(model_dir) + meta_path = model_dir / "meta.json" + if not meta_path.is_file(): + raise FileNotFoundError(f"model not found: {meta_path}") + meta = json.loads(meta_path.read_text(encoding="utf-8")) + tokenizer = Tokenizer.from_file(str(model_dir / "tokenizer.json")) + tokenizer.enable_truncation(max_length=int(meta.get("max_length", MAX_LENGTH))) + tokenizer.enable_padding(pad_id=int(meta["pad_token_id"]), pad_token=meta["pad_token"]) + options = ort.SessionOptions() + if threads: + options.intra_op_num_threads = max(1, min(int(threads), os.cpu_count() or 1)) + options.inter_op_num_threads = 1 + # ORT intra-op threads spin-wait after each run by default; in a mostly + # idle bot process that is 4 cores at 100% between cycles. + options.add_session_config_entry("session.intra_op.allow_spinning", "0") + session = ort.InferenceSession(str(model_dir / meta.get("onnx_file", "model.onnx")), + options, providers=["CPUExecutionProvider"]) + return {"meta": meta, "tokenizer": tokenizer, "session": session} + + +def score_batch(bundle: dict[str, Any], headlines: Iterable[str]) -> list[dict[str, Any]]: + texts = [clean_headline(h) for h in headlines] + if not texts: + return [] + encodings = bundle["tokenizer"].encode_batch(texts) + ids = np.array([e.ids for e in encodings], dtype=np.int64) + mask = np.array([e.attention_mask for e in encodings], dtype=np.int64) + rel_logit, dir_logit = bundle["session"].run( + ["relevance_logit", "direction_logit"], + {"input_ids": ids, "attention_mask": mask}) + p_rel = _sigmoid(rel_logit.astype(np.float64)) + p_bull = _sigmoid(dir_logit.astype(np.float64)) + threshold = float(bundle["meta"].get("relevance_threshold", 0.5)) + out = [] + for text, pr, pb in zip(texts, p_rel, p_bull): + relevant = bool(pr >= threshold) + direction = "bullish" if pb >= 0.5 else "bearish" + label = direction if relevant else "neutral" + confidence = pr * max(pb, 1 - pb) if relevant else 1 - pr + out.append({ + "target": "XAUUSD", + "text": text, + "relevant": relevant, + "p_relevant": round(float(pr), 4), + "direction": direction, + "p_bullish": round(float(pb), 4), + "label": label, + "score": round(float(pr * (2 * pb - 1)), 4), + "confidence": round(float(confidence), 4), + }) + return out + + +def score_news(bundle: dict[str, Any], headline: str) -> dict[str, Any]: + return score_batch(bundle, [headline])[0] + + +# ── Process-wide singleton (ForexBot2 addition, not in ML-NEWS) ────────── +# +# One ONNX session per process (~0.6 GB RSS, 3–5 s to create). A failed +# load is remembered: the bot logs one warning and keeps the DeepSeek / +# keyword path for the rest of its life instead of retrying every cycle. + +_lock = threading.Lock() +_bundle: Optional[dict] = None +_failed: bool = False # tried and failed; don't retry, don't re-log +_warm_started: bool = False + + +def get_news_model() -> Optional[dict]: + """The loaded bundle, or None. Loads once per process. Never raises.""" + global _bundle, _failed + if _bundle is not None or _failed: + return _bundle + with _lock: + if _bundle is not None or _failed: + return _bundle + try: + import app_config + path = app_config.news_model_path() + if not path: + raise FileNotFoundError("news_model.path is not configured") + threads = app_config.news_model_threads() + b = load_model(path, threads=threads) + logger.info( + "news model loaded from %s (trained_at %s, threshold %s, " + "%d threads)", path, b["meta"].get("trained_at"), + b["meta"].get("relevance_threshold"), threads) + _bundle = b + except Exception as e: # ImportError, FileNotFoundError, bad JSON, ORT + logger.warning( + "news model unavailable, DeepSeek/keyword path stays: %s: %s", + type(e).__name__, e) + _failed = True + return _bundle + + +def teacher_tag() -> Optional[str]: + """Provenance tag for archive rows scored by the loaded model, e.g. + ``xau_transformer@2026-09-27T23:15:30…``. Reads the already-loaded + bundle only; never triggers a load. None when nothing is loaded.""" + b = _bundle + if b is None: + return None + return f"xau_transformer@{b['meta'].get('trained_at', 'unknown')}" + + +def warm_up_in_background() -> None: + """Create the ONNX session on a daemon thread, once per process, so + the first news cycle does not pay the 3–5 s load under + NewsEngine._cache_lock. Idempotent; a later get_news_model() call + simply waits on the lock if the thread is still loading.""" + global _warm_started + if _warm_started or _bundle is not None or _failed: + return + with _lock: + if _warm_started: + return + _warm_started = True + threading.Thread(target=get_news_model, name="news-model-warmup", + daemon=True).start() + + +def reset_for_tests() -> None: + """Forget any loaded or failed state (test_all.py only).""" + global _bundle, _failed, _warm_started + with _lock: + _bundle = None + _failed = False + _warm_started = False diff --git a/requirements.txt b/requirements.txt index 62f1e24..7fce20b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,7 @@ requests>=2.28 twelvedata>=1.2.0 python-telegram-bot>=20.0 websockets>=12.0 +# news_model.py CPU runtime. Dev box (Python 3.12): 1.30.0 / 0.23.2; live box +# (Python 3.9): 1.19.2 / 0.22.2 — identical scores on the 2026-09-27 bundle. +onnxruntime>=1.19,<2 +tokenizers>=0.22,<1 diff --git a/test_all.py b/test_all.py index a170e0e..1d47f2f 100644 --- a/test_all.py +++ b/test_all.py @@ -26,7 +26,7 @@ sys.path.insert(0, os.path.dirname(__file__)) import json import app_config for _k in ("TELEGRAM_TOKEN", "TELEGRAM_CHAT_ID", "TELEGRAM_CHANNEL_ID", - "TWELVE_DATA_API_KEY", "DEEPSEEK_API_KEY"): + "TWELVE_DATA_API_KEY", "DEEPSEEK_API_KEY", "NEWS_MODEL_MODE"): os.environ.pop(_k, None) # The suite must never write the developer's (or the live box's) real # databases either: the RSS fixture tests run the real archive hook and @@ -11436,8 +11436,10 @@ def _(): @test("news archive: shipped code imports no scikit-learn/scipy/lightgbm") def _(): - """The corpus is trained on offline in venv/; anything the live signal - path can reach must stay numpy-only (requirements.txt is the contract).""" + """The corpus is trained on offline (ML-NEWS); anything the live signal + path can reach stays numpy-only plus the ONNX runtime that scores + headlines (news_model.py: onnxruntime + tokenizers, imported lazily, + Module 38). requirements.txt is the contract: no training stack.""" banned = {"sklearn", "scipy", "lightgbm", "torch", "transformers", "cupy"} roots = [Path(".")] + [Path(p) for p in ("providers", "pair_signals")] offenders = [] @@ -11460,6 +11462,8 @@ def _(): req = open("requirements.txt").read() for name in banned: assert name not in req, f"{name} must not enter requirements.txt" + for name in ("onnxruntime", "tokenizers"): + assert name in req, f"{name} is the news model's CPU runtime" @test("news archive: DeepSeek's gold score is kept for every headline, gold keyword or not") @@ -15379,6 +15383,636 @@ if _GPU37: _gk37.BACKEND.release_device_memory() +# ══════════════════════════════════════════════════════════════════════ +# MODULE 38: local news model (news_model.py, 2026-09-27) +# The owner's ML-NEWS XAU transformer scores headlines locally. The +# model has a gold head only, so the rules pinned here are about what +# it may and may not write: xau_score and XAU's per-currency entry, never +# USD; the DeepSeek cache is never touched; primary mode never archives +# the model's own output (owner decision: newsdata.db stops growing once +# the model scores, except the optional DeepSeek audit sample); mode off +# is byte-for-byte today's path. The real bundle is exercised only when +# NEWS_MODEL_DIR or the configured path exists (SKIP otherwise). +# ══════════════════════════════════════════════════════════════════════ + +print("\n═══ Module 38: local news model ═══") + +import ast as _ast38 +import dataclasses as _dc38 +import logging as _log38 +import sqlite3 as _sqlite38 +from datetime import datetime as _dt38, timezone as _tz38 +from pathlib import Path as _Path38 +import app_config as _ac38 +import llm_client as _lc38 +import news_archive as _na38 +import news_engine as _ne38 +import news_model as _nm38 +from config import NewsConfig as _NC38 +from news_engine import NewsEngine as _NE38, NewsSentiment as _NS38 + +_FAKE_BUNDLE38 = {"meta": {"trained_at": "2026-09-27T00:00:00+00:00", + "relevance_threshold": 0.2}} + + +def _h38(headline, currencies=("XAU", "USD"), score=0.3, by=None, + source="keyword"): + h = _NS38(headline=headline, source="https://feed.example/rss", + timestamp=_dt38.now(_tz38.utc), + currencies_mentioned=list(currencies), sentiment_score=score, + keywords=["k"], + sentiment_by_currency=dict( + by if by is not None else {c: score for c in currencies})) + h.score_source = source + return h + + +class _Model38: + """Swap news_model's bundle + scorer for a scripted fake; restore after. + `results` is a dict applied to every headline or a callable(text).""" + + def __init__(self, results=None, raise_exc=None, bundle=_FAKE_BUNDLE38): + self.results = results if results is not None else \ + {"relevant": True, "score": 0.8} + self.raise_exc = raise_exc + self.bundle = bundle + self.calls = [] + + def __enter__(self): + self._saved = (_nm38.get_news_model, _nm38.score_batch, + _nm38.warm_up_in_background) + _nm38.get_news_model = lambda: self.bundle + _nm38._bundle = self.bundle # teacher_tag() reads it directly + _nm38.warm_up_in_background = lambda: None + + def sb(bundle, texts): + texts = list(texts) + self.calls.append(texts) + if self.raise_exc is not None: + raise self.raise_exc + if callable(self.results): + return [self.results(t) for t in texts] + return [dict(self.results) for _ in texts] + _nm38.score_batch = sb + return self + + def __exit__(self, *exc): + (_nm38.get_news_model, _nm38.score_batch, + _nm38.warm_up_in_background) = self._saved + _nm38.reset_for_tests() + return False + + +def _app38(mode="off", audit=0, path="/nonexistent/xau_transformer"): + app = _dc38.replace(_TEST_APP) + app.news_model_settings = {"mode": mode, "path": path, + "deepseek_audit_per_day": audit} + return app + + +@test("news model: scores keyword rows only; USD keeps its keyword value") +def _(): + """Rule A + F. The model has a gold head: it may write xau_score and + XAU's entry, never USD's (get_pair_sentiment forms XAU − USD), and it + never overwrites a DeepSeek-scored row.""" + llm = _h38("Fed hikes, dollar jumps", score=0.4, + by={"XAU": -0.7, "USD": 0.4}, source="llm") + llm.xau_score = -0.7 + kw = _h38("Gold climbs as yields slip", by={"XAU": 0.3, "USD": -0.2}) + before = _dc38.asdict(llm) + with _Model38({"relevant": True, "score": 0.81}): + n = _NE38(_NC38())._apply_model_sentiment([llm, kw]) + assert n == 1, n + assert _dc38.asdict(llm) == before, "an llm row is never touched" + assert kw.score_source == "model", kw.score_source + assert kw.xau_score == 0.81, kw.xau_score + assert kw.sentiment_by_currency["XAU"] == 0.81, kw.sentiment_by_currency + assert kw.sentiment_by_currency["USD"] == -0.2, \ + "USD must keep the keyword value" + assert kw.sentiment_score == 0.3, \ + "a headline naming USD keeps its keyword overall score" + + +@test("news model: overall score comes from the model only for [] or ['XAU']") +def _(): + """Rule B: the archive's `score` column is sentiment_score; for a + gold-only headline it is the model's number, for a mixed headline it + stays keyword so the row is honest about what the model scored.""" + gold = _h38("Gold rallies", currencies=("XAU",), score=0.2) + mixed = _h38("Fed hikes, gold falls", currencies=("USD", "XAU"), score=0.1) + none = _h38("Stocks crater worldwide", currencies=(), score=0.0, by={}) + with _Model38({"relevant": True, "score": -0.55}): + _NE38(_NC38())._apply_model_sentiment([gold, mixed, none]) + assert gold.sentiment_score == -0.55 and gold.sentiment_by_currency["XAU"] == -0.55 + assert mixed.sentiment_score == 0.1, mixed.sentiment_score + assert mixed.sentiment_by_currency["XAU"] == -0.55, mixed.sentiment_by_currency + assert mixed.sentiment_by_currency["USD"] == 0.1, mixed.sentiment_by_currency + assert none.sentiment_score == -0.55 and none.sentiment_by_currency == {}, \ + "no per-currency entry is invented for a headline naming none" + assert all(h.xau_score == -0.55 for h in (gold, mixed, none)) + assert all(h.score_source == "model" for h in (gold, mixed, none)) + + +@test("news model: a not-relevant headline shows 0.0 but archives its raw score") +def _(): + """Rule C + D. relevant=False can still carry |score| ~0.2 from the + direction head; NEUTRAL_HEADLINE_THRESHOLD (0.1) would show 0.15 as + "leaning bearish". The displayed value is zeroed; xau_score is not.""" + h = _h38("Copper slides on China data", currencies=("XAU",), score=0.05) + with _Model38({"relevant": False, "score": -0.18}): + _NE38(_NC38())._apply_model_sentiment([h]) + assert h.sentiment_by_currency["XAU"] == 0.0, h.sentiment_by_currency + assert h.sentiment_score == 0.0, h.sentiment_score + assert h.xau_score == -0.18, "training data stays raw" + assert _ne38._is_neutral_headline(h), "neutral means neutral" + + +@test("news model: never touches NewsEngine._llm_sent_cache") +def _(): + """Rule E: that cache decides what is re-sent to DeepSeek. A headline + the model scored must still be eligible for DeepSeek in fallback mode + and for the audit sample.""" + saved = _NE38._llm_sent_cache + _NE38._llm_sent_cache = {"deadbeef": (0.1, 0.2)} + try: + with _Model38(): + _NE38(_NC38())._apply_model_sentiment( + [_h38("Gold up"), _h38("Gold down")]) + assert _NE38._llm_sent_cache == {"deadbeef": (0.1, 0.2)}, \ + _NE38._llm_sent_cache + finally: + _NE38._llm_sent_cache = saved + + +@test("news model: a scoring failure or a missing model leaves keyword scores standing") +def _(): + rows = [_h38("Gold up"), _h38("Gold down")] + with _Model38(raise_exc=RuntimeError("ORT exploded")): + n = _NE38(_NC38())._apply_model_sentiment(rows) + assert n == 0 and all(h.score_source == "keyword" for h in rows) + assert all(h.xau_score is None for h in rows) + with _Model38(bundle=None) as m: + n = _NE38(_NC38())._apply_model_sentiment(rows) + assert n == 0 and m.calls == [], "no scoring call without a bundle" + assert all(h.score_source == "keyword" for h in rows) + # a length mismatch is a failure too, not a silent misalignment + saved = _nm38.score_batch + with _Model38() as m: + _nm38.score_batch = lambda b, texts: [{"relevant": True, "score": 0.5}] + n = _NE38(_NC38())._apply_model_sentiment(rows) + assert n == 0 and all(h.score_source == "keyword" for h in rows) + assert _nm38.score_batch is saved + + +@test("news model: primary mode with no model degrades to DeepSeek, not to keywords") +def _(): + saved_cache = _NE38._llm_sent_cache + _NE38._llm_sent_cache = {} + _lc38._client = _StubLLM({}) + try: + rows = [_h38("Gold up"), _h38("Stocks fall", currencies=())] + ne = _NE38(_NC38()) + ne._llm_headline_scores = lambda texts: [(0.4, 0.6)] * len(texts) + with _Model38(bundle=None): + out = ne._enrich_headline_scores(rows, "primary") + assert out == [], "no audit copies on the DeepSeek path" + assert all(h.score_source == "llm" for h in rows), \ + [h.score_source for h in rows] + assert rows[0].xau_score == 0.6 + finally: + _NE38._llm_sent_cache = saved_cache + _lc38._client = None + + +@test("news model: primary mode never calls DeepSeek for headlines; off mode never calls the model") +def _(): + saved_cache = _NE38._llm_sent_cache + saved_app = _ac38.get_app_config() + _NE38._llm_sent_cache = {} + _lc38._client = _StubLLM({}) + llm_calls = [] + try: + ne = _NE38(_NC38()) + + def scores(texts): + llm_calls.append(list(texts)) + return [(0.4, 0.6)] * len(texts) + ne._llm_headline_scores = scores + _ac38.set_app_config(_app38("primary", audit=0)) + rows = [_h38("Gold up"), _h38("Gold down")] + with _Model38({"relevant": True, "score": 0.9}) as m: + out = ne._enrich_headline_scores(rows, "primary") + assert out == [] and llm_calls == [], llm_calls + assert m.calls == [["Gold up", "Gold down"]], m.calls + assert all(h.score_source == "model" for h in rows) + + _ac38.set_app_config(saved_app) + rows = [_h38("Gold up"), _h38("Gold down")] + with _Model38() as m: + ne._enrich_headline_scores(rows, "off") + assert m.calls == [], "off mode is today's path exactly" + assert llm_calls == [["Gold up", "Gold down"]], llm_calls + assert all(h.score_source == "llm" for h in rows) + finally: + _ac38.set_app_config(saved_app) + _NE38._llm_sent_cache = saved_cache + _lc38._client = None + + +@test("news model: fallback mode scores only what DeepSeek left keyword-scored") +def _(): + saved_cache = _NE38._llm_sent_cache + _NE38._llm_sent_cache = {} + _lc38._client = _StubLLM({}) + try: + ne = _NE38(_NC38()) + ne._llm_headline_scores = lambda texts: [(0.4, 0.6)] * len(texts) + rows = [_h38("Gold up"), _h38("Gold down")] + with _Model38() as m: + ne._enrich_headline_scores(rows, "fallback") + assert m.calls == [], "DeepSeek scored everything; nothing left" + assert all(h.score_source == "llm" for h in rows) + # DeepSeek fails → the model fills in, and the headlines stay OUT + # of the DeepSeek cache so the next window retries them + ne._llm_headline_scores = lambda texts: None + rows = [_h38("Silver up"), _h38("Silver down")] + with _Model38({"relevant": True, "score": -0.3}) as m: + ne._enrich_headline_scores(rows, "fallback") + assert m.calls == [["Silver up", "Silver down"]], m.calls + assert all(h.score_source == "model" for h in rows) + assert len(_NE38._llm_sent_cache) == 2, "only the two DeepSeek scored" + finally: + _NE38._llm_sent_cache = saved_cache + _lc38._client = None + + +@test("news model: the DeepSeek audit sample is capped per UTC day and archive-only") +def _(): + saved_cache = _NE38._llm_sent_cache + saved_app = _ac38.get_app_config() + saved_day = (_NE38._audit_day, _NE38._audit_sent_today) + _NE38._llm_sent_cache = {} + _NE38._audit_day, _NE38._audit_sent_today = "", 0 + _lc38._client = _StubLLM({}) + sent = [] + try: + ne = _NE38(_NC38()) + + def scores(texts): + sent.append(list(texts)) + return [(0.4, -0.6)] * len(texts) + ne._llm_headline_scores = scores + rows = [_h38(f"Gold headline {i}") for i in range(10)] + _ac38.set_app_config(_app38("primary", audit=3)) + with _Model38({"relevant": True, "score": 0.7}): + copies = ne._enrich_headline_scores(rows, "primary") + assert [len(b) for b in sent] == [3], sent + assert len(copies) == 3, len(copies) + assert all(c.score_source == "llm" and c.xau_score == -0.6 + and c.sentiment_by_currency["XAU"] == -0.6 + and c.sentiment_by_currency["USD"] == 0.4 for c in copies) + assert all(h.score_source == "model" and h.xau_score == 0.7 + for h in rows), "the displayed objects keep the model's numbers" + assert copies[0] is not rows[0] + assert len(_NE38._llm_sent_cache) == 3, "audited once, never twice" + # same day, second cycle: the cap is spent + more = [_h38(f"Gold headline {i}") for i in range(10, 20)] + with _Model38({"relevant": True, "score": 0.7}): + copies2 = ne._enrich_headline_scores(more, "primary") + assert copies2 == [] and len(sent) == 1, (copies2, sent) + # cap 0 (the shipped value): no call at all + _NE38._audit_day, _NE38._audit_sent_today = "", 0 + _ac38.set_app_config(_app38("primary", audit=0)) + with _Model38({"relevant": True, "score": 0.7}): + copies3 = ne._enrich_headline_scores( + [_h38("Gold headline 99")], "primary") + assert copies3 == [] and len(sent) == 1 + finally: + _ac38.set_app_config(saved_app) + _NE38._llm_sent_cache = saved_cache + _NE38._audit_day, _NE38._audit_sent_today = saved_day + _lc38._client = None + + +def _cycle38(mode, audit, model_results, llm_scores): + """Run one fake RSS cycle under `mode` and return (returned headlines, + archived rows by headline). Restores every piece of shared state.""" + gold = "Gold surges as Fed signals rate cuts" + stocks = "Stocks crater worldwide as banking crisis fears spread" + saved_health = _NE38._source_health + saved_cache = _NE38._llm_sent_cache + saved_app = _ac38.get_app_config() + saved_env = os.environ.get("NEWS_ARCHIVE_PATH") + saved_archives = dict(_na38._archives) + saved_day = (_NE38._audit_day, _NE38._audit_sent_today) + _na38._archives.clear() + _NE38._source_health = {} + _NE38._llm_sent_cache = {} + _NE38._audit_day, _NE38._audit_sent_today = "", 0 + _lc38._client = _StubLLM({}) + _lc38._client._model = "deepseek-v4-flash" # the archive's teacher_model + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "newsdata.db") + os.environ["NEWS_ARCHIVE_PATH"] = path + stamp = b"" + _m19_pubdate(1) + b"" + rss = (b"" + b"" + gold.encode() + b"" + stamp + b"" + b"" + stocks.encode() + b"" + stamp + b"" + b"") + + class _Sess: + def get(self, url, timeout=None): + return type("R", (), {"status_code": 200, "content": rss})() + + try: + cfg = _NC38() + cfg.rss_feeds = ["https://feed.example/rss"] + ne = _NE38(cfg) + ne._session = _Sess() + ne._llm_headline_scores = llm_scores + _ac38.set_app_config(_app38(mode, audit=audit)) + with _Model38(model_results): + heads = ne._fetch_news_sentiment_raw() + rows = {} + if os.path.exists(path): + con = _sqlite38.connect(f"file:{path}?mode=ro", uri=True) + try: + rows = {r[0]: r for r in con.execute( + "SELECT headline, score_source, score, xau_score," + " teacher_model, teacher_prompt_sha1 FROM headlines")} + finally: + con.close() + return heads, rows + finally: + _ac38.set_app_config(saved_app) + _NE38._source_health = saved_health + _NE38._llm_sent_cache = saved_cache + _NE38._audit_day, _NE38._audit_sent_today = saved_day + _lc38._client = None + for arch in _na38._archives.values(): + arch.close() + _na38._archives.clear() + _na38._archives.update(saved_archives) + if saved_env is None: + os.environ.pop("NEWS_ARCHIVE_PATH", None) + else: + os.environ["NEWS_ARCHIVE_PATH"] = saved_env + + +@test("news model: primary mode archives nothing but the audit rows; off and fallback archive as before") +def _(): + """Owner decision 2026-09-27: newsdata.db stops growing once the + local model scores. Its output is not training data; only DeepSeek + audit copies (llm) are stored. The platform still receives the same + currency-matched headlines in every mode.""" + gold = "Gold surges as Fed signals rate cuts" + stocks = "Stocks crater worldwide as banking crisis fears spread" + model = {"relevant": True, "score": 0.9} + llm = lambda texts: [(0.5, 0.6)] * len(texts) + + heads, rows = _cycle38("primary", 0, model, llm) + assert [h.headline for h in heads] == [gold], heads + assert heads[0].score_source == "model" and heads[0].xau_score == 0.9 + assert rows == {}, f"primary mode must not archive: {rows}" + + heads, rows = _cycle38("primary", 25, model, llm) + assert set(rows) == {gold, stocks}, rows + assert all(r[1] == "llm" and r[3] == 0.6 and r[4] == "deepseek-v4-flash" + and r[5] == _ne38._LLM_SENTIMENT_PROMPT_SHA1 + for r in rows.values()), rows + assert heads[0].score_source == "model" and heads[0].xau_score == 0.9, \ + "the audit never changes what is displayed" + + heads, rows = _cycle38("off", 0, model, llm) + assert set(rows) == {gold, stocks} and all(r[1] == "llm" for r in rows.values()) + assert heads[0].score_source == "llm" + + heads, rows = _cycle38("fallback", 0, model, lambda texts: None) + assert set(rows) == {gold, stocks}, rows + assert all(r[1] == "model" and r[3] == 0.9 for r in rows.values()), rows + assert all(r[4] == "xau_transformer@2026-09-27T00:00:00+00:00" + for r in rows.values()), "model rows carry the bundle tag" + assert all(r[5] == _ne38._LLM_SENTIMENT_PROMPT_SHA1 for r in rows.values()), \ + "and the DeepSeek prompt the model was distilled from" + + +@test("news model: config block is validated; env NEWS_MODEL_MODE wins; absent = off") +def _(): + saved_app = _ac38.get_app_config() + try: + base = copy.deepcopy(_FULL_CFG) + base["news_model"] = {"path": "/models/xau", "threads": 4, + "mode": "primary", "deepseek_audit_per_day": 0} + app = load_app_config(_write_json(base)) + assert _ac38.news_model_mode(app) == "primary" + assert _ac38.news_model_path(app) == "/models/xau" + assert _ac38.news_model_threads(app) == 4 + assert _ac38.news_model_audit_per_day(app) == 0 + for bad in ({"mode": "banana", "path": "/m"}, + {"path": "/m", "threds": 4}, + {"mode": "primary"}, # no path + {"mode": "primary", "path": ""}, + {"mode": "off", "threads": 0}, + {"mode": "off", "threads": True}, + {"mode": "off", "deepseek_audit_per_day": -1}, + {"mode": "off", "deepseek_audit_per_day": 2.5}): + payload = copy.deepcopy(_FULL_CFG) + payload["news_model"] = bad + try: + load_app_config(_write_json(payload)) + except AppConfigError: + continue + raise AssertionError(f"accepted {bad}") + # absent section → off, defaults + plain = load_app_config(_write_json(copy.deepcopy(_FULL_CFG))) + assert plain.news_model_settings == {} + assert _ac38.news_model_mode(plain) == "off" + assert _ac38.news_model_path(plain) is None + assert _ac38.news_model_threads(plain) == 4 + assert _ac38.news_model_audit_per_day(plain) == 0 + # a relative path resolves against the repo root + rel = _dc38.replace(_TEST_APP) + rel.news_model_settings = {"path": "models/x", "mode": "off"} + assert _ac38.news_model_path(rel) == os.path.join( + os.path.dirname(os.path.abspath(_ac38.__file__)), "models/x") + # env override beats config.json; an invalid value falls back + with patch.dict(os.environ, {"NEWS_MODEL_MODE": "off"}): + assert _ac38.news_model_mode(app) == "off" + with patch.dict(os.environ, {"NEWS_MODEL_MODE": "Fallback"}): + assert _ac38.news_model_mode(app) == "fallback" + with patch.dict(os.environ, {"NEWS_MODEL_MODE": "banana"}): + assert _ac38.news_model_mode(app) == "primary" + # the suite's hermetic config never loads the model + assert _ac38.news_model_mode() == "off" + assert "news_model" in _ac38._KNOWN_TOP + finally: + _ac38.set_app_config(saved_app) + + +@test("news model: archive rows carry the model tag; an llm row is never downgraded by one") +def _(): + row = _na38.build_row(_m32_head(score=0.5, source="model"), + teacher_model="deepseek-v4-flash", + teacher_prompt_sha1="p1", + model_teacher="xau_transformer@T") + assert row["teacher_model"] == "xau_transformer@T", row + assert row["teacher_prompt_sha1"] == "p1", "the distillation prompt" + row = _na38.build_row(_m32_head(score=0.5, source="llm"), + teacher_model="deepseek-v4-flash", + teacher_prompt_sha1="p1", + model_teacher="xau_transformer@T") + assert row["teacher_model"] == "deepseek-v4-flash", row + row = _na38.build_row(_m32_head(score=0.5, source="keyword"), + teacher_model="deepseek-v4-flash", + teacher_prompt_sha1="p1", + model_teacher="xau_transformer@T") + assert row["teacher_model"] is None and row["teacher_prompt_sha1"] is None + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "newsdata.db") + arch = _na38.NewsArchive(path) + try: + arch.store([_na38.build_row(_m32_head(score=0.3, source="model"), + teacher_prompt_sha1="p1", + model_teacher="xau_transformer@T")]) + r = _m32_rows(path)["Gold surges as Fed signals rate cuts"] + assert r[2] == "model" and r[5] == "xau_transformer@T", r + arch.store([_na38.build_row(_m32_head(score=-0.7, source="llm"), + teacher_model="deepseek-v4-flash", + teacher_prompt_sha1="p1")]) + r = _m32_rows(path)["Gold surges as Fed signals rate cuts"] + assert r[1] == -0.7 and r[2] == "llm" and r[5] == "deepseek-v4-flash", \ + "llm upgrades model" + arch.store([_na38.build_row(_m32_head(score=0.3, source="model"), + model_teacher="xau_transformer@T2")]) + r = _m32_rows(path)["Gold surges as Fed signals rate cuts"] + assert r[1] == -0.7 and r[2] == "llm" and r[5] == "deepseek-v4-flash", \ + "model never downgrades llm" + assert r[3] == 3, "but re-seeing it still counts" + finally: + arch.close() + + +@test("news model: news_model.py imports onnxruntime/tokenizers lazily; requirements list them") +def _(): + """`import news_engine` must succeed on a box without the runtime; + the bot then logs one warning and keeps the DeepSeek/keyword path.""" + tree = _ast38.parse(_Path38("news_model.py").read_text(encoding="utf-8")) + top = set() + for node in tree.body: + if isinstance(node, _ast38.Import): + top.update(a.name.split(".")[0] for a in node.names) + elif isinstance(node, _ast38.ImportFrom): + top.add((node.module or "").split(".")[0]) + assert not top & {"onnxruntime", "tokenizers"}, top + assert "numpy" in top + lazy = set() + for node in _ast38.walk(tree): + if isinstance(node, (_ast38.Import, _ast38.ImportFrom)): + names = ([a.name for a in node.names] if isinstance(node, _ast38.Import) + else [node.module or ""]) + lazy.update(n.split(".")[0] for n in names) + assert {"onnxruntime", "tokenizers"} <= lazy, "load_model imports them" + req = open("requirements.txt").read() + assert "onnxruntime" in req and "tokenizers" in req + for word in ("torch", "transformers"): + assert word not in req + + +@test("news model: get_news_model never raises, fails once and remembers it") +def _(): + saved_app = _ac38.get_app_config() + _nm38.reset_for_tests() + log = _log38.getLogger("news_model") + records = [] + + class _H(_log38.Handler): + def emit(self, r): + records.append(r) + h = _H() + log.addHandler(h) + try: + _ac38.set_app_config(_app38("primary", path="/nonexistent/xau_transformer")) + assert _nm38.get_news_model() is None + assert _nm38.get_news_model() is None + assert _nm38.teacher_tag() is None + warns = [r for r in records if r.levelno == _log38.WARNING + and "news model unavailable" in r.getMessage()] + assert len(warns) == 1, "one warning, no retry spam" + # path not configured is the same outcome + _nm38.reset_for_tests() + _ac38.set_app_config(_app38("off", path="")) + _ac38.get_app_config().news_model_settings = {"mode": "primary"} + assert _nm38.get_news_model() is None + # warm-up is idempotent and never raises + _nm38.reset_for_tests() + _nm38.warm_up_in_background() + _nm38.warm_up_in_background() + assert _nm38._warm_started + # a NewsEngine built in mode off never starts it + _nm38.reset_for_tests() + _ac38.set_app_config(saved_app) + _NE38(_NC38()) + assert not _nm38._warm_started + finally: + log.removeHandler(h) + _nm38.reset_for_tests() + _ac38.set_app_config(saved_app) + + +def _bundle_dir38(): + d = os.environ.get("NEWS_MODEL_DIR") + if not d: + try: + d = _ac38.news_model_path(_ac38.load_app_config()) + except Exception: + d = None + if d and os.path.isfile(os.path.join(d, "meta.json")): + return d + return None + + +@test("news model: the real bundle loads, labels the golden pair and stays under 50 ms/headline") +def _(): + d = _bundle_dir38() + if d is None: + print(" SKIP: no bundle (set NEWS_MODEL_DIR or news_model.path)") + return + b = _nm38.load_model(d, threads=4) + up = _nm38.score_news(b, "Fed cuts rates, signals more easing") + off = _nm38.score_news(b, "Red Lobster endless shrimp losses") + assert up["label"] == "bullish" and up["score"] > 0.5, up + assert off["label"] == "neutral" and not off["relevant"], off + assert abs(off["score"]) < _nm38.RELEVANT_MAX_IRRELEVANT, off + batch = _nm38.score_batch(b, ["Gold jumps as dollar slides", + "Fed hikes 50bp, dollar surges"]) + assert batch[0]["label"] == "bullish" and batch[1]["label"] == "bearish", batch + t = time.time() + for _ in range(20): + _nm38.score_news(b, "Gold jumps as dollar slides") + assert time.time() - t < 1.0, "20 single calls must take under a second" + # the singleton path through app_config + saved_app = _ac38.get_app_config() + _nm38.reset_for_tests() + try: + _ac38.set_app_config(_app38("primary", path=d)) + loaded = _nm38.get_news_model() + assert loaded is not None and loaded["meta"]["relevance_threshold"] == 0.2 + assert _nm38.get_news_model() is loaded, "loaded once" + tag = _nm38.teacher_tag() + assert tag and tag.startswith("xau_transformer@2026-"), tag + # end to end through the engine: a real headline, real numbers + h = _h38("Gold jumps as dollar slides", currencies=("XAU", "USD"), + by={"XAU": 0.1, "USD": -0.1}) + n = _NE38(_NC38())._apply_model_sentiment([h]) + assert n == 1 and h.score_source == "model" + assert h.sentiment_by_currency["XAU"] > 0.5 and h.xau_score > 0.5, h + assert h.sentiment_by_currency["USD"] == -0.1 + finally: + _nm38.reset_for_tests() + _ac38.set_app_config(saved_app) + + + # ══════════════════════════════════════════════════════════════════════ # SUMMARY # ══════════════════════════════════════════════════════════════════════