news archive: keep every headline + its DeepSeek score in its own DB
Headlines only ever survived as a side effect of signal logging, and logs/signals.db was rotated around 2026-09-04 — destroying the 3-week, 4,054-headline corpus news_sentiment_value_2026-09-04.md was written on. logs/ is gitignored, so git could not recover it and nobody noticed for four days. There is no way to backfill a fetch window once it passes. news_archive.py writes to its OWN database (logs/newsdata.db) so a signal-log rotation cannot take the news with it. One row per distinct headline (sha256 key) with the text, description, link, publisher, publish time, the score, and — new — which scorer produced that score. Provenance is the load-bearing part. DeepSeek scores 78/78 on the 2026-08-06 golden set; the keyword scorer it overrides scores 24/78 (31%), and that is what silently stands on no key, disabled, daily cap, timeout, HTTP/parse failure, or the >25-new-headlines-per-cycle batch overflow. Until now both landed in the same column, indistinguishable, so a model trained on the mix would inherit the keyword scorer's errors. NewsSentiment.score_source now records it, mirrored into signals.db via signal_log._headline_to_dict, and teacher_model/teacher_prompt_sha1 keep labels made under a changed prompt from being pooled with older ones. Writes are one atomic UPSERT per row, upgrade-only: an llm score always replaces a keyword one (the batch-overflow tail gets labelled on the next window in a long-running process — verified 13/13), a keyword score never overwrites an llm one, and re-seeing a headline always counts. A read-then-write pair lost increments when the bot, the dashboard publisher and the tests first saw a headline together, and the WAL switch itself needed _with_lock_retry; both are regression-tested with three concurrent writers. The hook sits at the end of _fetch_news_sentiment_raw, BEFORE _is_neutral_headline runs in get_fundamental_context. Archiving downstream of that ±0.1 filter is why signals.db holds almost no neutral headlines and would have taught a trained scorer that "neutral" barely exists. scripts/backfill_news_archive.py seeds it from both DB shapes on disk, read-only and idempotent, recomputing provenance by comparing each stored score against the keyword scorer (neither source recorded it): 1,016 distinct headlines, 846 DeepSeek-labelled, growing ~1,400/week. Nothing in the signal core changed. Both news gating flags stay False, no gate reads any of this, and the backtester never builds a NewsEngine so replays never archive (test-locked). 468 -> 483 tests, including a new guard that AST-walks live code for sklearn/scipy/lightgbm imports — that check did not exist before. Do not train yet: ~800 labels gives a holdout too small to grade (SE +/-0.032). Target is ~4,000 deduplicated headlines over >=3 weeks. Claude-Session: https://claude.ai/code/session_01Bzv4ETVapFNauFDQht7QtA
This commit is contained in:
Harwinder Sekhon
+385
@@ -10739,6 +10739,391 @@ def _():
|
||||
assert _ac.oanda_api_token(app) == payload["oanda"]["api_token"]
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# MODULE 32: news archive (durable headline corpus for model training)
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
print("\n═══ Module 32: news archive ═══")
|
||||
|
||||
import ast
|
||||
import copy
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
import news_archive as _na
|
||||
|
||||
|
||||
def _m32_head(headline="Gold surges as Fed signals rate cuts", score=0.5,
|
||||
source="keyword", by=None, currencies=("XAU", "USD"),
|
||||
description="lede", keywords=("surges",)):
|
||||
"""A NewsSentiment carrying an explicit score provenance."""
|
||||
now = _dt_m19.datetime.now(_dt_m19.timezone.utc)
|
||||
h = NewsSentiment(
|
||||
headline=headline, source="https://feed.example/rss", timestamp=now,
|
||||
currencies_mentioned=list(currencies), sentiment_score=score,
|
||||
keywords=list(keywords),
|
||||
sentiment_by_currency=(by if by is not None
|
||||
else {c: score for c in currencies}),
|
||||
description=description, publisher="Example Wire",
|
||||
link="https://example.test/a")
|
||||
h.score_source = source
|
||||
return h
|
||||
|
||||
|
||||
def _m32_rows(path):
|
||||
con = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
|
||||
try:
|
||||
return {r[0]: r for r in con.execute(
|
||||
"SELECT headline, score, score_source, seen_count, description,"
|
||||
" teacher_model, teacher_prompt_sha1, origin, currencies_json"
|
||||
" FROM headlines")}
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
@test("news archive: creates its schema and stores a scored headline")
|
||||
def _():
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = os.path.join(d, "newsdata.db")
|
||||
arch = _na.NewsArchive(path)
|
||||
try:
|
||||
written = arch.store([_na.build_row(
|
||||
_m32_head(), teacher_model="deepseek-v4-flash",
|
||||
teacher_prompt_sha1="abc123")])
|
||||
finally:
|
||||
arch.close()
|
||||
assert written == 1, written
|
||||
rows = _m32_rows(path)
|
||||
assert len(rows) == 1, rows
|
||||
(head, score, src, seen, desc,
|
||||
model, prompt, origin, currencies) = rows[
|
||||
"Gold surges as Fed signals rate cuts"]
|
||||
assert score == 0.5 and src == "keyword" and seen == 1
|
||||
assert desc == "lede" and origin == "live"
|
||||
assert json.loads(currencies) == ["XAU", "USD"]
|
||||
# teacher identity is only meaningful for an LLM score
|
||||
assert model is None and prompt is None, (model, prompt)
|
||||
|
||||
|
||||
@test("news archive: the same headline twice is one row with seen_count bumped")
|
||||
def _():
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = os.path.join(d, "newsdata.db")
|
||||
arch = _na.NewsArchive(path)
|
||||
try:
|
||||
arch.store([_na.build_row(_m32_head())])
|
||||
arch.store([_na.build_row(_m32_head())])
|
||||
arch.store([_na.build_row(_m32_head())])
|
||||
finally:
|
||||
arch.close()
|
||||
rows = _m32_rows(path)
|
||||
assert len(rows) == 1, "dedup is by headline text"
|
||||
assert rows["Gold surges as Fed signals rate cuts"][3] == 3, rows
|
||||
|
||||
|
||||
@test("news archive: an LLM score upgrades a keyword one, never the reverse")
|
||||
def _():
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = os.path.join(d, "newsdata.db")
|
||||
arch = _na.NewsArchive(path)
|
||||
try:
|
||||
# Seen first under the >25/cycle batch-overflow cap: keyword only.
|
||||
arch.store([_na.build_row(_m32_head(score=0.5, source="keyword"))])
|
||||
# Next window DeepSeek labels it — the better label must win.
|
||||
arch.store([_na.build_row(
|
||||
_m32_head(score=-0.7, source="llm"),
|
||||
teacher_model="deepseek-v4-flash", teacher_prompt_sha1="p1")])
|
||||
row = _m32_rows(path)["Gold surges as Fed signals rate cuts"]
|
||||
assert row[1] == -0.7 and row[2] == "llm", row
|
||||
assert row[5] == "deepseek-v4-flash" and row[6] == "p1", row
|
||||
# DeepSeek goes down; the keyword scorer must NOT clobber it.
|
||||
arch.store([_na.build_row(_m32_head(score=0.5, source="keyword"))])
|
||||
row = _m32_rows(path)["Gold surges as Fed signals rate cuts"]
|
||||
assert row[1] == -0.7, "keyword must never overwrite an llm score"
|
||||
assert row[2] == "llm", row
|
||||
assert row[3] == 3, "but re-seeing it still counts"
|
||||
finally:
|
||||
arch.close()
|
||||
|
||||
|
||||
@test("news archive: a fresh llm score replaces an older llm score")
|
||||
def _():
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = os.path.join(d, "newsdata.db")
|
||||
arch = _na.NewsArchive(path)
|
||||
try:
|
||||
arch.store([_na.build_row(_m32_head(score=0.2, source="llm"))])
|
||||
arch.store([_na.build_row(_m32_head(score=0.9, source="llm"))])
|
||||
row = _m32_rows(path)["Gold surges as Fed signals rate cuts"]
|
||||
assert row[1] == 0.9 and row[2] == "llm", row
|
||||
finally:
|
||||
arch.close()
|
||||
|
||||
|
||||
@test("news archive: concurrent writers lose no rows and no seen_count")
|
||||
def _():
|
||||
"""The bot, the dashboard publisher and the tests all open this file at
|
||||
once. A read-then-write pair loses an increment when two connections
|
||||
first see the same headline together, and the WAL switch itself can
|
||||
hit 'database is locked' while they race to create the file."""
|
||||
import threading as _th
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = os.path.join(d, "newsdata.db")
|
||||
errors = []
|
||||
|
||||
def worker(tag):
|
||||
try:
|
||||
arch = _na.NewsArchive(path) # its own connection
|
||||
try:
|
||||
for i in range(40):
|
||||
arch.store([
|
||||
_na.build_row(_m32_head(
|
||||
headline=f"shared {i % 8}", score=0.3)),
|
||||
_na.build_row(_m32_head(
|
||||
headline=f"{tag} only {i}", score=-0.2)),
|
||||
])
|
||||
finally:
|
||||
arch.close()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
errors.append(f"{tag}: {exc!r}")
|
||||
|
||||
threads = [_th.Thread(target=worker, args=(t,))
|
||||
for t in ("bot", "dashboard", "tests")]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
assert not errors, errors
|
||||
|
||||
con = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
|
||||
try:
|
||||
total = con.execute("SELECT COUNT(*) FROM headlines").fetchone()[0]
|
||||
shared = con.execute(
|
||||
"SELECT COUNT(*) FROM headlines"
|
||||
" WHERE headline LIKE 'shared %'").fetchone()[0]
|
||||
seen = con.execute(
|
||||
"SELECT SUM(seen_count) FROM headlines"
|
||||
" WHERE headline LIKE 'shared %'").fetchone()[0]
|
||||
assert con.execute("PRAGMA integrity_check").fetchone()[0] == "ok"
|
||||
finally:
|
||||
con.close()
|
||||
assert shared == 8, f"dedup broke under contention: {shared}"
|
||||
assert total == 8 + 3 * 40, total
|
||||
assert seen == 3 * 40, f"lost {3 * 40 - seen} increments"
|
||||
|
||||
|
||||
@test("news archive: build_row rejects objects with no usable headline text")
|
||||
def _():
|
||||
assert _na.build_row(_m32_head(headline=" ")) is None
|
||||
assert _na.build_row(object()) is None
|
||||
row = _na.build_row(_m32_head(score=float("nan")))
|
||||
assert row is not None and row["score"] is None, "NaN stored as NULL"
|
||||
|
||||
|
||||
@test("news archive: disabled by config writes nothing and does not raise")
|
||||
def _():
|
||||
import app_config as _ac
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = os.path.join(d, "newsdata.db")
|
||||
saved = os.environ.get("NEWS_ARCHIVE_ENABLED")
|
||||
os.environ["NEWS_ARCHIVE_ENABLED"] = "0"
|
||||
try:
|
||||
assert not _ac.news_archive_enabled()
|
||||
_na.archive_headlines([_m32_head()])
|
||||
finally:
|
||||
if saved is None:
|
||||
os.environ.pop("NEWS_ARCHIVE_ENABLED", None)
|
||||
else:
|
||||
os.environ["NEWS_ARCHIVE_ENABLED"] = saved
|
||||
assert not os.path.exists(path), "nothing may be written when off"
|
||||
|
||||
|
||||
@test("news archive: a write failure never raises into a scan")
|
||||
def _():
|
||||
saved_store = _na.NewsArchive.store
|
||||
saved_archives = dict(_na._archives)
|
||||
saved_env = os.environ.get("NEWS_ARCHIVE_PATH")
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
os.environ["NEWS_ARCHIVE_PATH"] = os.path.join(d, "newsdata.db")
|
||||
|
||||
def _boom(self, rows):
|
||||
raise sqlite3.OperationalError("disk I/O error")
|
||||
|
||||
_na.NewsArchive.store = _boom
|
||||
try:
|
||||
_na.archive_headlines([_m32_head()]) # must NOT raise
|
||||
finally:
|
||||
_na.NewsArchive.store = saved_store
|
||||
for arch in _na._archives.values():
|
||||
arch.close()
|
||||
_na._archives.clear()
|
||||
_na._archives.update(saved_archives)
|
||||
if saved_env is None:
|
||||
os.environ.pop("NEWS_ARCHIVE_PATH", None)
|
||||
else:
|
||||
os.environ["NEWS_ARCHIVE_PATH"] = saved_env
|
||||
|
||||
|
||||
@test("news archive: config defaults are on, at logs/newsdata.db, env-overridable")
|
||||
def _():
|
||||
import app_config as _ac
|
||||
app = _ac.AppConfig()
|
||||
assert _ac.news_archive_enabled(app) is True, "accumulating is the point"
|
||||
assert _ac.news_archive_path(app).endswith("logs/newsdata.db")
|
||||
assert os.path.isabs(_ac.news_archive_path(app)), "CWD must not matter"
|
||||
saved = os.environ.get("NEWS_ARCHIVE_PATH")
|
||||
os.environ["NEWS_ARCHIVE_PATH"] = "/tmp/other.db"
|
||||
try:
|
||||
assert _ac.news_archive_path(app) == "/tmp/other.db"
|
||||
finally:
|
||||
if saved is None:
|
||||
os.environ.pop("NEWS_ARCHIVE_PATH", None)
|
||||
else:
|
||||
os.environ["NEWS_ARCHIVE_PATH"] = saved
|
||||
# the corpus must live in its own file: a signals.db rotation took the
|
||||
# previous three weeks of news with it (2026-09-04)
|
||||
assert _ac.news_archive_path(app) != _ac.signal_log_path(app)
|
||||
|
||||
|
||||
@test("news archive: config.json rejects unknown keys in the block")
|
||||
def _():
|
||||
payload = copy.deepcopy(_FULL_CFG)
|
||||
payload["news_archive"] = {"enabled": True, "path": "logs/newsdata.db"}
|
||||
app = load_app_config(_write_json(payload)) # must not raise
|
||||
assert app.news_archive_settings["path"] == "logs/newsdata.db"
|
||||
for bad in ({"enabled": "yes"}, {"path": ""}, {"nope": 1}):
|
||||
payload["news_archive"] = bad
|
||||
try:
|
||||
load_app_config(_write_json(payload))
|
||||
except AppConfigError:
|
||||
continue
|
||||
raise AssertionError(f"news_archive {bad} must be rejected")
|
||||
|
||||
|
||||
@test("news archive: score_source is 'keyword' at build and 'llm' after DeepSeek")
|
||||
def _():
|
||||
import llm_client as lc
|
||||
now = _dt_m19.datetime.now(_dt_m19.timezone.utc)
|
||||
saved_cache = NewsEngine._llm_sent_cache
|
||||
NewsEngine._llm_sent_cache = {}
|
||||
lc._client = _StubLLM({})
|
||||
try:
|
||||
ne = NewsEngine(NewsConfig())
|
||||
h = NewsSentiment(
|
||||
headline="Fed signals rate hike", source="s", timestamp=now,
|
||||
currencies_mentioned=["USD", "XAU"], sentiment_score=1.0,
|
||||
keywords=["rate hike"],
|
||||
sentiment_by_currency={"USD": 1.0, "XAU": -1.0})
|
||||
assert h.score_source == "keyword", "the dataclass default"
|
||||
ne._llm_headline_scores = lambda texts: [(0.4, -0.7)] * len(texts)
|
||||
ne._apply_llm_sentiment([h])
|
||||
assert h.score_source == "llm", "DeepSeek wrote this score"
|
||||
|
||||
# A failed batch changes nothing — including the provenance.
|
||||
h2 = NewsSentiment(
|
||||
headline="Dollar steady before CPI", source="s", timestamp=now,
|
||||
currencies_mentioned=["USD"], sentiment_score=0.2, keywords=[],
|
||||
sentiment_by_currency={"USD": 0.2})
|
||||
ne._llm_headline_scores = lambda texts: None
|
||||
ne._apply_llm_sentiment([h2])
|
||||
assert h2.score_source == "keyword", "keyword score, keyword label"
|
||||
assert h2.sentiment_score == 0.2
|
||||
finally:
|
||||
NewsEngine._llm_sent_cache = saved_cache
|
||||
lc._client = None
|
||||
|
||||
|
||||
@test("news archive: signal_log records the score provenance too")
|
||||
def _():
|
||||
from signal_log import _headline_to_dict
|
||||
h = _m32_head(source="llm")
|
||||
assert _headline_to_dict(h)["score_source"] == "llm"
|
||||
assert _headline_to_dict(_m32_head())["score_source"] == "keyword"
|
||||
|
||||
|
||||
@test("news archive: neutral headlines reach the archive, not the context")
|
||||
def _():
|
||||
"""get_fundamental_context drops everything inside ±0.1. Archiving
|
||||
downstream of that filter would build a corpus with almost no neutral
|
||||
examples and teach a trained scorer that 'neutral' barely exists."""
|
||||
from news_engine import _is_neutral_headline
|
||||
saved_health = NewsEngine._source_health
|
||||
saved_env = os.environ.get("NEWS_ARCHIVE_PATH")
|
||||
saved_archives = dict(_na._archives)
|
||||
_na._archives.clear()
|
||||
NewsEngine._source_health = {}
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = os.path.join(d, "newsdata.db")
|
||||
os.environ["NEWS_ARCHIVE_PATH"] = path
|
||||
rss = (b"<rss><channel>"
|
||||
b"<item><title>Gold steady as traders await US CPI</title>"
|
||||
b"<pubDate>" + _m19_pubdate(1) + b"</pubDate></item>"
|
||||
b"</channel></rss>")
|
||||
|
||||
class _Sess:
|
||||
def get(self, url, timeout=None):
|
||||
return type("R", (), {"status_code": 200, "content": rss})()
|
||||
|
||||
cfg = NewsConfig()
|
||||
cfg.rss_feeds = ["https://feed.example/rss"]
|
||||
ne = NewsEngine(cfg)
|
||||
ne._session = _Sess()
|
||||
try:
|
||||
heads = ne._fetch_news_sentiment_raw()
|
||||
assert len(heads) == 1, heads
|
||||
assert _is_neutral_headline(heads[0]), \
|
||||
"fixture must be a neutral headline for this test to mean anything"
|
||||
rows = _m32_rows(path)
|
||||
assert "Gold steady as traders await US CPI" in rows, \
|
||||
"the neutral headline must still be archived"
|
||||
finally:
|
||||
NewsEngine._source_health = saved_health
|
||||
for arch in _na._archives.values():
|
||||
arch.close()
|
||||
_na._archives.clear()
|
||||
_na._archives.update(saved_archives)
|
||||
if saved_env is None:
|
||||
os.environ.pop("NEWS_ARCHIVE_PATH", None)
|
||||
else:
|
||||
os.environ["NEWS_ARCHIVE_PATH"] = saved_env
|
||||
|
||||
|
||||
@test("news archive: the backtester never builds a NewsEngine, so replays never archive")
|
||||
def _():
|
||||
src = open("backtest/gpu_backtest.py").read()
|
||||
assert "NewsEngine(" not in src, "backtests must stay news-free (rule 4)"
|
||||
assert "news_archive" not in src
|
||||
assert "use_news=False" in src
|
||||
|
||||
|
||||
@test("news archive: shipped code imports no scikit-learn/scipy/lightgbm")
|
||||
def _():
|
||||
"""The corpus is trained on offline in venv/; anything the live signal
|
||||
path can reach must stay numpy-only (requirements.txt is the contract)."""
|
||||
banned = {"sklearn", "scipy", "lightgbm", "torch", "transformers"}
|
||||
roots = [Path(".")] + [Path(p) for p in ("providers", "pair_signals")]
|
||||
offenders = []
|
||||
for root in roots:
|
||||
for path in sorted(root.glob("*.py")):
|
||||
if "venv" in path.parts:
|
||||
continue
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"), str(path))
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
names = [a.name for a in node.names]
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
names = [node.module or ""]
|
||||
else:
|
||||
continue
|
||||
for name in names:
|
||||
if name.split(".")[0] in banned:
|
||||
offenders.append(f"{path}:{node.lineno} {name}")
|
||||
assert not offenders, f"live code must stay numpy-only: {offenders}"
|
||||
req = open("requirements.txt").read()
|
||||
for name in banned:
|
||||
assert name not in req, f"{name} must not enter requirements.txt"
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# SUMMARY
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
Reference in New Issue
Block a user