harry/News-ML-Decoder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial import: XAUUSD news sentiment models, evals, and data tooling

Browse Source 
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the
two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py)
with its Google News crawler and DeepSeek labeller (data_collect/), the
golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27
results and progress notes.

Transformer weights (model.pt, model.onnx) and the crawl database are not
tracked; rebuild with transformer/train.py and transformer/export_onnx.py.
The DeepSeek label cache in data/ is tracked.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Harwinder Sekhon
2026-09-27 19:22:30 -04:00
commit 1d72e4006a
98 changed files with 1719757 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data_collect/crawl_google_news.py
+290
View File
@@ -0,0 +1,290 @@
#!/usr/bin/env python3
"""Collect historical headlines from Google News RSS into data/history.db.
One request per (query, date window), e.g.
``gold site:reuters.com after:2019-07-22 before:2019-07-29``. Google
returns at most 100 items per request. The crawl is
resumable: finished (query, window) pairs are recorded in ``crawl_log``
and skipped on the next run. Requests are sequential with a fixed
delay; on HTTP 429/503 the crawler backs off and stops after repeated
refusals rather than hammering the service.
Headlines are stored once per normalised text (``hash``); the queries
that found a headline are kept in ``queries``. Nothing here labels or
filters against the golden set; label_history.py and the trainer do.
Usage:
python data_collect/crawl_google_news.py # precise set
python data_collect/crawl_google_news.py --query-set broad # off-topic side
python data_collect/crawl_google_news.py --query-set growth --window-days 30 # growth data + equities
python data_collect/crawl_google_news.py --limit 20 # smoke test
"""
from __future__ import annotations
import argparse
import email.utils
import hashlib
import html
import random
import re
import sqlite3
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from datetime import date, datetime, timedelta, timezone
from pathlib import Path
ML_NEWS_DIR = Path(__file__).resolve().parent.parent
DEFAULT_DB = ML_NEWS_DIR / "data" / "history.db"
FEED_URL = "https://news.google.com/rss/search?q={q}&hl=en-US&gl=US&ceid=US:en"
USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) ML-NEWS research crawler"
#: Scoped to financial publishers: unscoped queries return mostly
#: general news ("gold price" -> phone prices, "safe haven" -> baby-box
#: laws). Reuters answers each weekly window with up to 100 market
#: headlines. Topics cover gold doctrine (Fed, dollar, yields,
#: inflation, jobs, central banks, geopolitics, risk appetite) plus
#: off-topic controls (oil, crypto, single stocks) so the relevance head
#: sees news that does not move gold.
BROAD_QUERIES = (
"gold site:reuters.com",
"Fed site:reuters.com",
"dollar site:reuters.com",
"yields site:reuters.com",
"inflation site:reuters.com",
"payrolls OR jobless site:reuters.com",
"\"central bank\" site:reuters.com",
"stocks site:reuters.com",
"war OR sanctions OR tensions site:reuters.com",
"euro OR yen OR sterling site:reuters.com",
"oil site:reuters.com",
"bitcoin OR crypto site:reuters.com",
"gold site:cnbc.com",
"markets site:cnbc.com",
"site:marketwatch.com",
)
#: A keyword in the query matches anywhere in the article, so the broad
#: set is ~93% headlines that do not move gold (2026-09-23 pilot: 70 of
#: 1,000 DeepSeek-relevant). ``intitle:`` restricts the match to the
#: headline: 3-15 items per week per query, mostly gold doctrine. The
#: broad set supplies the not-relevant side, this one the relevant side.
PRECISE_QUERIES = tuple(
f"intitle:{term} site:{site}"
for site in ("reuters.com", "cnbc.com", "marketwatch.com")
for term in ("gold", "bullion", "Fed", "Powell", "dollar", "yields",
"inflation", "payrolls", '"rate cut"', '"rate hike"', '"safe haven"')
if site == "reuters.com" or term in ("gold", "Fed", "dollar", "yields", "inflation")
)
#: The two golden-set categories the 2026-09-27 model still fails are
#: strong-US-data (3/7) and risk-on (1/4): "GDP beats", "stocks at record
#: highs", "ceasefire lifts markets" should all be bearish for gold, but
#: no precise query covers growth data or equities. ``intitle:`` again,
#: so label_history.py puts these in the precise pool.
GROWTH_TERMS = ("GDP", '"retail sales"', "ISM", "PMI", '"consumer confidence"',
'"jobless claims"', '"consumer spending"', "manufacturing")
RISK_TERMS = ('"record high"', '"risk appetite"', '"Wall Street"', '"S&P 500"',
"ceasefire", '"trade deal"', "equities")
GROWTH_QUERIES = tuple(
f"intitle:{term} site:{site}"
for site in ("reuters.com", "cnbc.com")
for term in GROWTH_TERMS + RISK_TERMS
if site == "reuters.com" or term in ("GDP", '"retail sales"', '"jobless claims"',
'"record high"', '"S&P 500"')
)
QUERY_SETS = {"broad": BROAD_QUERIES, "precise": PRECISE_QUERIES, "growth": GROWTH_QUERIES}
SCHEMA = """
CREATE TABLE IF NOT EXISTS headlines (
hash TEXT PRIMARY KEY,
headline TEXT NOT NULL,
raw_title TEXT NOT NULL,
publisher TEXT,
published_at TEXT,
first_query TEXT NOT NULL,
queries TEXT NOT NULL,
link TEXT,
crawled_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_headlines_published ON headlines (published_at);
CREATE TABLE IF NOT EXISTS crawl_log (
query TEXT NOT NULL,
window_start TEXT NOT NULL,
items INTEGER NOT NULL,
new_headlines INTEGER NOT NULL,
crawled_at TEXT NOT NULL,
PRIMARY KEY (query, window_start)
);
"""
_SPACE = re.compile(r"\s+")
def normalise(text: str) -> str:
return _SPACE.sub(" ", html.unescape(text)).strip()
def headline_key(headline: str) -> str:
"""Hash of the case-folded, whitespace-collapsed headline."""
return hashlib.sha256(normalise(headline).casefold().encode("utf-8")).hexdigest()
def strip_publisher(title: str, publisher: str | None) -> str:
"""Google appends ' - Publisher' to every title; drop it."""
title = normalise(title)
if publisher:
suffix = f" - {normalise(publisher)}"
if title.endswith(suffix):
return title[: -len(suffix)].strip()
return title
def windows_of(start: date, end: date, days: int):
day = start
while day < end:
yield day, min(day + timedelta(days=days), end)
day += timedelta(days=days)
def fetch(url: str, timeout: float) -> bytes:
request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
with urllib.request.urlopen(request, timeout=timeout) as response:
return response.read()
def parse_items(payload: bytes):
root = ET.fromstring(payload)
for item in root.findall("./channel/item"):
title = item.findtext("title") or ""
publisher = item.findtext("source")
published = item.findtext("pubDate")
published_at = None
if published:
try:
published_at = email.utils.parsedate_to_datetime(published).astimezone(
timezone.utc).isoformat()
except (TypeError, ValueError):
published_at = None
headline = strip_publisher(title, publisher)
if headline:
yield {
"headline": headline,
"raw_title": normalise(title),
"publisher": publisher,
"published_at": published_at,
"link": item.findtext("link"),
}
def store(conn: sqlite3.Connection, query: str, items) -> tuple[int, int]:
now = datetime.now(timezone.utc).isoformat()
seen = new = 0
for it in items:
seen += 1
key = headline_key(it["headline"])
row = conn.execute("SELECT queries FROM headlines WHERE hash = ?", (key,)).fetchone()
if row is None:
conn.execute(
"INSERT INTO headlines (hash, headline, raw_title, publisher, published_at,"
" first_query, queries, link, crawled_at) VALUES (?,?,?,?,?,?,?,?,?)",
(key, it["headline"], it["raw_title"], it["publisher"], it["published_at"],
query, query, it["link"], now))
new += 1
elif query not in row[0].split("|"):
conn.execute("UPDATE headlines SET queries = ? WHERE hash = ?",
(row[0] + "|" + query, key))
return seen, new
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
ap.add_argument("--db", type=Path, default=DEFAULT_DB)
ap.add_argument("--start", type=date.fromisoformat, default=date(2019, 1, 7))
ap.add_argument("--end", type=date.fromisoformat, default=date(2026, 9, 1))
ap.add_argument("--delay", type=float, default=2.0, help="seconds between requests")
ap.add_argument("--timeout", type=float, default=20.0)
ap.add_argument("--limit", type=int, default=0, help="stop after N requests (0 = all)")
ap.add_argument("--query-set", choices=sorted(QUERY_SETS), default="precise")
ap.add_argument("--window-days", type=int, default=0,
help="search window length (default: 7 broad, 14 precise; precise "
"queries return 3-15 items a week, far under the 100 cap)")
ap.add_argument("--max-refusals", type=int, default=5,
help="stop after this many consecutive 429/503 responses")
args = ap.parse_args()
args.db.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(args.db)
conn.executescript(SCHEMA)
done = {(q, w) for q, w in conn.execute("SELECT query, window_start FROM crawl_log")}
queries = QUERY_SETS[args.query_set]
days = args.window_days or (14 if args.query_set == "precise" else 7)
windows = list(windows_of(args.start, args.end, days))
# Seeded shuffle, window-major: an interrupted crawl still covers
# every year and every query evenly, so a pilot on a partial crawl
# is not all 2019.
random.Random(0).shuffle(windows)
todo = [(q, a, b) for a, b in windows for q in queries if (q, a.isoformat()) not in done]
print(f"{args.query_set}: {len(windows)} windows of {days} days x {len(queries)} queries; "
f"{len(done)} done, {len(todo)} to go", flush=True)
requests = refusals = 0
started = time.monotonic()
for query, a, b in todo:
if args.limit and requests >= args.limit:
break
q = f"{query} after:{a.isoformat()} before:{b.isoformat()}"
url = FEED_URL.format(q=urllib.parse.quote_plus(q))
try:
payload = fetch(url, args.timeout)
except urllib.error.HTTPError as error:
if error.code in (429, 503):
refusals += 1
wait = min(600, 30 * 2 ** (refusals - 1))
print(f"HTTP {error.code} on {q!r}; refusal {refusals}, sleeping {wait}s",
flush=True)
if refusals >= args.max_refusals:
print("too many refusals, stopping; rerun later to resume", flush=True)
break
time.sleep(wait)
continue
print(f"HTTP {error.code} on {q!r}; skipped", flush=True)
time.sleep(args.delay)
continue
except (urllib.error.URLError, TimeoutError, OSError) as error:
print(f"network error on {q!r}: {error}; skipped", flush=True)
time.sleep(args.delay * 5)
continue
refusals = 0
requests += 1
try:
seen, new = store(conn, query, parse_items(payload))
except ET.ParseError as error:
print(f"bad XML for {q!r}: {error}; skipped", flush=True)
time.sleep(args.delay)
continue
conn.execute(
"INSERT OR REPLACE INTO crawl_log VALUES (?,?,?,?,?)",
(query, a.isoformat(), seen, new, datetime.now(timezone.utc).isoformat()))
conn.commit()
if requests % 100 == 0:
total = conn.execute("SELECT count(*) FROM headlines").fetchone()[0]
rate = requests / (time.monotonic() - started)
print(f"{requests} requests, window {a}, {total} unique headlines, "
f"{rate * 60:.0f} req/min", flush=True)
time.sleep(args.delay)
total = conn.execute("SELECT count(*) FROM headlines").fetchone()[0]
print(f"done: {requests} requests this run, {total} unique headlines in {args.db}",
flush=True)
conn.close()
return 0
if __name__ == "__main__":
sys.exit(main())
data_collect/label_history.py
+219
View File
@@ -0,0 +1,219 @@
#!/usr/bin/env python3
"""Label crawled historical headlines with DeepSeek's live XAU prompt.
Reads data/history.db (crawl_google_news.py), picks unlabelled
headlines by round-robin over (quarter, query) strata so any budget
covers every year and topic evenly — ``--broad-share`` of them from the
broad crawl (mostly not gold-relevant), the rest from the precise
``intitle:`` crawl — and sends them through the live
ForexBot2 DeepSeek path (eval_common.DeepSeekScorer: same prompt sha1,
batches of 25, disk cache). Answers go to the ``labels`` table.
Excluded before any spend: headlines already labelled, anything whose
normalised text is in the golden set (it must never reach training) or
in newsdata.db (already labelled there), junk titles shorter than
20 or longer than 200 characters, and non-Latin (CJK) wire headlines.
Budget: ``--max-calls`` is a hard stop on DeepSeek requests for this
run. The research client gets its own UTC-day cap (``--daily-cap``),
counted in this process only, so the live bot's cap is untouched.
Usage (ForexBot2 venv, it has the DeepSeek client):
PY=/home/harwinder/Development/ForexBot2/venv/bin/python
$PY data_collect/label_history.py --max-calls 40 # pilot
$PY data_collect/label_history.py --report-only # label mix per year
"""
from __future__ import annotations
import argparse
import random
import sqlite3
import sys
from collections import defaultdict
from datetime import datetime, timezone
from pathlib import Path
ML_NEWS_DIR = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ML_NEWS_DIR / "eval"))
sys.path.insert(0, str(Path(__file__).resolve().parent))
from crawl_google_news import DEFAULT_DB, QUERY_SETS, normalise # noqa: E402
from eval_common import BATCH_CAP, load_golden # noqa: E402
LABEL_CACHE = ML_NEWS_DIR / "data" / "history_deepseek_cache.json"
NEWSDATA_DB = ML_NEWS_DIR / "newsdata.db"
CHUNK = BATCH_CAP * 10
LABELS_SCHEMA = """
CREATE TABLE IF NOT EXISTS labels (
hash TEXT PRIMARY KEY,
s REAL NOT NULL,
xau REAL NOT NULL,
model TEXT NOT NULL,
prompt_sha1 TEXT NOT NULL,
labelled_at TEXT NOT NULL
);
"""
def text_key(headline: str) -> str:
return normalise(headline).casefold()
def is_latin(headline: str) -> bool:
"""Reuters' Chinese and Japanese wires match ``intitle:GDP`` and
``intitle:PMI`` too; the live feed is English, so don't pay to label
them. Allows accents and symbols, rejects CJK-heavy text."""
non_ascii = sum(ord(ch) > 0x24F for ch in headline)
return non_ascii <= max(2, len(headline) // 10)
def excluded_texts() -> set[str]:
keys = {text_key(g["headline"]) for g in load_golden()}
if NEWSDATA_DB.is_file():
uri = f"{NEWSDATA_DB.resolve().as_uri()}?mode=ro"
with sqlite3.connect(uri, uri=True) as conn:
keys |= {text_key(h) for (h,) in conn.execute("SELECT headline FROM headlines")}
return keys
def quarter(published_at: str | None) -> str:
if not published_at or len(published_at) < 7:
return "unknown"
return f"{published_at[:4]}Q{(int(published_at[5:7]) - 1) // 3 + 1}"
def _round_robin(strata: dict, n: int, rng: random.Random) -> list[tuple[str, str]]:
for items in strata.values():
rng.shuffle(items)
order = sorted(strata)
rng.shuffle(order)
chosen: list[tuple[str, str]] = []
depth = 0
while len(chosen) < n and any(len(strata[s]) > depth for s in order):
for s in order:
if depth < len(strata[s]):
chosen.append(strata[s][depth])
if len(chosen) >= n:
break
depth += 1
return chosen
def pick(conn: sqlite3.Connection, n: int, seed: int,
broad_share: float, only_queries: set[str] | None = None) -> list[tuple[str, str]]:
"""Round-robin over (quarter, query) strata, random within each, from
two pools: headlines any ``intitle:`` query found (mostly gold
doctrine) and broad-only ones (mostly news that does not move gold).
``broad_share`` of the picks come from the broad pool. With
``only_queries`` (a crawler query set), only headlines one of those
queries found are eligible, and the stratum is that query."""
skip = excluded_texts()
pools: dict[str, dict] = {"precise": defaultdict(list), "broad": defaultdict(list)}
rows = conn.execute(
"SELECT h.hash, h.headline, h.published_at, h.first_query, h.queries FROM headlines h"
" LEFT JOIN labels l ON l.hash = h.hash WHERE l.hash IS NULL")
for key, headline, published_at, first_query, queries in rows:
if not 20 <= len(headline) <= 200 or text_key(headline) in skip \
or not is_latin(headline):
continue
found = queries.split("|")
if only_queries is not None:
found = [q for q in found if q in only_queries]
if not found:
continue
precise = [q for q in found if q.startswith("intitle:")]
pool = "precise" if precise else "broad"
pools[pool][(quarter(published_at), precise[0] if precise else first_query)].append(
(key, headline))
rng = random.Random(seed)
n_broad = round(n * broad_share)
chosen = _round_robin(pools["precise"], n - n_broad, rng)
chosen += _round_robin(pools["broad"], n - len(chosen), rng)
rng.shuffle(chosen)
print(f"pools: precise {sum(map(len, pools['precise'].values()))}, broad "
f"{sum(map(len, pools['broad'].values()))} unlabelled; picked {len(chosen)}")
return chosen
def report(conn: sqlite3.Connection) -> None:
rows = conn.execute(
"SELECT substr(h.published_at, 1, 4), l.xau FROM labels l"
" JOIN headlines h ON h.hash = l.hash").fetchall()
by_year: dict[str, list[float]] = defaultdict(list)
for year, xau in rows:
by_year[year or "????"].append(xau)
print(f"{len(rows)} labelled headlines")
print(f" {'year':<5} {'n':>6} {'bear':>6} {'irrel':>6} {'bull':>6} "
f"{'relevant':>9} {'mean xau':>9}")
for year in sorted(by_year):
xs = by_year[year]
bear = sum(x <= -0.25 for x in xs)
bull = sum(x >= 0.25 for x in xs)
irrel = sum(abs(x) <= 0.10 for x in xs)
print(f" {year:<5} {len(xs):>6} {bear:>6} {irrel:>6} {bull:>6} "
f"{(bear + bull) / len(xs):>8.1%} {sum(xs) / len(xs):>+9.3f}")
print(" (relevant = |xau| >= 0.25, irrel = |xau| <= 0.10, the rest is the grey zone)")
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
ap.add_argument("--db", type=Path, default=DEFAULT_DB)
ap.add_argument("--max-calls", type=int, default=40,
help="hard stop on DeepSeek requests this run (25 headlines each)")
ap.add_argument("--daily-cap", type=int, default=0,
help="research client UTC-day cap (default: max-calls + 20)")
ap.add_argument("--broad-share", type=float, default=0.3,
help="share of picks from broad-only headlines (not-relevant side)")
ap.add_argument("--query-set", choices=sorted(QUERY_SETS), default=None,
help="only label headlines found by this crawler query set")
ap.add_argument("--seed", type=int, default=42)
ap.add_argument("--timeout", type=float, default=60.0)
ap.add_argument("--no-network", action="store_true")
ap.add_argument("--report-only", action="store_true")
args = ap.parse_args()
conn = sqlite3.connect(args.db, timeout=60)
conn.executescript(LABELS_SCHEMA)
if args.report_only:
report(conn)
return 0
from eval_common import DeepSeekScorer
ds = DeepSeekScorer(cache_path=LABEL_CACHE, timeout=args.timeout,
no_network=args.no_network,
daily_cap=args.daily_cap or args.max_calls + 20)
print(f"DeepSeek {ds.model}, prompt {ds.prompt_sha1[:8]}; budget {args.max_calls} calls")
chosen = pick(conn, args.max_calls * BATCH_CAP, args.seed, args.broad_share,
set(QUERY_SETS[args.query_set]) if args.query_set else None)
print(f"selected {len(chosen)} unlabelled headlines")
labelled = 0
for start in range(0, len(chosen), CHUNK):
if ds.calls >= args.max_calls:
break
room = (args.max_calls - ds.calls) * BATCH_CAP
part = chosen[start:start + min(CHUNK, room)]
answers = ds.score_many([h for _, h in part])
now = datetime.now(timezone.utc).isoformat()
rows = [(key, a["s"], a["xau"], ds.model, ds.prompt_sha1, now)
for (key, _), a in zip(part, answers) if a is not None]
conn.executemany("INSERT OR REPLACE INTO labels VALUES (?,?,?,?,?,?)", rows)
conn.commit()
labelled += len(rows)
print(f" {labelled} labelled, {ds.calls} calls "
f"({ds.failed_calls} failed), {ds.cache_hits} from cache", flush=True)
if not ds.available and not rows:
break
stats = ds.stats()
print(f"this run: {stats['calls']} calls, {stats['failed_calls']} failed, "
f"{labelled} headlines labelled")
report(conn)
conn.close()
return 0
if __name__ == "__main__":
sys.exit(main())