Harwinder Sekhon
01baef48ff
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked.
220 lines
9.0 KiB
Python
220 lines
9.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Label crawled historical headlines with DeepSeek's live XAU prompt.
|
|
|
|
Reads data/history.db (crawl_google_news.py), picks unlabelled
|
|
headlines by round-robin over (quarter, query) strata so any budget
|
|
covers every year and topic evenly — ``--broad-share`` of them from the
|
|
broad crawl (mostly not gold-relevant), the rest from the precise
|
|
``intitle:`` crawl — and sends them through the live
|
|
ForexBot2 DeepSeek path (eval_common.DeepSeekScorer: same prompt sha1,
|
|
batches of 25, disk cache). Answers go to the ``labels`` table.
|
|
|
|
Excluded before any spend: headlines already labelled, anything whose
|
|
normalised text is in the golden set (it must never reach training) or
|
|
in newsdata.db (already labelled there), junk titles shorter than
|
|
20 or longer than 200 characters, and non-Latin (CJK) wire headlines.
|
|
|
|
Budget: ``--max-calls`` is a hard stop on DeepSeek requests for this
|
|
run. The research client gets its own UTC-day cap (``--daily-cap``),
|
|
counted in this process only, so the live bot's cap is untouched.
|
|
|
|
Usage (ForexBot2 venv, it has the DeepSeek client):
|
|
PY=/home/harwinder/Development/ForexBot2/venv/bin/python
|
|
$PY data_collect/label_history.py --max-calls 40 # pilot
|
|
$PY data_collect/label_history.py --report-only # label mix per year
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import random
|
|
import sqlite3
|
|
import sys
|
|
from collections import defaultdict
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
ML_NEWS_DIR = Path(__file__).resolve().parent.parent
|
|
sys.path.insert(0, str(ML_NEWS_DIR / "eval"))
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
|
|
from crawl_google_news import DEFAULT_DB, QUERY_SETS, normalise # noqa: E402
|
|
from eval_common import BATCH_CAP, load_golden # noqa: E402
|
|
|
|
LABEL_CACHE = ML_NEWS_DIR / "data" / "history_deepseek_cache.json"
|
|
NEWSDATA_DB = ML_NEWS_DIR / "newsdata.db"
|
|
CHUNK = BATCH_CAP * 10
|
|
|
|
LABELS_SCHEMA = """
|
|
CREATE TABLE IF NOT EXISTS labels (
|
|
hash TEXT PRIMARY KEY,
|
|
s REAL NOT NULL,
|
|
xau REAL NOT NULL,
|
|
model TEXT NOT NULL,
|
|
prompt_sha1 TEXT NOT NULL,
|
|
labelled_at TEXT NOT NULL
|
|
);
|
|
"""
|
|
|
|
|
|
def text_key(headline: str) -> str:
|
|
return normalise(headline).casefold()
|
|
|
|
|
|
def is_latin(headline: str) -> bool:
|
|
"""Reuters' Chinese and Japanese wires match ``intitle:GDP`` and
|
|
``intitle:PMI`` too; the live feed is English, so don't pay to label
|
|
them. Allows accents and symbols, rejects CJK-heavy text."""
|
|
non_ascii = sum(ord(ch) > 0x24F for ch in headline)
|
|
return non_ascii <= max(2, len(headline) // 10)
|
|
|
|
|
|
def excluded_texts() -> set[str]:
|
|
keys = {text_key(g["headline"]) for g in load_golden()}
|
|
if NEWSDATA_DB.is_file():
|
|
uri = f"{NEWSDATA_DB.resolve().as_uri()}?mode=ro"
|
|
with sqlite3.connect(uri, uri=True) as conn:
|
|
keys |= {text_key(h) for (h,) in conn.execute("SELECT headline FROM headlines")}
|
|
return keys
|
|
|
|
|
|
def quarter(published_at: str | None) -> str:
|
|
if not published_at or len(published_at) < 7:
|
|
return "unknown"
|
|
return f"{published_at[:4]}Q{(int(published_at[5:7]) - 1) // 3 + 1}"
|
|
|
|
|
|
def _round_robin(strata: dict, n: int, rng: random.Random) -> list[tuple[str, str]]:
|
|
for items in strata.values():
|
|
rng.shuffle(items)
|
|
order = sorted(strata)
|
|
rng.shuffle(order)
|
|
chosen: list[tuple[str, str]] = []
|
|
depth = 0
|
|
while len(chosen) < n and any(len(strata[s]) > depth for s in order):
|
|
for s in order:
|
|
if depth < len(strata[s]):
|
|
chosen.append(strata[s][depth])
|
|
if len(chosen) >= n:
|
|
break
|
|
depth += 1
|
|
return chosen
|
|
|
|
|
|
def pick(conn: sqlite3.Connection, n: int, seed: int,
|
|
broad_share: float, only_queries: set[str] | None = None) -> list[tuple[str, str]]:
|
|
"""Round-robin over (quarter, query) strata, random within each, from
|
|
two pools: headlines any ``intitle:`` query found (mostly gold
|
|
doctrine) and broad-only ones (mostly news that does not move gold).
|
|
``broad_share`` of the picks come from the broad pool. With
|
|
``only_queries`` (a crawler query set), only headlines one of those
|
|
queries found are eligible, and the stratum is that query."""
|
|
skip = excluded_texts()
|
|
pools: dict[str, dict] = {"precise": defaultdict(list), "broad": defaultdict(list)}
|
|
rows = conn.execute(
|
|
"SELECT h.hash, h.headline, h.published_at, h.first_query, h.queries FROM headlines h"
|
|
" LEFT JOIN labels l ON l.hash = h.hash WHERE l.hash IS NULL")
|
|
for key, headline, published_at, first_query, queries in rows:
|
|
if not 20 <= len(headline) <= 200 or text_key(headline) in skip \
|
|
or not is_latin(headline):
|
|
continue
|
|
found = queries.split("|")
|
|
if only_queries is not None:
|
|
found = [q for q in found if q in only_queries]
|
|
if not found:
|
|
continue
|
|
precise = [q for q in found if q.startswith("intitle:")]
|
|
pool = "precise" if precise else "broad"
|
|
pools[pool][(quarter(published_at), precise[0] if precise else first_query)].append(
|
|
(key, headline))
|
|
rng = random.Random(seed)
|
|
n_broad = round(n * broad_share)
|
|
chosen = _round_robin(pools["precise"], n - n_broad, rng)
|
|
chosen += _round_robin(pools["broad"], n - len(chosen), rng)
|
|
rng.shuffle(chosen)
|
|
print(f"pools: precise {sum(map(len, pools['precise'].values()))}, broad "
|
|
f"{sum(map(len, pools['broad'].values()))} unlabelled; picked {len(chosen)}")
|
|
return chosen
|
|
|
|
|
|
def report(conn: sqlite3.Connection) -> None:
|
|
rows = conn.execute(
|
|
"SELECT substr(h.published_at, 1, 4), l.xau FROM labels l"
|
|
" JOIN headlines h ON h.hash = l.hash").fetchall()
|
|
by_year: dict[str, list[float]] = defaultdict(list)
|
|
for year, xau in rows:
|
|
by_year[year or "????"].append(xau)
|
|
print(f"{len(rows)} labelled headlines")
|
|
print(f" {'year':<5} {'n':>6} {'bear':>6} {'irrel':>6} {'bull':>6} "
|
|
f"{'relevant':>9} {'mean xau':>9}")
|
|
for year in sorted(by_year):
|
|
xs = by_year[year]
|
|
bear = sum(x <= -0.25 for x in xs)
|
|
bull = sum(x >= 0.25 for x in xs)
|
|
irrel = sum(abs(x) <= 0.10 for x in xs)
|
|
print(f" {year:<5} {len(xs):>6} {bear:>6} {irrel:>6} {bull:>6} "
|
|
f"{(bear + bull) / len(xs):>8.1%} {sum(xs) / len(xs):>+9.3f}")
|
|
print(" (relevant = |xau| >= 0.25, irrel = |xau| <= 0.10, the rest is the grey zone)")
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
ap.add_argument("--db", type=Path, default=DEFAULT_DB)
|
|
ap.add_argument("--max-calls", type=int, default=40,
|
|
help="hard stop on DeepSeek requests this run (25 headlines each)")
|
|
ap.add_argument("--daily-cap", type=int, default=0,
|
|
help="research client UTC-day cap (default: max-calls + 20)")
|
|
ap.add_argument("--broad-share", type=float, default=0.3,
|
|
help="share of picks from broad-only headlines (not-relevant side)")
|
|
ap.add_argument("--query-set", choices=sorted(QUERY_SETS), default=None,
|
|
help="only label headlines found by this crawler query set")
|
|
ap.add_argument("--seed", type=int, default=42)
|
|
ap.add_argument("--timeout", type=float, default=60.0)
|
|
ap.add_argument("--no-network", action="store_true")
|
|
ap.add_argument("--report-only", action="store_true")
|
|
args = ap.parse_args()
|
|
|
|
conn = sqlite3.connect(args.db, timeout=60)
|
|
conn.executescript(LABELS_SCHEMA)
|
|
if args.report_only:
|
|
report(conn)
|
|
return 0
|
|
|
|
from eval_common import DeepSeekScorer
|
|
ds = DeepSeekScorer(cache_path=LABEL_CACHE, timeout=args.timeout,
|
|
no_network=args.no_network,
|
|
daily_cap=args.daily_cap or args.max_calls + 20)
|
|
print(f"DeepSeek {ds.model}, prompt {ds.prompt_sha1[:8]}; budget {args.max_calls} calls")
|
|
|
|
chosen = pick(conn, args.max_calls * BATCH_CAP, args.seed, args.broad_share,
|
|
set(QUERY_SETS[args.query_set]) if args.query_set else None)
|
|
print(f"selected {len(chosen)} unlabelled headlines")
|
|
labelled = 0
|
|
for start in range(0, len(chosen), CHUNK):
|
|
if ds.calls >= args.max_calls:
|
|
break
|
|
room = (args.max_calls - ds.calls) * BATCH_CAP
|
|
part = chosen[start:start + min(CHUNK, room)]
|
|
answers = ds.score_many([h for _, h in part])
|
|
now = datetime.now(timezone.utc).isoformat()
|
|
rows = [(key, a["s"], a["xau"], ds.model, ds.prompt_sha1, now)
|
|
for (key, _), a in zip(part, answers) if a is not None]
|
|
conn.executemany("INSERT OR REPLACE INTO labels VALUES (?,?,?,?,?,?)", rows)
|
|
conn.commit()
|
|
labelled += len(rows)
|
|
print(f" {labelled} labelled, {ds.calls} calls "
|
|
f"({ds.failed_calls} failed), {ds.cache_hits} from cache", flush=True)
|
|
if not ds.available and not rows:
|
|
break
|
|
|
|
stats = ds.stats()
|
|
print(f"this run: {stats['calls']} calls, {stats['failed_calls']} failed, "
|
|
f"{labelled} headlines labelled")
|
|
report(conn)
|
|
conn.close()
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|