Initial import: XAUUSD news sentiment models, evals, and data tooling

TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-27 19:22:30 -04:00
commit 1d72e4006a
98 changed files with 1719757 additions and 0 deletions
@@ -0,0 +1,219 @@
+#!/usr/bin/env python3
+"""Label crawled historical headlines with DeepSeek's live XAU prompt.
+
+Reads data/history.db (crawl_google_news.py), picks unlabelled
+headlines by round-robin over (quarter, query) strata so any budget
+covers every year and topic evenly — ``--broad-share`` of them from the
+broad crawl (mostly not gold-relevant), the rest from the precise
+``intitle:`` crawl — and sends them through the live
+ForexBot2 DeepSeek path (eval_common.DeepSeekScorer: same prompt sha1,
+batches of 25, disk cache). Answers go to the ``labels`` table.
+
+Excluded before any spend: headlines already labelled, anything whose
+normalised text is in the golden set (it must never reach training) or
+in newsdata.db (already labelled there), junk titles shorter than
+20 or longer than 200 characters, and non-Latin (CJK) wire headlines.
+
+Budget: ``--max-calls`` is a hard stop on DeepSeek requests for this
+run. The research client gets its own UTC-day cap (``--daily-cap``),
+counted in this process only, so the live bot's cap is untouched.
+
+Usage (ForexBot2 venv, it has the DeepSeek client):
+    PY=/home/harwinder/Development/ForexBot2/venv/bin/python
+    $PY data_collect/label_history.py --max-calls 40     # pilot
+    $PY data_collect/label_history.py --report-only      # label mix per year
+"""
+
+from __future__ import annotations
+
+import argparse
+import random
+import sqlite3
+import sys
+from collections import defaultdict
+from datetime import datetime, timezone
+from pathlib import Path
+
+ML_NEWS_DIR = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ML_NEWS_DIR / "eval"))
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from crawl_google_news import DEFAULT_DB, QUERY_SETS, normalise  # noqa: E402
+from eval_common import BATCH_CAP, load_golden  # noqa: E402
+
+LABEL_CACHE = ML_NEWS_DIR / "data" / "history_deepseek_cache.json"
+NEWSDATA_DB = ML_NEWS_DIR / "newsdata.db"
+CHUNK = BATCH_CAP * 10
+
+LABELS_SCHEMA = """
+CREATE TABLE IF NOT EXISTS labels (
+    hash TEXT PRIMARY KEY,
+    s REAL NOT NULL,
+    xau REAL NOT NULL,
+    model TEXT NOT NULL,
+    prompt_sha1 TEXT NOT NULL,
+    labelled_at TEXT NOT NULL
+);
+"""
+
+
+def text_key(headline: str) -> str:
+    return normalise(headline).casefold()
+
+
+def is_latin(headline: str) -> bool:
+    """Reuters' Chinese and Japanese wires match ``intitle:GDP`` and
+    ``intitle:PMI`` too; the live feed is English, so don't pay to label
+    them. Allows accents and symbols, rejects CJK-heavy text."""
+    non_ascii = sum(ord(ch) > 0x24F for ch in headline)
+    return non_ascii <= max(2, len(headline) // 10)
+
+
+def excluded_texts() -> set[str]:
+    keys = {text_key(g["headline"]) for g in load_golden()}
+    if NEWSDATA_DB.is_file():
+        uri = f"{NEWSDATA_DB.resolve().as_uri()}?mode=ro"
+        with sqlite3.connect(uri, uri=True) as conn:
+            keys |= {text_key(h) for (h,) in conn.execute("SELECT headline FROM headlines")}
+    return keys
+
+
+def quarter(published_at: str | None) -> str:
+    if not published_at or len(published_at) < 7:
+        return "unknown"
+    return f"{published_at[:4]}Q{(int(published_at[5:7]) - 1) // 3 + 1}"
+
+
+def _round_robin(strata: dict, n: int, rng: random.Random) -> list[tuple[str, str]]:
+    for items in strata.values():
+        rng.shuffle(items)
+    order = sorted(strata)
+    rng.shuffle(order)
+    chosen: list[tuple[str, str]] = []
+    depth = 0
+    while len(chosen) < n and any(len(strata[s]) > depth for s in order):
+        for s in order:
+            if depth < len(strata[s]):
+                chosen.append(strata[s][depth])
+                if len(chosen) >= n:
+                    break
+        depth += 1
+    return chosen
+
+
+def pick(conn: sqlite3.Connection, n: int, seed: int,
+         broad_share: float, only_queries: set[str] | None = None) -> list[tuple[str, str]]:
+    """Round-robin over (quarter, query) strata, random within each, from
+    two pools: headlines any ``intitle:`` query found (mostly gold
+    doctrine) and broad-only ones (mostly news that does not move gold).
+    ``broad_share`` of the picks come from the broad pool. With
+    ``only_queries`` (a crawler query set), only headlines one of those
+    queries found are eligible, and the stratum is that query."""
+    skip = excluded_texts()
+    pools: dict[str, dict] = {"precise": defaultdict(list), "broad": defaultdict(list)}
+    rows = conn.execute(
+        "SELECT h.hash, h.headline, h.published_at, h.first_query, h.queries FROM headlines h"
+        " LEFT JOIN labels l ON l.hash = h.hash WHERE l.hash IS NULL")
+    for key, headline, published_at, first_query, queries in rows:
+        if not 20 <= len(headline) <= 200 or text_key(headline) in skip \
+                or not is_latin(headline):
+            continue
+        found = queries.split("|")
+        if only_queries is not None:
+            found = [q for q in found if q in only_queries]
+            if not found:
+                continue
+        precise = [q for q in found if q.startswith("intitle:")]
+        pool = "precise" if precise else "broad"
+        pools[pool][(quarter(published_at), precise[0] if precise else first_query)].append(
+            (key, headline))
+    rng = random.Random(seed)
+    n_broad = round(n * broad_share)
+    chosen = _round_robin(pools["precise"], n - n_broad, rng)
+    chosen += _round_robin(pools["broad"], n - len(chosen), rng)
+    rng.shuffle(chosen)
+    print(f"pools: precise {sum(map(len, pools['precise'].values()))}, broad "
+          f"{sum(map(len, pools['broad'].values()))} unlabelled; picked {len(chosen)}")
+    return chosen
+
+
+def report(conn: sqlite3.Connection) -> None:
+    rows = conn.execute(
+        "SELECT substr(h.published_at, 1, 4), l.xau FROM labels l"
+        " JOIN headlines h ON h.hash = l.hash").fetchall()
+    by_year: dict[str, list[float]] = defaultdict(list)
+    for year, xau in rows:
+        by_year[year or "????"].append(xau)
+    print(f"{len(rows)} labelled headlines")
+    print(f"  {'year':<5} {'n':>6} {'bear':>6} {'irrel':>6} {'bull':>6} "
+          f"{'relevant':>9} {'mean xau':>9}")
+    for year in sorted(by_year):
+        xs = by_year[year]
+        bear = sum(x <= -0.25 for x in xs)
+        bull = sum(x >= 0.25 for x in xs)
+        irrel = sum(abs(x) <= 0.10 for x in xs)
+        print(f"  {year:<5} {len(xs):>6} {bear:>6} {irrel:>6} {bull:>6} "
+              f"{(bear + bull) / len(xs):>8.1%} {sum(xs) / len(xs):>+9.3f}")
+    print("  (relevant = |xau| >= 0.25, irrel = |xau| <= 0.10, the rest is the grey zone)")
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    ap.add_argument("--db", type=Path, default=DEFAULT_DB)
+    ap.add_argument("--max-calls", type=int, default=40,
+                    help="hard stop on DeepSeek requests this run (25 headlines each)")
+    ap.add_argument("--daily-cap", type=int, default=0,
+                    help="research client UTC-day cap (default: max-calls + 20)")
+    ap.add_argument("--broad-share", type=float, default=0.3,
+                    help="share of picks from broad-only headlines (not-relevant side)")
+    ap.add_argument("--query-set", choices=sorted(QUERY_SETS), default=None,
+                    help="only label headlines found by this crawler query set")
+    ap.add_argument("--seed", type=int, default=42)
+    ap.add_argument("--timeout", type=float, default=60.0)
+    ap.add_argument("--no-network", action="store_true")
+    ap.add_argument("--report-only", action="store_true")
+    args = ap.parse_args()
+
+    conn = sqlite3.connect(args.db, timeout=60)
+    conn.executescript(LABELS_SCHEMA)
+    if args.report_only:
+        report(conn)
+        return 0
+
+    from eval_common import DeepSeekScorer
+    ds = DeepSeekScorer(cache_path=LABEL_CACHE, timeout=args.timeout,
+                        no_network=args.no_network,
+                        daily_cap=args.daily_cap or args.max_calls + 20)
+    print(f"DeepSeek {ds.model}, prompt {ds.prompt_sha1[:8]}; budget {args.max_calls} calls")
+
+    chosen = pick(conn, args.max_calls * BATCH_CAP, args.seed, args.broad_share,
+                  set(QUERY_SETS[args.query_set]) if args.query_set else None)
+    print(f"selected {len(chosen)} unlabelled headlines")
+    labelled = 0
+    for start in range(0, len(chosen), CHUNK):
+        if ds.calls >= args.max_calls:
+            break
+        room = (args.max_calls - ds.calls) * BATCH_CAP
+        part = chosen[start:start + min(CHUNK, room)]
+        answers = ds.score_many([h for _, h in part])
+        now = datetime.now(timezone.utc).isoformat()
+        rows = [(key, a["s"], a["xau"], ds.model, ds.prompt_sha1, now)
+                for (key, _), a in zip(part, answers) if a is not None]
+        conn.executemany("INSERT OR REPLACE INTO labels VALUES (?,?,?,?,?,?)", rows)
+        conn.commit()
+        labelled += len(rows)
+        print(f"  {labelled} labelled, {ds.calls} calls "
+              f"({ds.failed_calls} failed), {ds.cache_hits} from cache", flush=True)
+        if not ds.available and not rows:
+            break
+
+    stats = ds.stats()
+    print(f"this run: {stats['calls']} calls, {stats['failed_calls']} failed, "
+          f"{labelled} headlines labelled")
+    report(conn)
+    conn.close()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())