Initial import: XAUUSD news sentiment models, evals, and data tooling
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked.
This commit is contained in:
Harwinder Sekhon
@@ -0,0 +1,290 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Collect historical headlines from Google News RSS into data/history.db.
|
||||
|
||||
One request per (query, date window), e.g.
|
||||
``gold site:reuters.com after:2019-07-22 before:2019-07-29``. Google
|
||||
returns at most 100 items per request. The crawl is
|
||||
resumable: finished (query, window) pairs are recorded in ``crawl_log``
|
||||
and skipped on the next run. Requests are sequential with a fixed
|
||||
delay; on HTTP 429/503 the crawler backs off and stops after repeated
|
||||
refusals rather than hammering the service.
|
||||
|
||||
Headlines are stored once per normalised text (``hash``); the queries
|
||||
that found a headline are kept in ``queries``. Nothing here labels or
|
||||
filters against the golden set; label_history.py and the trainer do.
|
||||
|
||||
Usage:
|
||||
python data_collect/crawl_google_news.py # precise set
|
||||
python data_collect/crawl_google_news.py --query-set broad # off-topic side
|
||||
python data_collect/crawl_google_news.py --query-set growth --window-days 30 # growth data + equities
|
||||
python data_collect/crawl_google_news.py --limit 20 # smoke test
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import email.utils
|
||||
import hashlib
|
||||
import html
|
||||
import random
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
ML_NEWS_DIR = Path(__file__).resolve().parent.parent
|
||||
DEFAULT_DB = ML_NEWS_DIR / "data" / "history.db"
|
||||
FEED_URL = "https://news.google.com/rss/search?q={q}&hl=en-US&gl=US&ceid=US:en"
|
||||
USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) ML-NEWS research crawler"
|
||||
|
||||
#: Scoped to financial publishers: unscoped queries return mostly
|
||||
#: general news ("gold price" -> phone prices, "safe haven" -> baby-box
|
||||
#: laws). Reuters answers each weekly window with up to 100 market
|
||||
#: headlines. Topics cover gold doctrine (Fed, dollar, yields,
|
||||
#: inflation, jobs, central banks, geopolitics, risk appetite) plus
|
||||
#: off-topic controls (oil, crypto, single stocks) so the relevance head
|
||||
#: sees news that does not move gold.
|
||||
BROAD_QUERIES = (
|
||||
"gold site:reuters.com",
|
||||
"Fed site:reuters.com",
|
||||
"dollar site:reuters.com",
|
||||
"yields site:reuters.com",
|
||||
"inflation site:reuters.com",
|
||||
"payrolls OR jobless site:reuters.com",
|
||||
"\"central bank\" site:reuters.com",
|
||||
"stocks site:reuters.com",
|
||||
"war OR sanctions OR tensions site:reuters.com",
|
||||
"euro OR yen OR sterling site:reuters.com",
|
||||
"oil site:reuters.com",
|
||||
"bitcoin OR crypto site:reuters.com",
|
||||
"gold site:cnbc.com",
|
||||
"markets site:cnbc.com",
|
||||
"site:marketwatch.com",
|
||||
)
|
||||
|
||||
#: A keyword in the query matches anywhere in the article, so the broad
|
||||
#: set is ~93% headlines that do not move gold (2026-09-23 pilot: 70 of
|
||||
#: 1,000 DeepSeek-relevant). ``intitle:`` restricts the match to the
|
||||
#: headline: 3-15 items per week per query, mostly gold doctrine. The
|
||||
#: broad set supplies the not-relevant side, this one the relevant side.
|
||||
PRECISE_QUERIES = tuple(
|
||||
f"intitle:{term} site:{site}"
|
||||
for site in ("reuters.com", "cnbc.com", "marketwatch.com")
|
||||
for term in ("gold", "bullion", "Fed", "Powell", "dollar", "yields",
|
||||
"inflation", "payrolls", '"rate cut"', '"rate hike"', '"safe haven"')
|
||||
if site == "reuters.com" or term in ("gold", "Fed", "dollar", "yields", "inflation")
|
||||
)
|
||||
|
||||
#: The two golden-set categories the 2026-09-27 model still fails are
|
||||
#: strong-US-data (3/7) and risk-on (1/4): "GDP beats", "stocks at record
|
||||
#: highs", "ceasefire lifts markets" should all be bearish for gold, but
|
||||
#: no precise query covers growth data or equities. ``intitle:`` again,
|
||||
#: so label_history.py puts these in the precise pool.
|
||||
GROWTH_TERMS = ("GDP", '"retail sales"', "ISM", "PMI", '"consumer confidence"',
|
||||
'"jobless claims"', '"consumer spending"', "manufacturing")
|
||||
RISK_TERMS = ('"record high"', '"risk appetite"', '"Wall Street"', '"S&P 500"',
|
||||
"ceasefire", '"trade deal"', "equities")
|
||||
GROWTH_QUERIES = tuple(
|
||||
f"intitle:{term} site:{site}"
|
||||
for site in ("reuters.com", "cnbc.com")
|
||||
for term in GROWTH_TERMS + RISK_TERMS
|
||||
if site == "reuters.com" or term in ("GDP", '"retail sales"', '"jobless claims"',
|
||||
'"record high"', '"S&P 500"')
|
||||
)
|
||||
|
||||
QUERY_SETS = {"broad": BROAD_QUERIES, "precise": PRECISE_QUERIES, "growth": GROWTH_QUERIES}
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS headlines (
|
||||
hash TEXT PRIMARY KEY,
|
||||
headline TEXT NOT NULL,
|
||||
raw_title TEXT NOT NULL,
|
||||
publisher TEXT,
|
||||
published_at TEXT,
|
||||
first_query TEXT NOT NULL,
|
||||
queries TEXT NOT NULL,
|
||||
link TEXT,
|
||||
crawled_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_headlines_published ON headlines (published_at);
|
||||
CREATE TABLE IF NOT EXISTS crawl_log (
|
||||
query TEXT NOT NULL,
|
||||
window_start TEXT NOT NULL,
|
||||
items INTEGER NOT NULL,
|
||||
new_headlines INTEGER NOT NULL,
|
||||
crawled_at TEXT NOT NULL,
|
||||
PRIMARY KEY (query, window_start)
|
||||
);
|
||||
"""
|
||||
|
||||
_SPACE = re.compile(r"\s+")
|
||||
|
||||
|
||||
def normalise(text: str) -> str:
|
||||
return _SPACE.sub(" ", html.unescape(text)).strip()
|
||||
|
||||
|
||||
def headline_key(headline: str) -> str:
|
||||
"""Hash of the case-folded, whitespace-collapsed headline."""
|
||||
return hashlib.sha256(normalise(headline).casefold().encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def strip_publisher(title: str, publisher: str | None) -> str:
|
||||
"""Google appends ' - Publisher' to every title; drop it."""
|
||||
title = normalise(title)
|
||||
if publisher:
|
||||
suffix = f" - {normalise(publisher)}"
|
||||
if title.endswith(suffix):
|
||||
return title[: -len(suffix)].strip()
|
||||
return title
|
||||
|
||||
|
||||
def windows_of(start: date, end: date, days: int):
|
||||
day = start
|
||||
while day < end:
|
||||
yield day, min(day + timedelta(days=days), end)
|
||||
day += timedelta(days=days)
|
||||
|
||||
|
||||
def fetch(url: str, timeout: float) -> bytes:
|
||||
request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
return response.read()
|
||||
|
||||
|
||||
def parse_items(payload: bytes):
|
||||
root = ET.fromstring(payload)
|
||||
for item in root.findall("./channel/item"):
|
||||
title = item.findtext("title") or ""
|
||||
publisher = item.findtext("source")
|
||||
published = item.findtext("pubDate")
|
||||
published_at = None
|
||||
if published:
|
||||
try:
|
||||
published_at = email.utils.parsedate_to_datetime(published).astimezone(
|
||||
timezone.utc).isoformat()
|
||||
except (TypeError, ValueError):
|
||||
published_at = None
|
||||
headline = strip_publisher(title, publisher)
|
||||
if headline:
|
||||
yield {
|
||||
"headline": headline,
|
||||
"raw_title": normalise(title),
|
||||
"publisher": publisher,
|
||||
"published_at": published_at,
|
||||
"link": item.findtext("link"),
|
||||
}
|
||||
|
||||
|
||||
def store(conn: sqlite3.Connection, query: str, items) -> tuple[int, int]:
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
seen = new = 0
|
||||
for it in items:
|
||||
seen += 1
|
||||
key = headline_key(it["headline"])
|
||||
row = conn.execute("SELECT queries FROM headlines WHERE hash = ?", (key,)).fetchone()
|
||||
if row is None:
|
||||
conn.execute(
|
||||
"INSERT INTO headlines (hash, headline, raw_title, publisher, published_at,"
|
||||
" first_query, queries, link, crawled_at) VALUES (?,?,?,?,?,?,?,?,?)",
|
||||
(key, it["headline"], it["raw_title"], it["publisher"], it["published_at"],
|
||||
query, query, it["link"], now))
|
||||
new += 1
|
||||
elif query not in row[0].split("|"):
|
||||
conn.execute("UPDATE headlines SET queries = ? WHERE hash = ?",
|
||||
(row[0] + "|" + query, key))
|
||||
return seen, new
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
ap.add_argument("--db", type=Path, default=DEFAULT_DB)
|
||||
ap.add_argument("--start", type=date.fromisoformat, default=date(2019, 1, 7))
|
||||
ap.add_argument("--end", type=date.fromisoformat, default=date(2026, 9, 1))
|
||||
ap.add_argument("--delay", type=float, default=2.0, help="seconds between requests")
|
||||
ap.add_argument("--timeout", type=float, default=20.0)
|
||||
ap.add_argument("--limit", type=int, default=0, help="stop after N requests (0 = all)")
|
||||
ap.add_argument("--query-set", choices=sorted(QUERY_SETS), default="precise")
|
||||
ap.add_argument("--window-days", type=int, default=0,
|
||||
help="search window length (default: 7 broad, 14 precise; precise "
|
||||
"queries return 3-15 items a week, far under the 100 cap)")
|
||||
ap.add_argument("--max-refusals", type=int, default=5,
|
||||
help="stop after this many consecutive 429/503 responses")
|
||||
args = ap.parse_args()
|
||||
|
||||
args.db.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(args.db)
|
||||
conn.executescript(SCHEMA)
|
||||
done = {(q, w) for q, w in conn.execute("SELECT query, window_start FROM crawl_log")}
|
||||
queries = QUERY_SETS[args.query_set]
|
||||
days = args.window_days or (14 if args.query_set == "precise" else 7)
|
||||
windows = list(windows_of(args.start, args.end, days))
|
||||
# Seeded shuffle, window-major: an interrupted crawl still covers
|
||||
# every year and every query evenly, so a pilot on a partial crawl
|
||||
# is not all 2019.
|
||||
random.Random(0).shuffle(windows)
|
||||
todo = [(q, a, b) for a, b in windows for q in queries if (q, a.isoformat()) not in done]
|
||||
print(f"{args.query_set}: {len(windows)} windows of {days} days x {len(queries)} queries; "
|
||||
f"{len(done)} done, {len(todo)} to go", flush=True)
|
||||
|
||||
requests = refusals = 0
|
||||
started = time.monotonic()
|
||||
for query, a, b in todo:
|
||||
if args.limit and requests >= args.limit:
|
||||
break
|
||||
q = f"{query} after:{a.isoformat()} before:{b.isoformat()}"
|
||||
url = FEED_URL.format(q=urllib.parse.quote_plus(q))
|
||||
try:
|
||||
payload = fetch(url, args.timeout)
|
||||
except urllib.error.HTTPError as error:
|
||||
if error.code in (429, 503):
|
||||
refusals += 1
|
||||
wait = min(600, 30 * 2 ** (refusals - 1))
|
||||
print(f"HTTP {error.code} on {q!r}; refusal {refusals}, sleeping {wait}s",
|
||||
flush=True)
|
||||
if refusals >= args.max_refusals:
|
||||
print("too many refusals, stopping; rerun later to resume", flush=True)
|
||||
break
|
||||
time.sleep(wait)
|
||||
continue
|
||||
print(f"HTTP {error.code} on {q!r}; skipped", flush=True)
|
||||
time.sleep(args.delay)
|
||||
continue
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as error:
|
||||
print(f"network error on {q!r}: {error}; skipped", flush=True)
|
||||
time.sleep(args.delay * 5)
|
||||
continue
|
||||
refusals = 0
|
||||
requests += 1
|
||||
try:
|
||||
seen, new = store(conn, query, parse_items(payload))
|
||||
except ET.ParseError as error:
|
||||
print(f"bad XML for {q!r}: {error}; skipped", flush=True)
|
||||
time.sleep(args.delay)
|
||||
continue
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO crawl_log VALUES (?,?,?,?,?)",
|
||||
(query, a.isoformat(), seen, new, datetime.now(timezone.utc).isoformat()))
|
||||
conn.commit()
|
||||
if requests % 100 == 0:
|
||||
total = conn.execute("SELECT count(*) FROM headlines").fetchone()[0]
|
||||
rate = requests / (time.monotonic() - started)
|
||||
print(f"{requests} requests, window {a}, {total} unique headlines, "
|
||||
f"{rate * 60:.0f} req/min", flush=True)
|
||||
time.sleep(args.delay)
|
||||
|
||||
total = conn.execute("SELECT count(*) FROM headlines").fetchone()[0]
|
||||
print(f"done: {requests} requests this run, {total} unique headlines in {args.db}",
|
||||
flush=True)
|
||||
conn.close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,219 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Label crawled historical headlines with DeepSeek's live XAU prompt.
|
||||
|
||||
Reads data/history.db (crawl_google_news.py), picks unlabelled
|
||||
headlines by round-robin over (quarter, query) strata so any budget
|
||||
covers every year and topic evenly — ``--broad-share`` of them from the
|
||||
broad crawl (mostly not gold-relevant), the rest from the precise
|
||||
``intitle:`` crawl — and sends them through the live
|
||||
ForexBot2 DeepSeek path (eval_common.DeepSeekScorer: same prompt sha1,
|
||||
batches of 25, disk cache). Answers go to the ``labels`` table.
|
||||
|
||||
Excluded before any spend: headlines already labelled, anything whose
|
||||
normalised text is in the golden set (it must never reach training) or
|
||||
in newsdata.db (already labelled there), junk titles shorter than
|
||||
20 or longer than 200 characters, and non-Latin (CJK) wire headlines.
|
||||
|
||||
Budget: ``--max-calls`` is a hard stop on DeepSeek requests for this
|
||||
run. The research client gets its own UTC-day cap (``--daily-cap``),
|
||||
counted in this process only, so the live bot's cap is untouched.
|
||||
|
||||
Usage (ForexBot2 venv, it has the DeepSeek client):
|
||||
PY=/home/harwinder/Development/ForexBot2/venv/bin/python
|
||||
$PY data_collect/label_history.py --max-calls 40 # pilot
|
||||
$PY data_collect/label_history.py --report-only # label mix per year
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import random
|
||||
import sqlite3
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
ML_NEWS_DIR = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ML_NEWS_DIR / "eval"))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from crawl_google_news import DEFAULT_DB, QUERY_SETS, normalise # noqa: E402
|
||||
from eval_common import BATCH_CAP, load_golden # noqa: E402
|
||||
|
||||
LABEL_CACHE = ML_NEWS_DIR / "data" / "history_deepseek_cache.json"
|
||||
NEWSDATA_DB = ML_NEWS_DIR / "newsdata.db"
|
||||
CHUNK = BATCH_CAP * 10
|
||||
|
||||
LABELS_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS labels (
|
||||
hash TEXT PRIMARY KEY,
|
||||
s REAL NOT NULL,
|
||||
xau REAL NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
prompt_sha1 TEXT NOT NULL,
|
||||
labelled_at TEXT NOT NULL
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def text_key(headline: str) -> str:
|
||||
return normalise(headline).casefold()
|
||||
|
||||
|
||||
def is_latin(headline: str) -> bool:
|
||||
"""Reuters' Chinese and Japanese wires match ``intitle:GDP`` and
|
||||
``intitle:PMI`` too; the live feed is English, so don't pay to label
|
||||
them. Allows accents and symbols, rejects CJK-heavy text."""
|
||||
non_ascii = sum(ord(ch) > 0x24F for ch in headline)
|
||||
return non_ascii <= max(2, len(headline) // 10)
|
||||
|
||||
|
||||
def excluded_texts() -> set[str]:
|
||||
keys = {text_key(g["headline"]) for g in load_golden()}
|
||||
if NEWSDATA_DB.is_file():
|
||||
uri = f"{NEWSDATA_DB.resolve().as_uri()}?mode=ro"
|
||||
with sqlite3.connect(uri, uri=True) as conn:
|
||||
keys |= {text_key(h) for (h,) in conn.execute("SELECT headline FROM headlines")}
|
||||
return keys
|
||||
|
||||
|
||||
def quarter(published_at: str | None) -> str:
|
||||
if not published_at or len(published_at) < 7:
|
||||
return "unknown"
|
||||
return f"{published_at[:4]}Q{(int(published_at[5:7]) - 1) // 3 + 1}"
|
||||
|
||||
|
||||
def _round_robin(strata: dict, n: int, rng: random.Random) -> list[tuple[str, str]]:
|
||||
for items in strata.values():
|
||||
rng.shuffle(items)
|
||||
order = sorted(strata)
|
||||
rng.shuffle(order)
|
||||
chosen: list[tuple[str, str]] = []
|
||||
depth = 0
|
||||
while len(chosen) < n and any(len(strata[s]) > depth for s in order):
|
||||
for s in order:
|
||||
if depth < len(strata[s]):
|
||||
chosen.append(strata[s][depth])
|
||||
if len(chosen) >= n:
|
||||
break
|
||||
depth += 1
|
||||
return chosen
|
||||
|
||||
|
||||
def pick(conn: sqlite3.Connection, n: int, seed: int,
|
||||
broad_share: float, only_queries: set[str] | None = None) -> list[tuple[str, str]]:
|
||||
"""Round-robin over (quarter, query) strata, random within each, from
|
||||
two pools: headlines any ``intitle:`` query found (mostly gold
|
||||
doctrine) and broad-only ones (mostly news that does not move gold).
|
||||
``broad_share`` of the picks come from the broad pool. With
|
||||
``only_queries`` (a crawler query set), only headlines one of those
|
||||
queries found are eligible, and the stratum is that query."""
|
||||
skip = excluded_texts()
|
||||
pools: dict[str, dict] = {"precise": defaultdict(list), "broad": defaultdict(list)}
|
||||
rows = conn.execute(
|
||||
"SELECT h.hash, h.headline, h.published_at, h.first_query, h.queries FROM headlines h"
|
||||
" LEFT JOIN labels l ON l.hash = h.hash WHERE l.hash IS NULL")
|
||||
for key, headline, published_at, first_query, queries in rows:
|
||||
if not 20 <= len(headline) <= 200 or text_key(headline) in skip \
|
||||
or not is_latin(headline):
|
||||
continue
|
||||
found = queries.split("|")
|
||||
if only_queries is not None:
|
||||
found = [q for q in found if q in only_queries]
|
||||
if not found:
|
||||
continue
|
||||
precise = [q for q in found if q.startswith("intitle:")]
|
||||
pool = "precise" if precise else "broad"
|
||||
pools[pool][(quarter(published_at), precise[0] if precise else first_query)].append(
|
||||
(key, headline))
|
||||
rng = random.Random(seed)
|
||||
n_broad = round(n * broad_share)
|
||||
chosen = _round_robin(pools["precise"], n - n_broad, rng)
|
||||
chosen += _round_robin(pools["broad"], n - len(chosen), rng)
|
||||
rng.shuffle(chosen)
|
||||
print(f"pools: precise {sum(map(len, pools['precise'].values()))}, broad "
|
||||
f"{sum(map(len, pools['broad'].values()))} unlabelled; picked {len(chosen)}")
|
||||
return chosen
|
||||
|
||||
|
||||
def report(conn: sqlite3.Connection) -> None:
|
||||
rows = conn.execute(
|
||||
"SELECT substr(h.published_at, 1, 4), l.xau FROM labels l"
|
||||
" JOIN headlines h ON h.hash = l.hash").fetchall()
|
||||
by_year: dict[str, list[float]] = defaultdict(list)
|
||||
for year, xau in rows:
|
||||
by_year[year or "????"].append(xau)
|
||||
print(f"{len(rows)} labelled headlines")
|
||||
print(f" {'year':<5} {'n':>6} {'bear':>6} {'irrel':>6} {'bull':>6} "
|
||||
f"{'relevant':>9} {'mean xau':>9}")
|
||||
for year in sorted(by_year):
|
||||
xs = by_year[year]
|
||||
bear = sum(x <= -0.25 for x in xs)
|
||||
bull = sum(x >= 0.25 for x in xs)
|
||||
irrel = sum(abs(x) <= 0.10 for x in xs)
|
||||
print(f" {year:<5} {len(xs):>6} {bear:>6} {irrel:>6} {bull:>6} "
|
||||
f"{(bear + bull) / len(xs):>8.1%} {sum(xs) / len(xs):>+9.3f}")
|
||||
print(" (relevant = |xau| >= 0.25, irrel = |xau| <= 0.10, the rest is the grey zone)")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
ap.add_argument("--db", type=Path, default=DEFAULT_DB)
|
||||
ap.add_argument("--max-calls", type=int, default=40,
|
||||
help="hard stop on DeepSeek requests this run (25 headlines each)")
|
||||
ap.add_argument("--daily-cap", type=int, default=0,
|
||||
help="research client UTC-day cap (default: max-calls + 20)")
|
||||
ap.add_argument("--broad-share", type=float, default=0.3,
|
||||
help="share of picks from broad-only headlines (not-relevant side)")
|
||||
ap.add_argument("--query-set", choices=sorted(QUERY_SETS), default=None,
|
||||
help="only label headlines found by this crawler query set")
|
||||
ap.add_argument("--seed", type=int, default=42)
|
||||
ap.add_argument("--timeout", type=float, default=60.0)
|
||||
ap.add_argument("--no-network", action="store_true")
|
||||
ap.add_argument("--report-only", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
conn = sqlite3.connect(args.db, timeout=60)
|
||||
conn.executescript(LABELS_SCHEMA)
|
||||
if args.report_only:
|
||||
report(conn)
|
||||
return 0
|
||||
|
||||
from eval_common import DeepSeekScorer
|
||||
ds = DeepSeekScorer(cache_path=LABEL_CACHE, timeout=args.timeout,
|
||||
no_network=args.no_network,
|
||||
daily_cap=args.daily_cap or args.max_calls + 20)
|
||||
print(f"DeepSeek {ds.model}, prompt {ds.prompt_sha1[:8]}; budget {args.max_calls} calls")
|
||||
|
||||
chosen = pick(conn, args.max_calls * BATCH_CAP, args.seed, args.broad_share,
|
||||
set(QUERY_SETS[args.query_set]) if args.query_set else None)
|
||||
print(f"selected {len(chosen)} unlabelled headlines")
|
||||
labelled = 0
|
||||
for start in range(0, len(chosen), CHUNK):
|
||||
if ds.calls >= args.max_calls:
|
||||
break
|
||||
room = (args.max_calls - ds.calls) * BATCH_CAP
|
||||
part = chosen[start:start + min(CHUNK, room)]
|
||||
answers = ds.score_many([h for _, h in part])
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
rows = [(key, a["s"], a["xau"], ds.model, ds.prompt_sha1, now)
|
||||
for (key, _), a in zip(part, answers) if a is not None]
|
||||
conn.executemany("INSERT OR REPLACE INTO labels VALUES (?,?,?,?,?,?)", rows)
|
||||
conn.commit()
|
||||
labelled += len(rows)
|
||||
print(f" {labelled} labelled, {ds.calls} calls "
|
||||
f"({ds.failed_calls} failed), {ds.cache_hits} from cache", flush=True)
|
||||
if not ds.available and not rows:
|
||||
break
|
||||
|
||||
stats = ds.stats()
|
||||
print(f"this run: {stats['calls']} calls, {stats['failed_calls']} failed, "
|
||||
f"{labelled} headlines labelled")
|
||||
report(conn)
|
||||
conn.close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user