Harwinder Sekhon
1d72e4006a
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
278 lines
5.2 KiB
JSON
278 lines
5.2 KiB
JSON
{
|
|
"band": 0.15,
|
|
"band_sweep": {
|
|
"0.05": {
|
|
"hits": 45,
|
|
"n": 78,
|
|
"neutral_predicted": 5
|
|
},
|
|
"0.1": {
|
|
"hits": 43,
|
|
"n": 78,
|
|
"neutral_predicted": 13
|
|
},
|
|
"0.15": {
|
|
"hits": 43,
|
|
"n": 78,
|
|
"neutral_predicted": 16
|
|
},
|
|
"0.2": {
|
|
"hits": 39,
|
|
"n": 78,
|
|
"neutral_predicted": 22
|
|
},
|
|
"0.3": {
|
|
"hits": 37,
|
|
"n": 78,
|
|
"neutral_predicted": 31
|
|
},
|
|
"0.5": {
|
|
"hits": 23,
|
|
"n": 78,
|
|
"neutral_predicted": 56
|
|
}
|
|
},
|
|
"blocks": {
|
|
"DeepSeek live (xau through band \u00b10.15)": {
|
|
"accuracy": 1.0,
|
|
"confusion": [
|
|
[
|
|
32,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
8,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
0,
|
|
38
|
|
]
|
|
],
|
|
"directional_accuracy": 1.0,
|
|
"directional_hits": 70,
|
|
"directional_n": 70,
|
|
"hits": 78,
|
|
"macro_f1": 1.0,
|
|
"n": 78,
|
|
"sign_agreement": 1.0,
|
|
"sign_hits": 70,
|
|
"sign_n": 70
|
|
},
|
|
"keyword fallback (band \u00b10.15)": {
|
|
"accuracy": 0.3076923076923077,
|
|
"confusion": [
|
|
[
|
|
6,
|
|
15,
|
|
11
|
|
],
|
|
[
|
|
0,
|
|
8,
|
|
0
|
|
],
|
|
[
|
|
12,
|
|
16,
|
|
10
|
|
]
|
|
],
|
|
"directional_accuracy": 0.22857142857142856,
|
|
"directional_hits": 16,
|
|
"directional_n": 70,
|
|
"hits": 24,
|
|
"macro_f1": 0.30646952758745044,
|
|
"n": 78,
|
|
"sign_agreement": 0.41025641025641024,
|
|
"sign_hits": 16,
|
|
"sign_n": 39
|
|
},
|
|
"model (native argmax label)": {
|
|
"accuracy": 0.6025641025641025,
|
|
"confusion": [
|
|
[
|
|
20,
|
|
1,
|
|
11
|
|
],
|
|
[
|
|
0,
|
|
5,
|
|
3
|
|
],
|
|
[
|
|
10,
|
|
6,
|
|
22
|
|
]
|
|
],
|
|
"directional_accuracy": 0.6,
|
|
"directional_hits": 42,
|
|
"directional_n": 70,
|
|
"hits": 47,
|
|
"macro_f1": 0.579918628305725,
|
|
"n": 78,
|
|
"sign_agreement": 0.6666666666666666,
|
|
"sign_hits": 42,
|
|
"sign_n": 63
|
|
},
|
|
"model (score through band \u00b10.15)": {
|
|
"accuracy": 0.5512820512820513,
|
|
"confusion": [
|
|
[
|
|
19,
|
|
4,
|
|
9
|
|
],
|
|
[
|
|
0,
|
|
3,
|
|
5
|
|
],
|
|
[
|
|
8,
|
|
9,
|
|
21
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5714285714285714,
|
|
"directional_hits": 40,
|
|
"directional_n": 70,
|
|
"hits": 43,
|
|
"macro_f1": 0.48980342078786476,
|
|
"n": 78,
|
|
"sign_agreement": 0.7017543859649122,
|
|
"sign_hits": 40,
|
|
"sign_n": 57
|
|
}
|
|
},
|
|
"deepseek": {
|
|
"batch_ms_max": null,
|
|
"batch_ms_mean": null,
|
|
"cache_hits": 78,
|
|
"calls": 0,
|
|
"failed_calls": 0,
|
|
"headlines_sent": 0,
|
|
"model": "deepseek-v4-flash",
|
|
"per_headline_ms": null,
|
|
"prompt_sha1": "d32e04c6a87d019f326f111b39c99b6dd8ce794d"
|
|
},
|
|
"deepseek_scored": 78,
|
|
"generated_at": "20260923_0037_xau_all_20260922",
|
|
"mcnemar": {
|
|
"model_vs_deepseek": {
|
|
"deepseek_only_right": 31,
|
|
"model_only_right": 0,
|
|
"p": 9.313225746154785e-10
|
|
},
|
|
"model_vs_keyword": {
|
|
"keyword_only_right": 8,
|
|
"model_only_right": 31,
|
|
"p": 0.0002940768754342571
|
|
}
|
|
},
|
|
"model": {
|
|
"label_distribution": {
|
|
"bearish": 909,
|
|
"bullish": 818,
|
|
"neutral": 893
|
|
},
|
|
"model_path": "../models/candidates/xau_all_20260922.joblib",
|
|
"model_type": "domain_augmented_tfidf_logistic_regression",
|
|
"neutral_threshold": 0.05,
|
|
"sklearn_version": "1.9.1",
|
|
"trained_at": "2026-09-23T00:35:47.838884+00:00",
|
|
"training_examples": 2620
|
|
},
|
|
"model_latency_ms": {
|
|
"max": 4.03,
|
|
"mean": 0.9802564102564102,
|
|
"median": 0.935
|
|
},
|
|
"n": 78,
|
|
"per_category": {
|
|
"distractors": {
|
|
"deepseek": 8,
|
|
"deepseek_n": 8,
|
|
"keyword": 8,
|
|
"model": 5,
|
|
"n": 8
|
|
},
|
|
"dovish_fed": {
|
|
"deepseek": 8,
|
|
"deepseek_n": 8,
|
|
"keyword": 7,
|
|
"model": 4,
|
|
"n": 8
|
|
},
|
|
"gold_direct": {
|
|
"deepseek": 8,
|
|
"deepseek_n": 8,
|
|
"keyword": 4,
|
|
"model": 7,
|
|
"n": 8
|
|
},
|
|
"hawkish_fed": {
|
|
"deepseek": 8,
|
|
"deepseek_n": 8,
|
|
"keyword": 2,
|
|
"model": 8,
|
|
"n": 8
|
|
},
|
|
"inverted_metrics": {
|
|
"deepseek": 6,
|
|
"deepseek_n": 6,
|
|
"keyword": 0,
|
|
"model": 3,
|
|
"n": 6
|
|
},
|
|
"negation_traps": {
|
|
"deepseek": 8,
|
|
"deepseek_n": 8,
|
|
"keyword": 2,
|
|
"model": 1,
|
|
"n": 8
|
|
},
|
|
"real_yields": {
|
|
"deepseek": 6,
|
|
"deepseek_n": 6,
|
|
"keyword": 1,
|
|
"model": 3,
|
|
"n": 6
|
|
},
|
|
"risk_on": {
|
|
"deepseek": 4,
|
|
"deepseek_n": 4,
|
|
"keyword": 0,
|
|
"model": 0,
|
|
"n": 4
|
|
},
|
|
"safe_haven": {
|
|
"deepseek": 8,
|
|
"deepseek_n": 8,
|
|
"keyword": 0,
|
|
"model": 7,
|
|
"n": 8
|
|
},
|
|
"strong_usd": {
|
|
"deepseek": 7,
|
|
"deepseek_n": 7,
|
|
"keyword": 0,
|
|
"model": 5,
|
|
"n": 7
|
|
},
|
|
"weak_usd": {
|
|
"deepseek": 7,
|
|
"deepseek_n": 7,
|
|
"keyword": 0,
|
|
"model": 4,
|
|
"n": 7
|
|
}
|
|
},
|
|
"spearman_model_vs_deepseek": 0.4233409741735717
|
|
}
|