Harwinder Sekhon
01baef48ff
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked.
450 lines
8.9 KiB
JSON
450 lines
8.9 KiB
JSON
{
|
|
"deepseek_self_agreement": {
|
|
"accuracy": 0.7,
|
|
"confusion": [
|
|
[
|
|
4,
|
|
0,
|
|
1
|
|
],
|
|
[
|
|
0,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
1,
|
|
1,
|
|
3
|
|
]
|
|
],
|
|
"directional_accuracy": 0.7,
|
|
"directional_hits": 7,
|
|
"directional_n": 10,
|
|
"hits": 7,
|
|
"identical": 2,
|
|
"macro_f1": 0.48888888888888893,
|
|
"mae": 0.25,
|
|
"n": 10,
|
|
"sign_agreement": 0.7777777777777778,
|
|
"sign_hits": 7,
|
|
"sign_n": 9,
|
|
"spearman": 0.5569257132640542
|
|
},
|
|
"generated_at": "20260917_2241",
|
|
"latency": {
|
|
"deepseek_ms_per_headline": 56.414902748947796,
|
|
"model_ms_mean": 1.0686842105263157
|
|
},
|
|
"log_rows": 76,
|
|
"n": 76,
|
|
"subsets": {
|
|
"all headlines": {
|
|
"keyword": {
|
|
"accuracy": 0.3815789473684211,
|
|
"confusion": [
|
|
[
|
|
9,
|
|
17,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
16,
|
|
0
|
|
],
|
|
[
|
|
2,
|
|
28,
|
|
4
|
|
]
|
|
],
|
|
"directional_accuracy": 0.21666666666666667,
|
|
"directional_hits": 13,
|
|
"directional_n": 60,
|
|
"hits": 29,
|
|
"macro_f1": 0.37086573928679195,
|
|
"n": 76,
|
|
"sign_agreement": 0.8666666666666667,
|
|
"sign_hits": 13,
|
|
"sign_n": 15
|
|
},
|
|
"model_full": {
|
|
"accuracy": 0.6052631578947368,
|
|
"confusion": [
|
|
[
|
|
16,
|
|
3,
|
|
7
|
|
],
|
|
[
|
|
5,
|
|
6,
|
|
5
|
|
],
|
|
[
|
|
7,
|
|
3,
|
|
24
|
|
]
|
|
],
|
|
"directional_accuracy": 0.6666666666666666,
|
|
"directional_hits": 40,
|
|
"directional_n": 60,
|
|
"hits": 46,
|
|
"macro_f1": 0.5689594356261023,
|
|
"mae": 0.33246710526315787,
|
|
"n": 76,
|
|
"sign_agreement": 0.7407407407407407,
|
|
"sign_hits": 40,
|
|
"sign_n": 54,
|
|
"spearman": 0.4808996156923549
|
|
},
|
|
"model_head": {
|
|
"accuracy": 0.6842105263157895,
|
|
"confusion": [
|
|
[
|
|
20,
|
|
0,
|
|
6
|
|
],
|
|
[
|
|
2,
|
|
6,
|
|
8
|
|
],
|
|
[
|
|
6,
|
|
2,
|
|
26
|
|
]
|
|
],
|
|
"directional_accuracy": 0.7666666666666667,
|
|
"directional_hits": 46,
|
|
"directional_n": 60,
|
|
"hits": 52,
|
|
"macro_f1": 0.6478144811478145,
|
|
"mae": 0.3404934210526316,
|
|
"n": 76,
|
|
"sign_agreement": 0.7931034482758621,
|
|
"sign_hits": 46,
|
|
"sign_n": 58,
|
|
"spearman": 0.5639368037865334
|
|
},
|
|
"n": 76
|
|
},
|
|
"mentions gold (XAU keyword)": {
|
|
"keyword": {
|
|
"accuracy": 0.3404255319148936,
|
|
"confusion": [
|
|
[
|
|
5,
|
|
6,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
9,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
25,
|
|
2
|
|
]
|
|
],
|
|
"directional_accuracy": 0.18421052631578946,
|
|
"directional_hits": 7,
|
|
"directional_n": 38,
|
|
"hits": 16,
|
|
"macro_f1": 0.37675932441942295,
|
|
"n": 47,
|
|
"sign_agreement": 1.0,
|
|
"sign_hits": 7,
|
|
"sign_n": 7
|
|
},
|
|
"model_full": {
|
|
"accuracy": 0.723404255319149,
|
|
"confusion": [
|
|
[
|
|
10,
|
|
0,
|
|
1
|
|
],
|
|
[
|
|
2,
|
|
4,
|
|
3
|
|
],
|
|
[
|
|
4,
|
|
3,
|
|
20
|
|
]
|
|
],
|
|
"directional_accuracy": 0.7894736842105263,
|
|
"directional_hits": 30,
|
|
"directional_n": 38,
|
|
"hits": 34,
|
|
"macro_f1": 0.6750181554103122,
|
|
"mae": 0.2999148936170213,
|
|
"n": 47,
|
|
"sign_agreement": 0.8571428571428571,
|
|
"sign_hits": 30,
|
|
"sign_n": 35,
|
|
"spearman": 0.5816073836387256
|
|
},
|
|
"model_head": {
|
|
"accuracy": 0.7659574468085106,
|
|
"confusion": [
|
|
[
|
|
10,
|
|
0,
|
|
1
|
|
],
|
|
[
|
|
2,
|
|
4,
|
|
3
|
|
],
|
|
[
|
|
3,
|
|
2,
|
|
22
|
|
]
|
|
],
|
|
"directional_accuracy": 0.8421052631578947,
|
|
"directional_hits": 32,
|
|
"directional_n": 38,
|
|
"hits": 36,
|
|
"macro_f1": 0.7109175939364617,
|
|
"mae": 0.302268085106383,
|
|
"n": 47,
|
|
"sign_agreement": 0.8888888888888888,
|
|
"sign_hits": 32,
|
|
"sign_n": 36,
|
|
"spearman": 0.6912522895796331
|
|
},
|
|
"n": 47
|
|
},
|
|
"not in training & mentions gold": {
|
|
"keyword": {
|
|
"accuracy": 0.35135135135135137,
|
|
"confusion": [
|
|
[
|
|
3,
|
|
3,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
8,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
21,
|
|
2
|
|
]
|
|
],
|
|
"directional_accuracy": 0.1724137931034483,
|
|
"directional_hits": 5,
|
|
"directional_n": 29,
|
|
"hits": 13,
|
|
"macro_f1": 0.40888888888888886,
|
|
"n": 37,
|
|
"sign_agreement": 1.0,
|
|
"sign_hits": 5,
|
|
"sign_n": 5
|
|
},
|
|
"model_full": {
|
|
"accuracy": 0.7027027027027027,
|
|
"confusion": [
|
|
[
|
|
6,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
2,
|
|
4,
|
|
2
|
|
],
|
|
[
|
|
4,
|
|
3,
|
|
16
|
|
]
|
|
],
|
|
"directional_accuracy": 0.7586206896551724,
|
|
"directional_hits": 22,
|
|
"directional_n": 29,
|
|
"hits": 26,
|
|
"macro_f1": 0.6601626016260163,
|
|
"mae": 0.25864864864864867,
|
|
"n": 37,
|
|
"sign_agreement": 0.8461538461538461,
|
|
"sign_hits": 22,
|
|
"sign_n": 26,
|
|
"spearman": 0.5732166454312866
|
|
},
|
|
"model_head": {
|
|
"accuracy": 0.7567567567567568,
|
|
"confusion": [
|
|
[
|
|
6,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
2,
|
|
4,
|
|
2
|
|
],
|
|
[
|
|
3,
|
|
2,
|
|
18
|
|
]
|
|
],
|
|
"directional_accuracy": 0.8275862068965517,
|
|
"directional_hits": 24,
|
|
"directional_n": 29,
|
|
"hits": 28,
|
|
"macro_f1": 0.7048400755651097,
|
|
"mae": 0.2563864864864865,
|
|
"n": 37,
|
|
"sign_agreement": 0.8888888888888888,
|
|
"sign_hits": 24,
|
|
"sign_n": 27,
|
|
"spearman": 0.6927913852069925
|
|
},
|
|
"n": 37
|
|
},
|
|
"not in training set": {
|
|
"keyword": {
|
|
"accuracy": 0.3939393939393939,
|
|
"confusion": [
|
|
[
|
|
7,
|
|
14,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
15,
|
|
0
|
|
],
|
|
[
|
|
2,
|
|
24,
|
|
4
|
|
]
|
|
],
|
|
"directional_accuracy": 0.21568627450980393,
|
|
"directional_hits": 11,
|
|
"directional_n": 51,
|
|
"hits": 26,
|
|
"macro_f1": 0.3810457516339869,
|
|
"n": 66,
|
|
"sign_agreement": 0.8461538461538461,
|
|
"sign_hits": 11,
|
|
"sign_n": 13
|
|
},
|
|
"model_full": {
|
|
"accuracy": 0.5757575757575758,
|
|
"confusion": [
|
|
[
|
|
12,
|
|
3,
|
|
6
|
|
],
|
|
[
|
|
5,
|
|
6,
|
|
4
|
|
],
|
|
[
|
|
7,
|
|
3,
|
|
20
|
|
]
|
|
],
|
|
"directional_accuracy": 0.6274509803921569,
|
|
"directional_hits": 32,
|
|
"directional_n": 51,
|
|
"hits": 38,
|
|
"macro_f1": 0.5481481481481482,
|
|
"mae": 0.3142651515151515,
|
|
"n": 66,
|
|
"sign_agreement": 0.7111111111111111,
|
|
"sign_hits": 32,
|
|
"sign_n": 45,
|
|
"spearman": 0.4561157459616647
|
|
},
|
|
"model_head": {
|
|
"accuracy": 0.6666666666666666,
|
|
"confusion": [
|
|
[
|
|
16,
|
|
0,
|
|
5
|
|
],
|
|
[
|
|
2,
|
|
6,
|
|
7
|
|
],
|
|
[
|
|
6,
|
|
2,
|
|
22
|
|
]
|
|
],
|
|
"directional_accuracy": 0.7450980392156863,
|
|
"directional_hits": 38,
|
|
"directional_n": 51,
|
|
"hits": 44,
|
|
"macro_f1": 0.6401167471819647,
|
|
"mae": 0.3205636363636364,
|
|
"n": 66,
|
|
"sign_agreement": 0.7755102040816326,
|
|
"sign_hits": 38,
|
|
"sign_n": 49,
|
|
"spearman": 0.5415716725239331
|
|
},
|
|
"n": 66
|
|
}
|
|
},
|
|
"threshold": 0.05,
|
|
"usd_only_display_vs_xau": {
|
|
"accuracy": 0.875,
|
|
"confusion": [
|
|
[
|
|
9,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
3,
|
|
0
|
|
],
|
|
[
|
|
2,
|
|
0,
|
|
2
|
|
]
|
|
],
|
|
"directional_accuracy": 0.8461538461538461,
|
|
"directional_hits": 11,
|
|
"directional_n": 13,
|
|
"hits": 14,
|
|
"macro_f1": 0.8555555555555555,
|
|
"n": 16,
|
|
"sign_agreement": 0.8461538461538461,
|
|
"sign_hits": 11,
|
|
"sign_n": 13
|
|
}
|
|
}
|