harry/News-ML-Decoder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial import: XAUUSD news sentiment models, evals, and data tooling

Browse Source 
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the
two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py)
with its Google News crawler and DeepSeek labeller (data_collect/), the
golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27
results and progress notes.

Transformer weights (model.pt, model.onnx) and the crawl database are not
tracked; rebuild with transformer/train.py and transformer/export_onnx.py.
The DeepSeek label cache in data/ is tracked.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Harwinder Sekhon
2026-09-27 19:22:30 -04:00
commit 1d72e4006a
98 changed files with 1719757 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
eval/results/live_20260917_2241.json
+449
View File
@@ -0,0 +1,449 @@
{
"deepseek_self_agreement": {
"accuracy": 0.7,
"confusion": [
[
4,
0,
1
],
[
0,
0,
0
],
[
1,
1,
3
]
],
"directional_accuracy": 0.7,
"directional_hits": 7,
"directional_n": 10,
"hits": 7,
"identical": 2,
"macro_f1": 0.48888888888888893,
"mae": 0.25,
"n": 10,
"sign_agreement": 0.7777777777777778,
"sign_hits": 7,
"sign_n": 9,
"spearman": 0.5569257132640542
},
"generated_at": "20260917_2241",
"latency": {
"deepseek_ms_per_headline": 56.414902748947796,
"model_ms_mean": 1.0686842105263157
},
"log_rows": 76,
"n": 76,
"subsets": {
"all headlines": {
"keyword": {
"accuracy": 0.3815789473684211,
"confusion": [
[
9,
17,
0
],
[
0,
16,
0
],
[
2,
28,
4
]
],
"directional_accuracy": 0.21666666666666667,
"directional_hits": 13,
"directional_n": 60,
"hits": 29,
"macro_f1": 0.37086573928679195,
"n": 76,
"sign_agreement": 0.8666666666666667,
"sign_hits": 13,
"sign_n": 15
},
"model_full": {
"accuracy": 0.6052631578947368,
"confusion": [
[
16,
3,
7
],
[
5,
6,
5
],
[
7,
3,
24
]
],
"directional_accuracy": 0.6666666666666666,
"directional_hits": 40,
"directional_n": 60,
"hits": 46,
"macro_f1": 0.5689594356261023,
"mae": 0.33246710526315787,
"n": 76,
"sign_agreement": 0.7407407407407407,
"sign_hits": 40,
"sign_n": 54,
"spearman": 0.4808996156923549
},
"model_head": {
"accuracy": 0.6842105263157895,
"confusion": [
[
20,
0,
6
],
[
2,
6,
8
],
[
6,
2,
26
]
],
"directional_accuracy": 0.7666666666666667,
"directional_hits": 46,
"directional_n": 60,
"hits": 52,
"macro_f1": 0.6478144811478145,
"mae": 0.3404934210526316,
"n": 76,
"sign_agreement": 0.7931034482758621,
"sign_hits": 46,
"sign_n": 58,
"spearman": 0.5639368037865334
},
"n": 76
},
"mentions gold (XAU keyword)": {
"keyword": {
"accuracy": 0.3404255319148936,
"confusion": [
[
5,
6,
0
],
[
0,
9,
0
],
[
0,
25,
2
]
],
"directional_accuracy": 0.18421052631578946,
"directional_hits": 7,
"directional_n": 38,
"hits": 16,
"macro_f1": 0.37675932441942295,
"n": 47,
"sign_agreement": 1.0,
"sign_hits": 7,
"sign_n": 7
},
"model_full": {
"accuracy": 0.723404255319149,
"confusion": [
[
10,
0,
1
],
[
2,
4,
3
],
[
4,
3,
20
]
],
"directional_accuracy": 0.7894736842105263,
"directional_hits": 30,
"directional_n": 38,
"hits": 34,
"macro_f1": 0.6750181554103122,
"mae": 0.2999148936170213,
"n": 47,
"sign_agreement": 0.8571428571428571,
"sign_hits": 30,
"sign_n": 35,
"spearman": 0.5816073836387256
},
"model_head": {
"accuracy": 0.7659574468085106,
"confusion": [
[
10,
0,
1
],
[
2,
4,
3
],
[
3,
2,
22
]
],
"directional_accuracy": 0.8421052631578947,
"directional_hits": 32,
"directional_n": 38,
"hits": 36,
"macro_f1": 0.7109175939364617,
"mae": 0.302268085106383,
"n": 47,
"sign_agreement": 0.8888888888888888,
"sign_hits": 32,
"sign_n": 36,
"spearman": 0.6912522895796331
},
"n": 47
},
"not in training & mentions gold": {
"keyword": {
"accuracy": 0.35135135135135137,
"confusion": [
[
3,
3,
0
],
[
0,
8,
0
],
[
0,
21,
2
]
],
"directional_accuracy": 0.1724137931034483,
"directional_hits": 5,
"directional_n": 29,
"hits": 13,
"macro_f1": 0.40888888888888886,
"n": 37,
"sign_agreement": 1.0,
"sign_hits": 5,
"sign_n": 5
},
"model_full": {
"accuracy": 0.7027027027027027,
"confusion": [
[
6,
0,
0
],
[
2,
4,
2
],
[
4,
3,
16
]
],
"directional_accuracy": 0.7586206896551724,
"directional_hits": 22,
"directional_n": 29,
"hits": 26,
"macro_f1": 0.6601626016260163,
"mae": 0.25864864864864867,
"n": 37,
"sign_agreement": 0.8461538461538461,
"sign_hits": 22,
"sign_n": 26,
"spearman": 0.5732166454312866
},
"model_head": {
"accuracy": 0.7567567567567568,
"confusion": [
[
6,
0,
0
],
[
2,
4,
2
],
[
3,
2,
18
]
],
"directional_accuracy": 0.8275862068965517,
"directional_hits": 24,
"directional_n": 29,
"hits": 28,
"macro_f1": 0.7048400755651097,
"mae": 0.2563864864864865,
"n": 37,
"sign_agreement": 0.8888888888888888,
"sign_hits": 24,
"sign_n": 27,
"spearman": 0.6927913852069925
},
"n": 37
},
"not in training set": {
"keyword": {
"accuracy": 0.3939393939393939,
"confusion": [
[
7,
14,
0
],
[
0,
15,
0
],
[
2,
24,
4
]
],
"directional_accuracy": 0.21568627450980393,
"directional_hits": 11,
"directional_n": 51,
"hits": 26,
"macro_f1": 0.3810457516339869,
"n": 66,
"sign_agreement": 0.8461538461538461,
"sign_hits": 11,
"sign_n": 13
},
"model_full": {
"accuracy": 0.5757575757575758,
"confusion": [
[
12,
3,
6
],
[
5,
6,
4
],
[
7,
3,
20
]
],
"directional_accuracy": 0.6274509803921569,
"directional_hits": 32,
"directional_n": 51,
"hits": 38,
"macro_f1": 0.5481481481481482,
"mae": 0.3142651515151515,
"n": 66,
"sign_agreement": 0.7111111111111111,
"sign_hits": 32,
"sign_n": 45,
"spearman": 0.4561157459616647
},
"model_head": {
"accuracy": 0.6666666666666666,
"confusion": [
[
16,
0,
5
],
[
2,
6,
7
],
[
6,
2,
22
]
],
"directional_accuracy": 0.7450980392156863,
"directional_hits": 38,
"directional_n": 51,
"hits": 44,
"macro_f1": 0.6401167471819647,
"mae": 0.3205636363636364,
"n": 66,
"sign_agreement": 0.7755102040816326,
"sign_hits": 38,
"sign_n": 49,
"spearman": 0.5415716725239331
},
"n": 66
}
},
"threshold": 0.05,
"usd_only_display_vs_xau": {
"accuracy": 0.875,
"confusion": [
[
9,
0,
0
],
[
0,
3,
0
],
[
2,
0,
2
]
],
"directional_accuracy": 0.8461538461538461,
"directional_hits": 11,
"directional_n": 13,
"hits": 14,
"macro_f1": 0.8555555555555555,
"n": 16,
"sign_agreement": 0.8461538461538461,
"sign_hits": 11,
"sign_n": 13
}
}