Harwinder Sekhon
01baef48ff
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked.
190 lines
3.5 KiB
JSON
190 lines
3.5 KiB
JSON
{
|
|
"generated_at": "20260917_2322",
|
|
"gold_rows": 616,
|
|
"golden": {
|
|
"gold+macro (backfilled)": {
|
|
"accuracy": 0.5384615384615384,
|
|
"confusion": [
|
|
[
|
|
16,
|
|
4,
|
|
12
|
|
],
|
|
[
|
|
1,
|
|
6,
|
|
1
|
|
],
|
|
[
|
|
11,
|
|
7,
|
|
20
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5142857142857142,
|
|
"directional_hits": 36,
|
|
"directional_n": 70,
|
|
"hits": 42,
|
|
"macro_f1": 0.5255712050078247,
|
|
"n": 78,
|
|
"sign_agreement": 0.6101694915254238,
|
|
"sign_hits": 36,
|
|
"sign_n": 59
|
|
},
|
|
"gold-only (shipped data)": {
|
|
"accuracy": 0.5,
|
|
"confusion": [
|
|
[
|
|
13,
|
|
2,
|
|
17
|
|
],
|
|
[
|
|
2,
|
|
1,
|
|
5
|
|
],
|
|
[
|
|
12,
|
|
1,
|
|
25
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5428571428571428,
|
|
"directional_hits": 38,
|
|
"directional_n": 70,
|
|
"hits": 39,
|
|
"macro_f1": 0.3985266422953362,
|
|
"n": 78,
|
|
"sign_agreement": 0.5671641791044776,
|
|
"sign_hits": 38,
|
|
"sign_n": 67
|
|
}
|
|
},
|
|
"macro_label_distribution": {
|
|
"bearish": 350,
|
|
"bullish": 228,
|
|
"neutral": 383
|
|
},
|
|
"macro_rows": 961,
|
|
"pooled_oof": {
|
|
"gold/g+m": {
|
|
"accuracy": 0.6155202821869489,
|
|
"confusion": [
|
|
[
|
|
164,
|
|
27,
|
|
52
|
|
],
|
|
[
|
|
15,
|
|
74,
|
|
32
|
|
],
|
|
[
|
|
47,
|
|
45,
|
|
111
|
|
]
|
|
],
|
|
"directional_accuracy": 0.6165919282511211,
|
|
"directional_hits": 275,
|
|
"directional_n": 446,
|
|
"hits": 349,
|
|
"macro_f1": 0.6038188006599577,
|
|
"n": 567,
|
|
"sign_agreement": 0.7352941176470589,
|
|
"sign_hits": 275,
|
|
"sign_n": 374
|
|
},
|
|
"gold/g-only": {
|
|
"accuracy": 0.6437389770723104,
|
|
"confusion": [
|
|
[
|
|
165,
|
|
23,
|
|
55
|
|
],
|
|
[
|
|
13,
|
|
83,
|
|
25
|
|
],
|
|
[
|
|
40,
|
|
46,
|
|
117
|
|
]
|
|
],
|
|
"directional_accuracy": 0.6322869955156951,
|
|
"directional_hits": 282,
|
|
"directional_n": 446,
|
|
"hits": 365,
|
|
"macro_f1": 0.6362979163521462,
|
|
"n": 567,
|
|
"sign_agreement": 0.7480106100795756,
|
|
"sign_hits": 282,
|
|
"sign_n": 377
|
|
},
|
|
"macro/g+m": {
|
|
"accuracy": 0.5183129855715871,
|
|
"confusion": [
|
|
[
|
|
174,
|
|
127,
|
|
38
|
|
],
|
|
[
|
|
43,
|
|
244,
|
|
58
|
|
],
|
|
[
|
|
61,
|
|
107,
|
|
49
|
|
]
|
|
],
|
|
"directional_accuracy": 0.4010791366906475,
|
|
"directional_hits": 223,
|
|
"directional_n": 556,
|
|
"hits": 467,
|
|
"macro_f1": 0.47589676446146534,
|
|
"n": 901,
|
|
"sign_agreement": 0.6925465838509317,
|
|
"sign_hits": 223,
|
|
"sign_n": 322
|
|
},
|
|
"macro/g-only": {
|
|
"accuracy": 0.38401775804661487,
|
|
"confusion": [
|
|
[
|
|
137,
|
|
71,
|
|
131
|
|
],
|
|
[
|
|
70,
|
|
96,
|
|
179
|
|
],
|
|
[
|
|
65,
|
|
39,
|
|
113
|
|
]
|
|
],
|
|
"directional_accuracy": 0.44964028776978415,
|
|
"directional_hits": 250,
|
|
"directional_n": 556,
|
|
"hits": 346,
|
|
"macro_f1": 0.3833425073738865,
|
|
"n": 901,
|
|
"sign_agreement": 0.5605381165919282,
|
|
"sign_hits": 250,
|
|
"sign_n": 446
|
|
}
|
|
},
|
|
"threshold": 0.05
|
|
}
|