Harwinder Sekhon
01baef48ff
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked.
146 lines
2.9 KiB
JSON
146 lines
2.9 KiB
JSON
{
|
|
"db": "../newsdata.db",
|
|
"generated_at": "20260927_1907_transformer",
|
|
"in_sample_or_mixed": {
|
|
"headline": {
|
|
"accuracy": 0.7375805839969662,
|
|
"confusion": [
|
|
[
|
|
774,
|
|
84,
|
|
58
|
|
],
|
|
[
|
|
64,
|
|
715,
|
|
117
|
|
],
|
|
[
|
|
109,
|
|
260,
|
|
456
|
|
]
|
|
],
|
|
"directional_accuracy": 0.7064905226881103,
|
|
"directional_hits": 1230,
|
|
"directional_n": 1741,
|
|
"hits": 1945,
|
|
"macro_f1": 0.7295831004270902,
|
|
"mae": 0.30792290481607887,
|
|
"n": 2637,
|
|
"sign_agreement": 0.8804581245526127,
|
|
"sign_hits": 1230,
|
|
"sign_n": 1397,
|
|
"spearman": 0.7678844230729466
|
|
}
|
|
},
|
|
"model": {
|
|
"final": false,
|
|
"model_path": "/home/harwinder/Development/ML-NEWS/models/xau_transformer",
|
|
"model_type": "two_head_transformer:microsoft/deberta-v3-small",
|
|
"oos_from": "2026-09-18",
|
|
"relevance_threshold": 0.2,
|
|
"trained_at": "2026-09-27T19:03:25.366953+00:00",
|
|
"training_examples": 2402
|
|
},
|
|
"out_of_sample": {
|
|
"all": {
|
|
"accuracy": 0.6565557729941291,
|
|
"confusion": [
|
|
[
|
|
227,
|
|
36,
|
|
44
|
|
],
|
|
[
|
|
26,
|
|
266,
|
|
74
|
|
],
|
|
[
|
|
53,
|
|
118,
|
|
178
|
|
]
|
|
],
|
|
"directional_accuracy": 0.6173780487804879,
|
|
"directional_hits": 405,
|
|
"directional_n": 656,
|
|
"hits": 671,
|
|
"macro_f1": 0.6564675567772849,
|
|
"n": 1022,
|
|
"sign_agreement": 0.8067729083665338,
|
|
"sign_hits": 405,
|
|
"sign_n": 502
|
|
},
|
|
"gold": {
|
|
"accuracy": 0.7507987220447284,
|
|
"confusion": [
|
|
[
|
|
91,
|
|
4,
|
|
14
|
|
],
|
|
[
|
|
9,
|
|
31,
|
|
19
|
|
],
|
|
[
|
|
10,
|
|
22,
|
|
113
|
|
]
|
|
],
|
|
"directional_accuracy": 0.8031496062992126,
|
|
"directional_hits": 204,
|
|
"directional_n": 254,
|
|
"hits": 235,
|
|
"macro_f1": 0.7140550964455633,
|
|
"n": 313,
|
|
"sign_agreement": 0.8947368421052632,
|
|
"sign_hits": 204,
|
|
"sign_n": 228
|
|
},
|
|
"macro": {
|
|
"accuracy": 0.614950634696756,
|
|
"confusion": [
|
|
[
|
|
136,
|
|
32,
|
|
30
|
|
],
|
|
[
|
|
17,
|
|
235,
|
|
55
|
|
],
|
|
[
|
|
43,
|
|
96,
|
|
65
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5,
|
|
"directional_hits": 201,
|
|
"directional_n": 402,
|
|
"hits": 436,
|
|
"macro_f1": 0.5863598352269169,
|
|
"n": 709,
|
|
"sign_agreement": 0.7335766423357665,
|
|
"sign_hits": 201,
|
|
"sign_n": 274
|
|
}
|
|
},
|
|
"relevance_out_of_sample": {
|
|
"direction_accuracy": 0.8976377952755905,
|
|
"direction_n": 381,
|
|
"f1": 0.775881683731513,
|
|
"n": 881,
|
|
"precision": 0.6847389558232931,
|
|
"recall": 0.89501312335958
|
|
},
|
|
"rows": 2637,
|
|
"threshold": 0.05
|
|
}
|