Harwinder Sekhon
01baef48ff
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked.
146 lines
2.9 KiB
JSON
146 lines
2.9 KiB
JSON
{
|
|
"db": "../newsdata.db",
|
|
"generated_at": "20260927_1927_transformer",
|
|
"in_sample_or_mixed": {
|
|
"headline": {
|
|
"accuracy": 0.7682973075464543,
|
|
"confusion": [
|
|
[
|
|
764,
|
|
117,
|
|
35
|
|
],
|
|
[
|
|
42,
|
|
821,
|
|
33
|
|
],
|
|
[
|
|
82,
|
|
302,
|
|
441
|
|
]
|
|
],
|
|
"directional_accuracy": 0.6921309592188397,
|
|
"directional_hits": 1205,
|
|
"directional_n": 1741,
|
|
"hits": 2026,
|
|
"macro_f1": 0.7589675529791181,
|
|
"mae": 0.2953307546454304,
|
|
"n": 2637,
|
|
"sign_agreement": 0.9114977307110439,
|
|
"sign_hits": 1205,
|
|
"sign_n": 1322,
|
|
"spearman": 0.7978166855256467
|
|
}
|
|
},
|
|
"model": {
|
|
"final": false,
|
|
"model_path": "/home/harwinder/Development/ML-NEWS/models/xau_transformer",
|
|
"model_type": "two_head_transformer:microsoft/deberta-v3-small",
|
|
"oos_from": "2026-09-18",
|
|
"relevance_threshold": 0.6,
|
|
"trained_at": "2026-09-27T19:22:03.286521+00:00",
|
|
"training_examples": 7460
|
|
},
|
|
"out_of_sample": {
|
|
"all": {
|
|
"accuracy": 0.7191780821917808,
|
|
"confusion": [
|
|
[
|
|
231,
|
|
52,
|
|
24
|
|
],
|
|
[
|
|
18,
|
|
327,
|
|
21
|
|
],
|
|
[
|
|
37,
|
|
135,
|
|
177
|
|
]
|
|
],
|
|
"directional_accuracy": 0.6219512195121951,
|
|
"directional_hits": 408,
|
|
"directional_n": 656,
|
|
"hits": 735,
|
|
"macro_f1": 0.7140787226553597,
|
|
"n": 1022,
|
|
"sign_agreement": 0.8699360341151386,
|
|
"sign_hits": 408,
|
|
"sign_n": 469
|
|
},
|
|
"gold": {
|
|
"accuracy": 0.8370607028753994,
|
|
"confusion": [
|
|
[
|
|
102,
|
|
2,
|
|
5
|
|
],
|
|
[
|
|
8,
|
|
46,
|
|
5
|
|
],
|
|
[
|
|
9,
|
|
22,
|
|
114
|
|
]
|
|
],
|
|
"directional_accuracy": 0.8503937007874016,
|
|
"directional_hits": 216,
|
|
"directional_n": 254,
|
|
"hits": 262,
|
|
"macro_f1": 0.8184995932671942,
|
|
"n": 313,
|
|
"sign_agreement": 0.9391304347826087,
|
|
"sign_hits": 216,
|
|
"sign_n": 230
|
|
},
|
|
"macro": {
|
|
"accuracy": 0.6671368124118476,
|
|
"confusion": [
|
|
[
|
|
129,
|
|
50,
|
|
19
|
|
],
|
|
[
|
|
10,
|
|
281,
|
|
16
|
|
],
|
|
[
|
|
28,
|
|
113,
|
|
63
|
|
]
|
|
],
|
|
"directional_accuracy": 0.47761194029850745,
|
|
"directional_hits": 192,
|
|
"directional_n": 402,
|
|
"hits": 473,
|
|
"macro_f1": 0.6241344702371295,
|
|
"n": 709,
|
|
"sign_agreement": 0.803347280334728,
|
|
"sign_hits": 192,
|
|
"sign_n": 239
|
|
}
|
|
},
|
|
"relevance_out_of_sample": {
|
|
"direction_accuracy": 0.9422572178477691,
|
|
"direction_n": 381,
|
|
"f1": 0.8414481897627966,
|
|
"n": 881,
|
|
"precision": 0.8023809523809524,
|
|
"recall": 0.884514435695538
|
|
},
|
|
"rows": 2637,
|
|
"threshold": 0.05
|
|
}
|