Harwinder Sekhon
1d72e4006a
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
146 lines
2.9 KiB
JSON
146 lines
2.9 KiB
JSON
{
|
|
"db": "../newsdata.db",
|
|
"generated_at": "20260927_1900_transformer",
|
|
"in_sample_or_mixed": {
|
|
"headline": {
|
|
"accuracy": 0.7133105802047781,
|
|
"confusion": [
|
|
[
|
|
769,
|
|
95,
|
|
52
|
|
],
|
|
[
|
|
95,
|
|
677,
|
|
124
|
|
],
|
|
[
|
|
140,
|
|
250,
|
|
435
|
|
]
|
|
],
|
|
"directional_accuracy": 0.691556576680069,
|
|
"directional_hits": 1204,
|
|
"directional_n": 1741,
|
|
"hits": 1881,
|
|
"macro_f1": 0.7042783133948399,
|
|
"mae": 0.24411205915813428,
|
|
"n": 2637,
|
|
"sign_agreement": 0.8624641833810889,
|
|
"sign_hits": 1204,
|
|
"sign_n": 1396,
|
|
"spearman": 0.7415919015733857
|
|
}
|
|
},
|
|
"model": {
|
|
"final": false,
|
|
"model_path": "/home/harwinder/Development/ML-NEWS/models/xau_transformer",
|
|
"model_type": "two_head_transformer:microsoft/deberta-v3-small",
|
|
"oos_from": "2026-09-18",
|
|
"relevance_threshold": 0.2,
|
|
"trained_at": "2026-09-27T18:58:27.596998+00:00",
|
|
"training_examples": 2402
|
|
},
|
|
"out_of_sample": {
|
|
"all": {
|
|
"accuracy": 0.6438356164383562,
|
|
"confusion": [
|
|
[
|
|
227,
|
|
40,
|
|
40
|
|
],
|
|
[
|
|
29,
|
|
270,
|
|
67
|
|
],
|
|
[
|
|
66,
|
|
122,
|
|
161
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5914634146341463,
|
|
"directional_hits": 388,
|
|
"directional_n": 656,
|
|
"hits": 658,
|
|
"macro_f1": 0.6401174660955584,
|
|
"n": 1022,
|
|
"sign_agreement": 0.7854251012145749,
|
|
"sign_hits": 388,
|
|
"sign_n": 494
|
|
},
|
|
"gold": {
|
|
"accuracy": 0.744408945686901,
|
|
"confusion": [
|
|
[
|
|
91,
|
|
5,
|
|
13
|
|
],
|
|
[
|
|
7,
|
|
33,
|
|
19
|
|
],
|
|
[
|
|
13,
|
|
23,
|
|
109
|
|
]
|
|
],
|
|
"directional_accuracy": 0.7874015748031497,
|
|
"directional_hits": 200,
|
|
"directional_n": 254,
|
|
"hits": 233,
|
|
"macro_f1": 0.7131701631701631,
|
|
"n": 313,
|
|
"sign_agreement": 0.8849557522123894,
|
|
"sign_hits": 200,
|
|
"sign_n": 226
|
|
},
|
|
"macro": {
|
|
"accuracy": 0.5994358251057827,
|
|
"confusion": [
|
|
[
|
|
136,
|
|
35,
|
|
27
|
|
],
|
|
[
|
|
22,
|
|
237,
|
|
48
|
|
],
|
|
[
|
|
53,
|
|
99,
|
|
52
|
|
]
|
|
],
|
|
"directional_accuracy": 0.46766169154228854,
|
|
"directional_hits": 188,
|
|
"directional_n": 402,
|
|
"hits": 425,
|
|
"macro_f1": 0.559450371611602,
|
|
"n": 709,
|
|
"sign_agreement": 0.7014925373134329,
|
|
"sign_hits": 188,
|
|
"sign_n": 268
|
|
}
|
|
},
|
|
"relevance_out_of_sample": {
|
|
"direction_accuracy": 0.8871391076115486,
|
|
"direction_n": 381,
|
|
"f1": 0.7733026467203681,
|
|
"n": 881,
|
|
"precision": 0.6885245901639344,
|
|
"recall": 0.8818897637795275
|
|
},
|
|
"rows": 2637,
|
|
"threshold": 0.05
|
|
}
|