Harwinder Sekhon
1d72e4006a
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
146 lines
2.9 KiB
JSON
146 lines
2.9 KiB
JSON
{
|
|
"db": "../newsdata.db",
|
|
"generated_at": "20260927_1947_transformer",
|
|
"in_sample_or_mixed": {
|
|
"headline": {
|
|
"accuracy": 0.7724687144482366,
|
|
"confusion": [
|
|
[
|
|
773,
|
|
97,
|
|
46
|
|
],
|
|
[
|
|
49,
|
|
784,
|
|
63
|
|
],
|
|
[
|
|
78,
|
|
267,
|
|
480
|
|
]
|
|
],
|
|
"directional_accuracy": 0.7197013210798392,
|
|
"directional_hits": 1253,
|
|
"directional_n": 1741,
|
|
"hits": 2037,
|
|
"macro_f1": 0.7657899696449985,
|
|
"mae": 0.2876833522942738,
|
|
"n": 2637,
|
|
"sign_agreement": 0.9099491648511256,
|
|
"sign_hits": 1253,
|
|
"sign_n": 1377,
|
|
"spearman": 0.8074118582323724
|
|
}
|
|
},
|
|
"model": {
|
|
"final": false,
|
|
"model_path": "/home/harwinder/Development/ML-NEWS/models/xau_transformer",
|
|
"model_type": "two_head_transformer:microsoft/deberta-v3-small",
|
|
"oos_from": "2026-09-18",
|
|
"relevance_threshold": 0.2,
|
|
"trained_at": "2026-09-27T19:46:15.509042+00:00",
|
|
"training_examples": 10690
|
|
},
|
|
"out_of_sample": {
|
|
"all": {
|
|
"accuracy": 0.7181996086105675,
|
|
"confusion": [
|
|
[
|
|
229,
|
|
45,
|
|
33
|
|
],
|
|
[
|
|
21,
|
|
308,
|
|
37
|
|
],
|
|
[
|
|
38,
|
|
114,
|
|
197
|
|
]
|
|
],
|
|
"directional_accuracy": 0.649390243902439,
|
|
"directional_hits": 426,
|
|
"directional_n": 656,
|
|
"hits": 734,
|
|
"macro_f1": 0.7162846956964604,
|
|
"n": 1022,
|
|
"sign_agreement": 0.8571428571428571,
|
|
"sign_hits": 426,
|
|
"sign_n": 497
|
|
},
|
|
"gold": {
|
|
"accuracy": 0.8306709265175719,
|
|
"confusion": [
|
|
[
|
|
100,
|
|
1,
|
|
8
|
|
],
|
|
[
|
|
9,
|
|
40,
|
|
10
|
|
],
|
|
[
|
|
13,
|
|
12,
|
|
120
|
|
]
|
|
],
|
|
"directional_accuracy": 0.8661417322834646,
|
|
"directional_hits": 220,
|
|
"directional_n": 254,
|
|
"hits": 260,
|
|
"macro_f1": 0.8093810390630178,
|
|
"n": 313,
|
|
"sign_agreement": 0.9128630705394191,
|
|
"sign_hits": 220,
|
|
"sign_n": 241
|
|
},
|
|
"macro": {
|
|
"accuracy": 0.6685472496473907,
|
|
"confusion": [
|
|
[
|
|
129,
|
|
44,
|
|
25
|
|
],
|
|
[
|
|
12,
|
|
268,
|
|
27
|
|
],
|
|
[
|
|
25,
|
|
102,
|
|
77
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5124378109452736,
|
|
"directional_hits": 206,
|
|
"directional_n": 402,
|
|
"hits": 474,
|
|
"macro_f1": 0.6382218663772061,
|
|
"n": 709,
|
|
"sign_agreement": 0.8046875,
|
|
"sign_hits": 206,
|
|
"sign_n": 256
|
|
}
|
|
},
|
|
"relevance_out_of_sample": {
|
|
"direction_accuracy": 0.931758530183727,
|
|
"direction_n": 381,
|
|
"f1": 0.8374851720047449,
|
|
"n": 881,
|
|
"precision": 0.7640692640692641,
|
|
"recall": 0.926509186351706
|
|
},
|
|
"rows": 2637,
|
|
"threshold": 0.05
|
|
}
|