Harwinder Sekhon
1d72e4006a
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
146 lines
2.9 KiB
JSON
146 lines
2.9 KiB
JSON
{
|
|
"db": "../newsdata.db",
|
|
"generated_at": "20260927_2314_transformer",
|
|
"in_sample_or_mixed": {
|
|
"headline": {
|
|
"accuracy": 0.7671596511186954,
|
|
"confusion": [
|
|
[
|
|
777,
|
|
86,
|
|
53
|
|
],
|
|
[
|
|
73,
|
|
754,
|
|
69
|
|
],
|
|
[
|
|
87,
|
|
246,
|
|
492
|
|
]
|
|
],
|
|
"directional_accuracy": 0.7288914417001723,
|
|
"directional_hits": 1269,
|
|
"directional_n": 1741,
|
|
"hits": 2023,
|
|
"macro_f1": 0.7610986239900465,
|
|
"mae": 0.2995589685248388,
|
|
"n": 2637,
|
|
"sign_agreement": 0.9006387508871541,
|
|
"sign_hits": 1269,
|
|
"sign_n": 1409,
|
|
"spearman": 0.802027157757526
|
|
}
|
|
},
|
|
"model": {
|
|
"final": false,
|
|
"model_path": "/home/harwinder/Development/ML-NEWS/models/xau_transformer",
|
|
"model_type": "two_head_transformer:microsoft/deberta-v3-small",
|
|
"oos_from": "2026-09-18",
|
|
"relevance_threshold": 0.2,
|
|
"trained_at": "2026-09-27T23:12:37.970737+00:00",
|
|
"training_examples": 15416
|
|
},
|
|
"out_of_sample": {
|
|
"all": {
|
|
"accuracy": 0.7152641878669276,
|
|
"confusion": [
|
|
[
|
|
236,
|
|
37,
|
|
34
|
|
],
|
|
[
|
|
42,
|
|
292,
|
|
32
|
|
],
|
|
[
|
|
42,
|
|
104,
|
|
203
|
|
]
|
|
],
|
|
"directional_accuracy": 0.6692073170731707,
|
|
"directional_hits": 439,
|
|
"directional_n": 656,
|
|
"hits": 731,
|
|
"macro_f1": 0.7135542131452324,
|
|
"n": 1022,
|
|
"sign_agreement": 0.8524271844660194,
|
|
"sign_hits": 439,
|
|
"sign_n": 515
|
|
},
|
|
"gold": {
|
|
"accuracy": 0.7859424920127795,
|
|
"confusion": [
|
|
[
|
|
99,
|
|
1,
|
|
9
|
|
],
|
|
[
|
|
13,
|
|
32,
|
|
14
|
|
],
|
|
[
|
|
19,
|
|
11,
|
|
115
|
|
]
|
|
],
|
|
"directional_accuracy": 0.84251968503937,
|
|
"directional_hits": 214,
|
|
"directional_n": 254,
|
|
"hits": 246,
|
|
"macro_f1": 0.7530266904525025,
|
|
"n": 313,
|
|
"sign_agreement": 0.8842975206611571,
|
|
"sign_hits": 214,
|
|
"sign_n": 242
|
|
},
|
|
"macro": {
|
|
"accuracy": 0.6840620592383639,
|
|
"confusion": [
|
|
[
|
|
137,
|
|
36,
|
|
25
|
|
],
|
|
[
|
|
29,
|
|
260,
|
|
18
|
|
],
|
|
[
|
|
23,
|
|
93,
|
|
88
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5597014925373134,
|
|
"directional_hits": 225,
|
|
"directional_n": 402,
|
|
"hits": 485,
|
|
"macro_f1": 0.6601699690090934,
|
|
"n": 709,
|
|
"sign_agreement": 0.8241758241758241,
|
|
"sign_hits": 225,
|
|
"sign_n": 273
|
|
}
|
|
},
|
|
"relevance_out_of_sample": {
|
|
"direction_accuracy": 0.8923884514435696,
|
|
"direction_n": 381,
|
|
"f1": 0.8143021914648213,
|
|
"n": 881,
|
|
"precision": 0.7263374485596708,
|
|
"recall": 0.926509186351706
|
|
},
|
|
"rows": 2637,
|
|
"threshold": 0.05
|
|
}
|