Harwinder Sekhon
1d72e4006a
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1886 lines
40 KiB
JSON
1886 lines
40 KiB
JSON
{
|
|
"after_training": {
|
|
"all": {
|
|
"accuracy": 0.45872420262664165,
|
|
"confusion": [
|
|
[
|
|
186,
|
|
27,
|
|
106
|
|
],
|
|
[
|
|
74,
|
|
109,
|
|
192
|
|
],
|
|
[
|
|
129,
|
|
49,
|
|
194
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5499276410998553,
|
|
"directional_hits": 380,
|
|
"directional_n": 691,
|
|
"hits": 489,
|
|
"macro_f1": 0.4545945057244492,
|
|
"n": 1066,
|
|
"sign_agreement": 0.6178861788617886,
|
|
"sign_hits": 380,
|
|
"sign_n": 615
|
|
},
|
|
"gold": {
|
|
"accuracy": 0.6822429906542056,
|
|
"confusion": [
|
|
[
|
|
84,
|
|
7,
|
|
19
|
|
],
|
|
[
|
|
8,
|
|
35,
|
|
16
|
|
],
|
|
[
|
|
42,
|
|
10,
|
|
100
|
|
]
|
|
],
|
|
"directional_accuracy": 0.7022900763358778,
|
|
"directional_hits": 184,
|
|
"directional_n": 262,
|
|
"hits": 219,
|
|
"macro_f1": 0.6720064440976077,
|
|
"n": 321,
|
|
"sign_agreement": 0.7510204081632653,
|
|
"sign_hits": 184,
|
|
"sign_n": 245
|
|
},
|
|
"macro": {
|
|
"accuracy": 0.3624161073825503,
|
|
"confusion": [
|
|
[
|
|
102,
|
|
20,
|
|
87
|
|
],
|
|
[
|
|
66,
|
|
74,
|
|
176
|
|
],
|
|
[
|
|
87,
|
|
39,
|
|
94
|
|
]
|
|
],
|
|
"directional_accuracy": 0.4568764568764569,
|
|
"directional_hits": 196,
|
|
"directional_n": 429,
|
|
"hits": 270,
|
|
"macro_f1": 0.36503325894343636,
|
|
"n": 745,
|
|
"sign_agreement": 0.5297297297297298,
|
|
"sign_hits": 196,
|
|
"sign_n": 370
|
|
}
|
|
},
|
|
"days": [
|
|
"2026-09-08",
|
|
"2026-09-09",
|
|
"2026-09-10",
|
|
"2026-09-11",
|
|
"2026-09-13",
|
|
"2026-09-14",
|
|
"2026-09-15",
|
|
"2026-09-16",
|
|
"2026-09-17",
|
|
"2026-09-18",
|
|
"2026-09-20",
|
|
"2026-09-21",
|
|
"2026-09-22"
|
|
],
|
|
"db": "../newsdata.db",
|
|
"extra_labels": "results/backfill_labels.jsonl",
|
|
"folds": [
|
|
{
|
|
"day": "2026-09-09",
|
|
"fit_seconds_full": 1.1599936260004142,
|
|
"fit_seconds_head": 0.02371588599999086,
|
|
"full": {
|
|
"accuracy": 0.5,
|
|
"confusion": [
|
|
[
|
|
4,
|
|
6,
|
|
0
|
|
],
|
|
[
|
|
3,
|
|
10,
|
|
0
|
|
],
|
|
[
|
|
5,
|
|
4,
|
|
4
|
|
]
|
|
],
|
|
"directional_accuracy": 0.34782608695652173,
|
|
"directional_hits": 8,
|
|
"directional_n": 23,
|
|
"hits": 18,
|
|
"macro_f1": 0.4800950683303624,
|
|
"n": 36,
|
|
"sign_agreement": 0.6153846153846154,
|
|
"sign_hits": 8,
|
|
"sign_n": 13
|
|
},
|
|
"head": {
|
|
"accuracy": 0.5277777777777778,
|
|
"confusion": [
|
|
[
|
|
5,
|
|
5,
|
|
0
|
|
],
|
|
[
|
|
1,
|
|
11,
|
|
1
|
|
],
|
|
[
|
|
3,
|
|
7,
|
|
3
|
|
]
|
|
],
|
|
"directional_accuracy": 0.34782608695652173,
|
|
"directional_hits": 8,
|
|
"directional_n": 23,
|
|
"hits": 19,
|
|
"macro_f1": 0.49678935901846116,
|
|
"n": 36,
|
|
"sign_agreement": 0.7272727272727273,
|
|
"sign_hits": 8,
|
|
"sign_n": 11
|
|
},
|
|
"keyword": {
|
|
"accuracy": 0.4166666666666667,
|
|
"confusion": [
|
|
[
|
|
2,
|
|
7,
|
|
1
|
|
],
|
|
[
|
|
0,
|
|
13,
|
|
0
|
|
],
|
|
[
|
|
1,
|
|
12,
|
|
0
|
|
]
|
|
],
|
|
"directional_accuracy": 0.08695652173913043,
|
|
"directional_hits": 2,
|
|
"directional_n": 23,
|
|
"hits": 15,
|
|
"macro_f1": 0.2951566951566951,
|
|
"n": 36,
|
|
"sign_agreement": 0.5,
|
|
"sign_hits": 2,
|
|
"sign_n": 4
|
|
},
|
|
"n_test": 36,
|
|
"n_train": 71
|
|
},
|
|
{
|
|
"day": "2026-09-10",
|
|
"fit_seconds_full": 1.522781863000091,
|
|
"fit_seconds_head": 0.026001763000294886,
|
|
"full": {
|
|
"accuracy": 0.49767441860465117,
|
|
"confusion": [
|
|
[
|
|
25,
|
|
42,
|
|
3
|
|
],
|
|
[
|
|
2,
|
|
60,
|
|
11
|
|
],
|
|
[
|
|
7,
|
|
43,
|
|
22
|
|
]
|
|
],
|
|
"directional_accuracy": 0.33098591549295775,
|
|
"directional_hits": 47,
|
|
"directional_n": 142,
|
|
"hits": 107,
|
|
"macro_f1": 0.4795451179243228,
|
|
"n": 215,
|
|
"sign_agreement": 0.8245614035087719,
|
|
"sign_hits": 47,
|
|
"sign_n": 57
|
|
},
|
|
"head": {
|
|
"accuracy": 0.5720930232558139,
|
|
"confusion": [
|
|
[
|
|
34,
|
|
29,
|
|
7
|
|
],
|
|
[
|
|
1,
|
|
59,
|
|
13
|
|
],
|
|
[
|
|
8,
|
|
34,
|
|
30
|
|
]
|
|
],
|
|
"directional_accuracy": 0.4507042253521127,
|
|
"directional_hits": 64,
|
|
"directional_n": 142,
|
|
"hits": 123,
|
|
"macro_f1": 0.5662337984403848,
|
|
"n": 215,
|
|
"sign_agreement": 0.810126582278481,
|
|
"sign_hits": 64,
|
|
"sign_n": 79
|
|
},
|
|
"keyword": {
|
|
"accuracy": 0.4558139534883721,
|
|
"confusion": [
|
|
[
|
|
23,
|
|
46,
|
|
1
|
|
],
|
|
[
|
|
3,
|
|
69,
|
|
1
|
|
],
|
|
[
|
|
11,
|
|
55,
|
|
6
|
|
]
|
|
],
|
|
"directional_accuracy": 0.20422535211267606,
|
|
"directional_hits": 29,
|
|
"directional_n": 142,
|
|
"hits": 98,
|
|
"macro_f1": 0.38260259220799203,
|
|
"n": 215,
|
|
"sign_agreement": 0.7073170731707317,
|
|
"sign_hits": 29,
|
|
"sign_n": 41
|
|
},
|
|
"n_test": 215,
|
|
"n_train": 107
|
|
},
|
|
{
|
|
"day": "2026-09-11",
|
|
"fit_seconds_full": 2.209334836999915,
|
|
"fit_seconds_head": 1.8741989730001478,
|
|
"full": {
|
|
"accuracy": 0.45365853658536587,
|
|
"confusion": [
|
|
[
|
|
32,
|
|
23,
|
|
15
|
|
],
|
|
[
|
|
11,
|
|
38,
|
|
14
|
|
],
|
|
[
|
|
19,
|
|
30,
|
|
23
|
|
]
|
|
],
|
|
"directional_accuracy": 0.3873239436619718,
|
|
"directional_hits": 55,
|
|
"directional_n": 142,
|
|
"hits": 93,
|
|
"macro_f1": 0.44977424009682077,
|
|
"n": 205,
|
|
"sign_agreement": 0.6179775280898876,
|
|
"sign_hits": 55,
|
|
"sign_n": 89
|
|
},
|
|
"head": {
|
|
"accuracy": 0.5365853658536586,
|
|
"confusion": [
|
|
[
|
|
39,
|
|
14,
|
|
17
|
|
],
|
|
[
|
|
7,
|
|
37,
|
|
19
|
|
],
|
|
[
|
|
18,
|
|
20,
|
|
34
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5140845070422535,
|
|
"directional_hits": 73,
|
|
"directional_n": 142,
|
|
"hits": 110,
|
|
"macro_f1": 0.5377338658818583,
|
|
"n": 205,
|
|
"sign_agreement": 0.6759259259259259,
|
|
"sign_hits": 73,
|
|
"sign_n": 108
|
|
},
|
|
"keyword": {
|
|
"accuracy": 0.40487804878048783,
|
|
"confusion": [
|
|
[
|
|
24,
|
|
43,
|
|
3
|
|
],
|
|
[
|
|
7,
|
|
54,
|
|
2
|
|
],
|
|
[
|
|
15,
|
|
52,
|
|
5
|
|
]
|
|
],
|
|
"directional_accuracy": 0.20422535211267606,
|
|
"directional_hits": 29,
|
|
"directional_n": 142,
|
|
"hits": 83,
|
|
"macro_f1": 0.3483927617415407,
|
|
"n": 205,
|
|
"sign_agreement": 0.6170212765957447,
|
|
"sign_hits": 29,
|
|
"sign_n": 47
|
|
},
|
|
"n_test": 205,
|
|
"n_train": 322
|
|
},
|
|
{
|
|
"day": "2026-09-13",
|
|
"fit_seconds_full": 3.299125021999771,
|
|
"fit_seconds_head": 3.1570361300000513,
|
|
"full": {
|
|
"accuracy": 0.4,
|
|
"confusion": [
|
|
[
|
|
1,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
1,
|
|
2
|
|
],
|
|
[
|
|
1,
|
|
0,
|
|
0
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5,
|
|
"directional_hits": 1,
|
|
"directional_n": 2,
|
|
"hits": 2,
|
|
"macro_f1": 0.38888888888888884,
|
|
"n": 5,
|
|
"sign_agreement": 0.5,
|
|
"sign_hits": 1,
|
|
"sign_n": 2
|
|
},
|
|
"head": {
|
|
"accuracy": 0.6,
|
|
"confusion": [
|
|
[
|
|
1,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
2,
|
|
1
|
|
],
|
|
[
|
|
0,
|
|
1,
|
|
0
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5,
|
|
"directional_hits": 1,
|
|
"directional_n": 2,
|
|
"hits": 3,
|
|
"macro_f1": 0.5555555555555555,
|
|
"n": 5,
|
|
"sign_agreement": 1.0,
|
|
"sign_hits": 1,
|
|
"sign_n": 1
|
|
},
|
|
"keyword": {
|
|
"accuracy": 0.8,
|
|
"confusion": [
|
|
[
|
|
1,
|
|
0,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
3,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
1,
|
|
0
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5,
|
|
"directional_hits": 1,
|
|
"directional_n": 2,
|
|
"hits": 4,
|
|
"macro_f1": 0.6190476190476191,
|
|
"n": 5,
|
|
"sign_agreement": 1.0,
|
|
"sign_hits": 1,
|
|
"sign_n": 1
|
|
},
|
|
"n_test": 5,
|
|
"n_train": 527
|
|
},
|
|
{
|
|
"day": "2026-09-14",
|
|
"fit_seconds_full": 3.3643084619998262,
|
|
"fit_seconds_head": 3.809506984000109,
|
|
"full": {
|
|
"accuracy": 0.6278195488721805,
|
|
"confusion": [
|
|
[
|
|
87,
|
|
35,
|
|
22
|
|
],
|
|
[
|
|
4,
|
|
59,
|
|
14
|
|
],
|
|
[
|
|
10,
|
|
14,
|
|
21
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5714285714285714,
|
|
"directional_hits": 108,
|
|
"directional_n": 189,
|
|
"hits": 167,
|
|
"macro_f1": 0.5866022084509479,
|
|
"n": 266,
|
|
"sign_agreement": 0.7714285714285715,
|
|
"sign_hits": 108,
|
|
"sign_n": 140
|
|
},
|
|
"head": {
|
|
"accuracy": 0.6390977443609023,
|
|
"confusion": [
|
|
[
|
|
89,
|
|
25,
|
|
30
|
|
],
|
|
[
|
|
2,
|
|
64,
|
|
11
|
|
],
|
|
[
|
|
4,
|
|
24,
|
|
17
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5608465608465608,
|
|
"directional_hits": 106,
|
|
"directional_n": 189,
|
|
"hits": 170,
|
|
"macro_f1": 0.5828503907939814,
|
|
"n": 266,
|
|
"sign_agreement": 0.7571428571428571,
|
|
"sign_hits": 106,
|
|
"sign_n": 140
|
|
},
|
|
"keyword": {
|
|
"accuracy": 0.5075187969924813,
|
|
"confusion": [
|
|
[
|
|
61,
|
|
77,
|
|
6
|
|
],
|
|
[
|
|
7,
|
|
69,
|
|
1
|
|
],
|
|
[
|
|
5,
|
|
35,
|
|
5
|
|
]
|
|
],
|
|
"directional_accuracy": 0.3492063492063492,
|
|
"directional_hits": 66,
|
|
"directional_n": 189,
|
|
"hits": 135,
|
|
"macro_f1": 0.42417809966276027,
|
|
"n": 266,
|
|
"sign_agreement": 0.8571428571428571,
|
|
"sign_hits": 66,
|
|
"sign_n": 77
|
|
},
|
|
"n_test": 266,
|
|
"n_train": 532
|
|
},
|
|
{
|
|
"day": "2026-09-15",
|
|
"fit_seconds_full": 3.2497777419998783,
|
|
"fit_seconds_head": 3.444847923999987,
|
|
"full": {
|
|
"accuracy": 0.576036866359447,
|
|
"confusion": [
|
|
[
|
|
57,
|
|
17,
|
|
16
|
|
],
|
|
[
|
|
6,
|
|
54,
|
|
8
|
|
],
|
|
[
|
|
24,
|
|
21,
|
|
14
|
|
]
|
|
],
|
|
"directional_accuracy": 0.47651006711409394,
|
|
"directional_hits": 71,
|
|
"directional_n": 149,
|
|
"hits": 125,
|
|
"macro_f1": 0.5359091968082008,
|
|
"n": 217,
|
|
"sign_agreement": 0.6396396396396397,
|
|
"sign_hits": 71,
|
|
"sign_n": 111
|
|
},
|
|
"head": {
|
|
"accuracy": 0.6082949308755761,
|
|
"confusion": [
|
|
[
|
|
55,
|
|
18,
|
|
17
|
|
],
|
|
[
|
|
3,
|
|
50,
|
|
15
|
|
],
|
|
[
|
|
10,
|
|
22,
|
|
27
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5503355704697986,
|
|
"directional_hits": 82,
|
|
"directional_n": 149,
|
|
"hits": 132,
|
|
"macro_f1": 0.5955803475649003,
|
|
"n": 217,
|
|
"sign_agreement": 0.7522935779816514,
|
|
"sign_hits": 82,
|
|
"sign_n": 109
|
|
},
|
|
"keyword": {
|
|
"accuracy": 0.4009216589861751,
|
|
"confusion": [
|
|
[
|
|
24,
|
|
59,
|
|
7
|
|
],
|
|
[
|
|
4,
|
|
59,
|
|
5
|
|
],
|
|
[
|
|
11,
|
|
44,
|
|
4
|
|
]
|
|
],
|
|
"directional_accuracy": 0.18791946308724833,
|
|
"directional_hits": 28,
|
|
"directional_n": 149,
|
|
"hits": 87,
|
|
"macro_f1": 0.3306010560611167,
|
|
"n": 217,
|
|
"sign_agreement": 0.6086956521739131,
|
|
"sign_hits": 28,
|
|
"sign_n": 46
|
|
},
|
|
"n_test": 217,
|
|
"n_train": 798
|
|
},
|
|
{
|
|
"day": "2026-09-16",
|
|
"fit_seconds_full": 4.366315348999706,
|
|
"fit_seconds_head": 4.552804755000125,
|
|
"full": {
|
|
"accuracy": 0.5375,
|
|
"confusion": [
|
|
[
|
|
69,
|
|
19,
|
|
17
|
|
],
|
|
[
|
|
22,
|
|
63,
|
|
24
|
|
],
|
|
[
|
|
36,
|
|
30,
|
|
40
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5165876777251185,
|
|
"directional_hits": 109,
|
|
"directional_n": 211,
|
|
"hits": 172,
|
|
"macro_f1": 0.5309236064814159,
|
|
"n": 320,
|
|
"sign_agreement": 0.6728395061728395,
|
|
"sign_hits": 109,
|
|
"sign_n": 162
|
|
},
|
|
"head": {
|
|
"accuracy": 0.5625,
|
|
"confusion": [
|
|
[
|
|
68,
|
|
18,
|
|
19
|
|
],
|
|
[
|
|
20,
|
|
66,
|
|
23
|
|
],
|
|
[
|
|
32,
|
|
28,
|
|
46
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5402843601895735,
|
|
"directional_hits": 114,
|
|
"directional_n": 211,
|
|
"hits": 180,
|
|
"macro_f1": 0.5586521054804863,
|
|
"n": 320,
|
|
"sign_agreement": 0.6909090909090909,
|
|
"sign_hits": 114,
|
|
"sign_n": 165
|
|
},
|
|
"keyword": {
|
|
"accuracy": 0.41875,
|
|
"confusion": [
|
|
[
|
|
32,
|
|
67,
|
|
6
|
|
],
|
|
[
|
|
11,
|
|
90,
|
|
8
|
|
],
|
|
[
|
|
15,
|
|
79,
|
|
12
|
|
]
|
|
],
|
|
"directional_accuracy": 0.20853080568720378,
|
|
"directional_hits": 44,
|
|
"directional_n": 211,
|
|
"hits": 134,
|
|
"macro_f1": 0.36539844968759344,
|
|
"n": 320,
|
|
"sign_agreement": 0.676923076923077,
|
|
"sign_hits": 44,
|
|
"sign_n": 65
|
|
},
|
|
"n_test": 320,
|
|
"n_train": 1015
|
|
},
|
|
{
|
|
"day": "2026-09-17",
|
|
"fit_seconds_full": 4.3225185520000196,
|
|
"fit_seconds_head": 5.175150255000062,
|
|
"full": {
|
|
"accuracy": 0.6,
|
|
"confusion": [
|
|
[
|
|
67,
|
|
22,
|
|
19
|
|
],
|
|
[
|
|
15,
|
|
55,
|
|
13
|
|
],
|
|
[
|
|
19,
|
|
24,
|
|
46
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5736040609137056,
|
|
"directional_hits": 113,
|
|
"directional_n": 197,
|
|
"hits": 168,
|
|
"macro_f1": 0.5966242053027293,
|
|
"n": 280,
|
|
"sign_agreement": 0.7483443708609272,
|
|
"sign_hits": 113,
|
|
"sign_n": 151
|
|
},
|
|
"head": {
|
|
"accuracy": 0.5964285714285714,
|
|
"confusion": [
|
|
[
|
|
72,
|
|
19,
|
|
17
|
|
],
|
|
[
|
|
12,
|
|
44,
|
|
27
|
|
],
|
|
[
|
|
18,
|
|
20,
|
|
51
|
|
]
|
|
],
|
|
"directional_accuracy": 0.6243654822335025,
|
|
"directional_hits": 123,
|
|
"directional_n": 197,
|
|
"hits": 167,
|
|
"macro_f1": 0.5900608645763178,
|
|
"n": 280,
|
|
"sign_agreement": 0.7784810126582279,
|
|
"sign_hits": 123,
|
|
"sign_n": 158
|
|
},
|
|
"keyword": {
|
|
"accuracy": 0.4107142857142857,
|
|
"confusion": [
|
|
[
|
|
37,
|
|
65,
|
|
6
|
|
],
|
|
[
|
|
13,
|
|
66,
|
|
4
|
|
],
|
|
[
|
|
11,
|
|
66,
|
|
12
|
|
]
|
|
],
|
|
"directional_accuracy": 0.24873096446700507,
|
|
"directional_hits": 49,
|
|
"directional_n": 197,
|
|
"hits": 115,
|
|
"macro_f1": 0.3751715367099983,
|
|
"n": 280,
|
|
"sign_agreement": 0.7424242424242424,
|
|
"sign_hits": 49,
|
|
"sign_n": 66
|
|
},
|
|
"n_test": 280,
|
|
"n_train": 1335
|
|
},
|
|
{
|
|
"day": "2026-09-18",
|
|
"fit_seconds_full": 5.254859737000061,
|
|
"fit_seconds_head": 4.997993596000015,
|
|
"full": {
|
|
"accuracy": 0.5835777126099707,
|
|
"confusion": [
|
|
[
|
|
43,
|
|
15,
|
|
17
|
|
],
|
|
[
|
|
10,
|
|
56,
|
|
37
|
|
],
|
|
[
|
|
34,
|
|
29,
|
|
100
|
|
]
|
|
],
|
|
"directional_accuracy": 0.6008403361344538,
|
|
"directional_hits": 143,
|
|
"directional_n": 238,
|
|
"hits": 199,
|
|
"macro_f1": 0.5711677206534405,
|
|
"n": 341,
|
|
"sign_agreement": 0.7371134020618557,
|
|
"sign_hits": 143,
|
|
"sign_n": 194
|
|
},
|
|
"head": {
|
|
"accuracy": 0.5894428152492669,
|
|
"confusion": [
|
|
[
|
|
47,
|
|
13,
|
|
15
|
|
],
|
|
[
|
|
11,
|
|
59,
|
|
33
|
|
],
|
|
[
|
|
40,
|
|
28,
|
|
95
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5966386554621849,
|
|
"directional_hits": 142,
|
|
"directional_n": 238,
|
|
"hits": 201,
|
|
"macro_f1": 0.5818494740043826,
|
|
"n": 341,
|
|
"sign_agreement": 0.7208121827411168,
|
|
"sign_hits": 142,
|
|
"sign_n": 197
|
|
},
|
|
"keyword": {
|
|
"accuracy": 0.41055718475073316,
|
|
"confusion": [
|
|
[
|
|
28,
|
|
34,
|
|
13
|
|
],
|
|
[
|
|
5,
|
|
89,
|
|
9
|
|
],
|
|
[
|
|
20,
|
|
120,
|
|
23
|
|
]
|
|
],
|
|
"directional_accuracy": 0.21428571428571427,
|
|
"directional_hits": 51,
|
|
"directional_n": 238,
|
|
"hits": 140,
|
|
"macro_f1": 0.3910349044019564,
|
|
"n": 341,
|
|
"sign_agreement": 0.6071428571428571,
|
|
"sign_hits": 51,
|
|
"sign_n": 84
|
|
},
|
|
"n_test": 341,
|
|
"n_train": 1615
|
|
},
|
|
{
|
|
"day": "2026-09-20",
|
|
"fit_seconds_full": 4.983563467999829,
|
|
"fit_seconds_head": 3.135302303000117,
|
|
"full": {
|
|
"accuracy": 0.6538461538461539,
|
|
"confusion": [
|
|
[
|
|
6,
|
|
0,
|
|
1
|
|
],
|
|
[
|
|
0,
|
|
8,
|
|
0
|
|
],
|
|
[
|
|
2,
|
|
6,
|
|
3
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5,
|
|
"directional_hits": 9,
|
|
"directional_n": 18,
|
|
"hits": 17,
|
|
"macro_f1": 0.6424242424242425,
|
|
"n": 26,
|
|
"sign_agreement": 0.75,
|
|
"sign_hits": 9,
|
|
"sign_n": 12
|
|
},
|
|
"head": {
|
|
"accuracy": 0.7307692307692307,
|
|
"confusion": [
|
|
[
|
|
3,
|
|
0,
|
|
4
|
|
],
|
|
[
|
|
0,
|
|
8,
|
|
0
|
|
],
|
|
[
|
|
1,
|
|
2,
|
|
8
|
|
]
|
|
],
|
|
"directional_accuracy": 0.6111111111111112,
|
|
"directional_hits": 11,
|
|
"directional_n": 18,
|
|
"hits": 19,
|
|
"macro_f1": 0.7099985360854926,
|
|
"n": 26,
|
|
"sign_agreement": 0.6875,
|
|
"sign_hits": 11,
|
|
"sign_n": 16
|
|
},
|
|
"keyword": {
|
|
"accuracy": 0.46153846153846156,
|
|
"confusion": [
|
|
[
|
|
0,
|
|
7,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
8,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
7,
|
|
4
|
|
]
|
|
],
|
|
"directional_accuracy": 0.2222222222222222,
|
|
"directional_hits": 4,
|
|
"directional_n": 18,
|
|
"hits": 12,
|
|
"macro_f1": 0.35555555555555557,
|
|
"n": 26,
|
|
"sign_agreement": 1.0,
|
|
"sign_hits": 4,
|
|
"sign_n": 4
|
|
},
|
|
"n_test": 26,
|
|
"n_train": 1956
|
|
},
|
|
{
|
|
"day": "2026-09-21",
|
|
"fit_seconds_full": 5.172674763000032,
|
|
"fit_seconds_head": 3.7032702040000913,
|
|
"full": {
|
|
"accuracy": 0.6430517711171662,
|
|
"confusion": [
|
|
[
|
|
84,
|
|
19,
|
|
29
|
|
],
|
|
[
|
|
16,
|
|
96,
|
|
31
|
|
],
|
|
[
|
|
10,
|
|
26,
|
|
56
|
|
]
|
|
],
|
|
"directional_accuracy": 0.625,
|
|
"directional_hits": 140,
|
|
"directional_n": 224,
|
|
"hits": 236,
|
|
"macro_f1": 0.6362442508409218,
|
|
"n": 367,
|
|
"sign_agreement": 0.7821229050279329,
|
|
"sign_hits": 140,
|
|
"sign_n": 179
|
|
},
|
|
"head": {
|
|
"accuracy": 0.6566757493188011,
|
|
"confusion": [
|
|
[
|
|
80,
|
|
16,
|
|
36
|
|
],
|
|
[
|
|
8,
|
|
105,
|
|
30
|
|
],
|
|
[
|
|
8,
|
|
28,
|
|
56
|
|
]
|
|
],
|
|
"directional_accuracy": 0.6071428571428571,
|
|
"directional_hits": 136,
|
|
"directional_n": 224,
|
|
"hits": 241,
|
|
"macro_f1": 0.6480989847126672,
|
|
"n": 367,
|
|
"sign_agreement": 0.7555555555555555,
|
|
"sign_hits": 136,
|
|
"sign_n": 180
|
|
},
|
|
"keyword": {
|
|
"accuracy": 0.4986376021798365,
|
|
"confusion": [
|
|
[
|
|
53,
|
|
72,
|
|
7
|
|
],
|
|
[
|
|
4,
|
|
125,
|
|
14
|
|
],
|
|
[
|
|
5,
|
|
82,
|
|
5
|
|
]
|
|
],
|
|
"directional_accuracy": 0.25892857142857145,
|
|
"directional_hits": 58,
|
|
"directional_n": 224,
|
|
"hits": 183,
|
|
"macro_f1": 0.4078515256335195,
|
|
"n": 367,
|
|
"sign_agreement": 0.8285714285714286,
|
|
"sign_hits": 58,
|
|
"sign_n": 70
|
|
},
|
|
"n_test": 367,
|
|
"n_train": 1982
|
|
},
|
|
{
|
|
"day": "2026-09-22",
|
|
"fit_seconds_full": 4.346579720999671,
|
|
"fit_seconds_head": 6.362624880000112,
|
|
"full": {
|
|
"accuracy": 0.6597222222222222,
|
|
"confusion": [
|
|
[
|
|
60,
|
|
16,
|
|
17
|
|
],
|
|
[
|
|
7,
|
|
85,
|
|
20
|
|
],
|
|
[
|
|
16,
|
|
22,
|
|
45
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5965909090909091,
|
|
"directional_hits": 105,
|
|
"directional_n": 176,
|
|
"hits": 190,
|
|
"macro_f1": 0.6502256608639586,
|
|
"n": 288,
|
|
"sign_agreement": 0.7608695652173914,
|
|
"sign_hits": 105,
|
|
"sign_n": 138
|
|
},
|
|
"head": {
|
|
"accuracy": 0.6527777777777778,
|
|
"confusion": [
|
|
[
|
|
56,
|
|
18,
|
|
19
|
|
],
|
|
[
|
|
2,
|
|
86,
|
|
24
|
|
],
|
|
[
|
|
18,
|
|
19,
|
|
46
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5795454545454546,
|
|
"directional_hits": 102,
|
|
"directional_n": 176,
|
|
"hits": 188,
|
|
"macro_f1": 0.6431735026794594,
|
|
"n": 288,
|
|
"sign_agreement": 0.7338129496402878,
|
|
"sign_hits": 102,
|
|
"sign_n": 139
|
|
},
|
|
"keyword": {
|
|
"accuracy": 0.4895833333333333,
|
|
"confusion": [
|
|
[
|
|
37,
|
|
49,
|
|
7
|
|
],
|
|
[
|
|
4,
|
|
99,
|
|
9
|
|
],
|
|
[
|
|
17,
|
|
61,
|
|
5
|
|
]
|
|
],
|
|
"directional_accuracy": 0.23863636363636365,
|
|
"directional_hits": 42,
|
|
"directional_n": 176,
|
|
"hits": 141,
|
|
"macro_f1": 0.40101416707531706,
|
|
"n": 288,
|
|
"sign_agreement": 0.6363636363636364,
|
|
"sign_hits": 42,
|
|
"sign_n": 66
|
|
},
|
|
"n_test": 288,
|
|
"n_train": 2349
|
|
}
|
|
],
|
|
"generated_at": "20260923_0038_all",
|
|
"in_sample": {
|
|
"headline_description": {
|
|
"accuracy": 0.5608646188850968,
|
|
"confusion": [
|
|
[
|
|
624,
|
|
73,
|
|
219
|
|
],
|
|
[
|
|
182,
|
|
335,
|
|
379
|
|
],
|
|
[
|
|
222,
|
|
83,
|
|
520
|
|
]
|
|
],
|
|
"directional_accuracy": 0.6570936243538197,
|
|
"directional_hits": 1144,
|
|
"directional_n": 1741,
|
|
"hits": 1479,
|
|
"macro_f1": 0.5534290089278285,
|
|
"mae": 0.2846768854759196,
|
|
"n": 2637,
|
|
"sign_agreement": 0.7217665615141956,
|
|
"sign_hits": 1144,
|
|
"sign_n": 1585,
|
|
"spearman": 0.5336683695481723
|
|
},
|
|
"headline_only": {
|
|
"accuracy": 0.5733788395904437,
|
|
"confusion": [
|
|
[
|
|
652,
|
|
43,
|
|
221
|
|
],
|
|
[
|
|
152,
|
|
304,
|
|
440
|
|
],
|
|
[
|
|
210,
|
|
59,
|
|
556
|
|
]
|
|
],
|
|
"directional_accuracy": 0.6938541068351523,
|
|
"directional_hits": 1208,
|
|
"directional_n": 1741,
|
|
"hits": 1512,
|
|
"macro_f1": 0.5623952358379628,
|
|
"mae": 0.30869570231323473,
|
|
"n": 2637,
|
|
"sign_agreement": 0.7370347773032336,
|
|
"sign_hits": 1208,
|
|
"sign_n": 1639,
|
|
"spearman": 0.554711637369288
|
|
}
|
|
},
|
|
"label_distribution": {
|
|
"bearish": 916,
|
|
"bullish": 825,
|
|
"neutral": 896
|
|
},
|
|
"model": {
|
|
"label_distribution": {
|
|
"bearish": 250,
|
|
"bullish": 221,
|
|
"neutral": 135
|
|
},
|
|
"model_path": "/home/harwinder/Development/ML-NEWS/models/xauusd_sentiment.joblib",
|
|
"model_type": "domain_augmented_tfidf_logistic_regression",
|
|
"neutral_threshold": 0.05,
|
|
"sklearn_version": "1.9.1",
|
|
"trained_at": "2026-09-17T18:54:46.714699+00:00",
|
|
"training_examples": 606
|
|
},
|
|
"pooled_oof": {
|
|
"full": {
|
|
"accuracy": 0.5822291504286827,
|
|
"confusion": [
|
|
[
|
|
535,
|
|
214,
|
|
156
|
|
],
|
|
[
|
|
96,
|
|
585,
|
|
174
|
|
],
|
|
[
|
|
183,
|
|
249,
|
|
374
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5312682641729982,
|
|
"directional_hits": 909,
|
|
"directional_n": 1711,
|
|
"hits": 1494,
|
|
"macro_f1": 0.5775459537878285,
|
|
"mae": 0.2729630943102105,
|
|
"n": 2566,
|
|
"sign_agreement": 0.7283653846153846,
|
|
"sign_hits": 909,
|
|
"sign_n": 1248,
|
|
"spearman": 0.45128845313804333
|
|
},
|
|
"full_gold": {
|
|
"accuracy": 0.6508620689655172,
|
|
"confusion": [
|
|
[
|
|
241,
|
|
34,
|
|
82
|
|
],
|
|
[
|
|
23,
|
|
122,
|
|
47
|
|
],
|
|
[
|
|
78,
|
|
60,
|
|
241
|
|
]
|
|
],
|
|
"directional_accuracy": 0.654891304347826,
|
|
"directional_hits": 482,
|
|
"directional_n": 736,
|
|
"hits": 604,
|
|
"macro_f1": 0.643706808194913,
|
|
"mae": 0.32061842672413793,
|
|
"n": 928,
|
|
"sign_agreement": 0.7507788161993769,
|
|
"sign_hits": 482,
|
|
"sign_n": 642,
|
|
"spearman": 0.5534196552017715
|
|
},
|
|
"full_macro": {
|
|
"accuracy": 0.5433455433455433,
|
|
"confusion": [
|
|
[
|
|
294,
|
|
180,
|
|
74
|
|
],
|
|
[
|
|
73,
|
|
463,
|
|
127
|
|
],
|
|
[
|
|
105,
|
|
189,
|
|
133
|
|
]
|
|
],
|
|
"directional_accuracy": 0.4379487179487179,
|
|
"directional_hits": 427,
|
|
"directional_n": 975,
|
|
"hits": 890,
|
|
"macro_f1": 0.5151362201299858,
|
|
"mae": 0.24596422466422468,
|
|
"n": 1638,
|
|
"sign_agreement": 0.7046204620462047,
|
|
"sign_hits": 427,
|
|
"sign_n": 606,
|
|
"spearman": 0.34905990065793735
|
|
},
|
|
"full_since_2026-09-18_all": {
|
|
"accuracy": 0.6281800391389433,
|
|
"confusion": [
|
|
[
|
|
193,
|
|
50,
|
|
64
|
|
],
|
|
[
|
|
33,
|
|
245,
|
|
88
|
|
],
|
|
[
|
|
62,
|
|
83,
|
|
204
|
|
]
|
|
],
|
|
"directional_accuracy": 0.6051829268292683,
|
|
"directional_hits": 397,
|
|
"directional_n": 656,
|
|
"hits": 642,
|
|
"macro_f1": 0.6286883501970909,
|
|
"n": 1022,
|
|
"sign_agreement": 0.7590822179732314,
|
|
"sign_hits": 397,
|
|
"sign_n": 523
|
|
},
|
|
"full_since_2026-09-18_gold": {
|
|
"accuracy": 0.7188498402555911,
|
|
"confusion": [
|
|
[
|
|
75,
|
|
6,
|
|
28
|
|
],
|
|
[
|
|
8,
|
|
36,
|
|
15
|
|
],
|
|
[
|
|
21,
|
|
10,
|
|
114
|
|
]
|
|
],
|
|
"directional_accuracy": 0.7440944881889764,
|
|
"directional_hits": 189,
|
|
"directional_n": 254,
|
|
"hits": 225,
|
|
"macro_f1": 0.7026136293928477,
|
|
"n": 313,
|
|
"sign_agreement": 0.7941176470588235,
|
|
"sign_hits": 189,
|
|
"sign_n": 238
|
|
},
|
|
"full_since_2026-09-18_macro": {
|
|
"accuracy": 0.5881523272214386,
|
|
"confusion": [
|
|
[
|
|
118,
|
|
44,
|
|
36
|
|
],
|
|
[
|
|
25,
|
|
209,
|
|
73
|
|
],
|
|
[
|
|
41,
|
|
73,
|
|
90
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5174129353233831,
|
|
"directional_hits": 208,
|
|
"directional_n": 402,
|
|
"hits": 417,
|
|
"macro_f1": 0.5749329075109032,
|
|
"n": 709,
|
|
"sign_agreement": 0.7298245614035088,
|
|
"sign_hits": 208,
|
|
"sign_n": 285
|
|
},
|
|
"full_train_ge_300": {
|
|
"accuracy": 0.5913606911447085,
|
|
"confusion": [
|
|
[
|
|
506,
|
|
166,
|
|
153
|
|
],
|
|
[
|
|
91,
|
|
515,
|
|
163
|
|
],
|
|
[
|
|
171,
|
|
202,
|
|
348
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5523932729624839,
|
|
"directional_hits": 854,
|
|
"directional_n": 1546,
|
|
"hits": 1369,
|
|
"macro_f1": 0.5870977019215816,
|
|
"mae": 0.2741298056155508,
|
|
"n": 2315,
|
|
"sign_agreement": 0.7249575551782682,
|
|
"sign_hits": 854,
|
|
"sign_n": 1178,
|
|
"spearman": 0.4632028749594082
|
|
},
|
|
"head": {
|
|
"accuracy": 0.6052221356196414,
|
|
"confusion": [
|
|
[
|
|
549,
|
|
175,
|
|
181
|
|
],
|
|
[
|
|
67,
|
|
591,
|
|
197
|
|
],
|
|
[
|
|
160,
|
|
233,
|
|
413
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5622443015780245,
|
|
"directional_hits": 962,
|
|
"directional_n": 1711,
|
|
"hits": 1553,
|
|
"macro_f1": 0.6026476231875058,
|
|
"mae": 0.3004310210444271,
|
|
"n": 2566,
|
|
"sign_agreement": 0.7382962394474291,
|
|
"sign_hits": 962,
|
|
"sign_n": 1303,
|
|
"spearman": 0.4780602704994956
|
|
},
|
|
"keyword": {
|
|
"accuracy": 0.4469992205767732,
|
|
"confusion": [
|
|
[
|
|
322,
|
|
526,
|
|
57
|
|
],
|
|
[
|
|
58,
|
|
744,
|
|
53
|
|
],
|
|
[
|
|
111,
|
|
614,
|
|
81
|
|
]
|
|
],
|
|
"directional_accuracy": 0.23553477498538866,
|
|
"directional_hits": 403,
|
|
"directional_n": 1711,
|
|
"hits": 1147,
|
|
"macro_f1": 0.38902315963793493,
|
|
"n": 2566,
|
|
"sign_agreement": 0.7057793345008757,
|
|
"sign_hits": 403,
|
|
"sign_n": 571
|
|
}
|
|
},
|
|
"reprobe": {},
|
|
"routing": [
|
|
{
|
|
"confidence_min": 0.0,
|
|
"hybrid_agreement": 0.5822291504286827,
|
|
"kept": 2566,
|
|
"kept_accuracy": 0.5822291504286827,
|
|
"kept_share": 1.0,
|
|
"llm_calls_saved_share": 1.0,
|
|
"margin_min": 0.0
|
|
},
|
|
{
|
|
"confidence_min": 0.0,
|
|
"hybrid_agreement": 0.6874512860483243,
|
|
"kept": 2096,
|
|
"kept_accuracy": 0.6173664122137404,
|
|
"kept_share": 0.8168355416991426,
|
|
"llm_calls_saved_share": 0.8168355416991426,
|
|
"margin_min": 0.1
|
|
},
|
|
{
|
|
"confidence_min": 0.0,
|
|
"hybrid_agreement": 0.7727981293842556,
|
|
"kept": 1700,
|
|
"kept_accuracy": 0.6570588235294118,
|
|
"kept_share": 0.6625097427903351,
|
|
"llm_calls_saved_share": 0.6625097427903351,
|
|
"margin_min": 0.2
|
|
},
|
|
{
|
|
"confidence_min": 0.0,
|
|
"hybrid_agreement": 0.8347622759158223,
|
|
"kept": 1358,
|
|
"kept_accuracy": 0.6877761413843888,
|
|
"kept_share": 0.5292283710054559,
|
|
"llm_calls_saved_share": 0.5292283710054559,
|
|
"margin_min": 0.3
|
|
},
|
|
{
|
|
"confidence_min": 0.0,
|
|
"hybrid_agreement": 0.886983632112237,
|
|
"kept": 1052,
|
|
"kept_accuracy": 0.7243346007604563,
|
|
"kept_share": 0.4099766173031956,
|
|
"llm_calls_saved_share": 0.4099766173031956,
|
|
"margin_min": 0.4
|
|
},
|
|
{
|
|
"confidence_min": 0.5,
|
|
"hybrid_agreement": 0.7603273577552611,
|
|
"kept": 1786,
|
|
"kept_accuracy": 0.6556550951847704,
|
|
"kept_share": 0.696024941543258,
|
|
"llm_calls_saved_share": 0.696024941543258,
|
|
"margin_min": 0.0
|
|
},
|
|
{
|
|
"confidence_min": 0.5,
|
|
"hybrid_agreement": 0.7626656274356975,
|
|
"kept": 1770,
|
|
"kept_accuracy": 0.6559322033898305,
|
|
"kept_share": 0.6897895557287607,
|
|
"llm_calls_saved_share": 0.6897895557287607,
|
|
"margin_min": 0.1
|
|
},
|
|
{
|
|
"confidence_min": 0.5,
|
|
"hybrid_agreement": 0.7848791893998441,
|
|
"kept": 1654,
|
|
"kept_accuracy": 0.6662636033857315,
|
|
"kept_share": 0.6445830085736555,
|
|
"llm_calls_saved_share": 0.6445830085736555,
|
|
"margin_min": 0.2
|
|
},
|
|
{
|
|
"confidence_min": 0.5,
|
|
"hybrid_agreement": 0.8347622759158223,
|
|
"kept": 1358,
|
|
"kept_accuracy": 0.6877761413843888,
|
|
"kept_share": 0.5292283710054559,
|
|
"llm_calls_saved_share": 0.5292283710054559,
|
|
"margin_min": 0.3
|
|
},
|
|
{
|
|
"confidence_min": 0.5,
|
|
"hybrid_agreement": 0.886983632112237,
|
|
"kept": 1052,
|
|
"kept_accuracy": 0.7243346007604563,
|
|
"kept_share": 0.4099766173031956,
|
|
"llm_calls_saved_share": 0.4099766173031956,
|
|
"margin_min": 0.4
|
|
},
|
|
{
|
|
"confidence_min": 0.6,
|
|
"hybrid_agreement": 0.8604832424006236,
|
|
"kept": 1240,
|
|
"kept_accuracy": 0.7112903225806452,
|
|
"kept_share": 0.48324240062353857,
|
|
"llm_calls_saved_share": 0.48324240062353857,
|
|
"margin_min": 0.0
|
|
},
|
|
{
|
|
"confidence_min": 0.6,
|
|
"hybrid_agreement": 0.8604832424006236,
|
|
"kept": 1240,
|
|
"kept_accuracy": 0.7112903225806452,
|
|
"kept_share": 0.48324240062353857,
|
|
"llm_calls_saved_share": 0.48324240062353857,
|
|
"margin_min": 0.1
|
|
},
|
|
{
|
|
"confidence_min": 0.6,
|
|
"hybrid_agreement": 0.8604832424006236,
|
|
"kept": 1240,
|
|
"kept_accuracy": 0.7112903225806452,
|
|
"kept_share": 0.48324240062353857,
|
|
"llm_calls_saved_share": 0.48324240062353857,
|
|
"margin_min": 0.2
|
|
},
|
|
{
|
|
"confidence_min": 0.6,
|
|
"hybrid_agreement": 0.8628215120810601,
|
|
"kept": 1218,
|
|
"kept_accuracy": 0.7110016420361248,
|
|
"kept_share": 0.47466874512860485,
|
|
"llm_calls_saved_share": 0.47466874512860485,
|
|
"margin_min": 0.3
|
|
},
|
|
{
|
|
"confidence_min": 0.6,
|
|
"hybrid_agreement": 0.886983632112237,
|
|
"kept": 1052,
|
|
"kept_accuracy": 0.7243346007604563,
|
|
"kept_share": 0.4099766173031956,
|
|
"llm_calls_saved_share": 0.4099766173031956,
|
|
"margin_min": 0.4
|
|
},
|
|
{
|
|
"confidence_min": 0.7,
|
|
"hybrid_agreement": 0.9368667186282151,
|
|
"kept": 747,
|
|
"kept_accuracy": 0.7831325301204819,
|
|
"kept_share": 0.29111457521434136,
|
|
"llm_calls_saved_share": 0.29111457521434136,
|
|
"margin_min": 0.0
|
|
},
|
|
{
|
|
"confidence_min": 0.7,
|
|
"hybrid_agreement": 0.9368667186282151,
|
|
"kept": 747,
|
|
"kept_accuracy": 0.7831325301204819,
|
|
"kept_share": 0.29111457521434136,
|
|
"llm_calls_saved_share": 0.29111457521434136,
|
|
"margin_min": 0.1
|
|
},
|
|
{
|
|
"confidence_min": 0.7,
|
|
"hybrid_agreement": 0.9368667186282151,
|
|
"kept": 747,
|
|
"kept_accuracy": 0.7831325301204819,
|
|
"kept_share": 0.29111457521434136,
|
|
"llm_calls_saved_share": 0.29111457521434136,
|
|
"margin_min": 0.2
|
|
},
|
|
{
|
|
"confidence_min": 0.7,
|
|
"hybrid_agreement": 0.9368667186282151,
|
|
"kept": 747,
|
|
"kept_accuracy": 0.7831325301204819,
|
|
"kept_share": 0.29111457521434136,
|
|
"llm_calls_saved_share": 0.29111457521434136,
|
|
"margin_min": 0.3
|
|
},
|
|
{
|
|
"confidence_min": 0.7,
|
|
"hybrid_agreement": 0.9368667186282151,
|
|
"kept": 747,
|
|
"kept_accuracy": 0.7831325301204819,
|
|
"kept_share": 0.29111457521434136,
|
|
"llm_calls_saved_share": 0.29111457521434136,
|
|
"margin_min": 0.4
|
|
},
|
|
{
|
|
"confidence_min": 0.8,
|
|
"hybrid_agreement": 0.9828526890101325,
|
|
"kept": 409,
|
|
"kept_accuracy": 0.8924205378973105,
|
|
"kept_share": 0.15939204988308653,
|
|
"llm_calls_saved_share": 0.15939204988308653,
|
|
"margin_min": 0.0
|
|
},
|
|
{
|
|
"confidence_min": 0.8,
|
|
"hybrid_agreement": 0.9828526890101325,
|
|
"kept": 409,
|
|
"kept_accuracy": 0.8924205378973105,
|
|
"kept_share": 0.15939204988308653,
|
|
"llm_calls_saved_share": 0.15939204988308653,
|
|
"margin_min": 0.1
|
|
},
|
|
{
|
|
"confidence_min": 0.8,
|
|
"hybrid_agreement": 0.9828526890101325,
|
|
"kept": 409,
|
|
"kept_accuracy": 0.8924205378973105,
|
|
"kept_share": 0.15939204988308653,
|
|
"llm_calls_saved_share": 0.15939204988308653,
|
|
"margin_min": 0.2
|
|
},
|
|
{
|
|
"confidence_min": 0.8,
|
|
"hybrid_agreement": 0.9828526890101325,
|
|
"kept": 409,
|
|
"kept_accuracy": 0.8924205378973105,
|
|
"kept_share": 0.15939204988308653,
|
|
"llm_calls_saved_share": 0.15939204988308653,
|
|
"margin_min": 0.3
|
|
},
|
|
{
|
|
"confidence_min": 0.8,
|
|
"hybrid_agreement": 0.9828526890101325,
|
|
"kept": 409,
|
|
"kept_accuracy": 0.8924205378973105,
|
|
"kept_share": 0.15939204988308653,
|
|
"llm_calls_saved_share": 0.15939204988308653,
|
|
"margin_min": 0.4
|
|
},
|
|
{
|
|
"confidence_min": 0.9,
|
|
"hybrid_agreement": 0.9964925954793453,
|
|
"kept": 152,
|
|
"kept_accuracy": 0.9407894736842105,
|
|
"kept_share": 0.059236165237724084,
|
|
"llm_calls_saved_share": 0.059236165237724084,
|
|
"margin_min": 0.0
|
|
},
|
|
{
|
|
"confidence_min": 0.9,
|
|
"hybrid_agreement": 0.9964925954793453,
|
|
"kept": 152,
|
|
"kept_accuracy": 0.9407894736842105,
|
|
"kept_share": 0.059236165237724084,
|
|
"llm_calls_saved_share": 0.059236165237724084,
|
|
"margin_min": 0.1
|
|
},
|
|
{
|
|
"confidence_min": 0.9,
|
|
"hybrid_agreement": 0.9964925954793453,
|
|
"kept": 152,
|
|
"kept_accuracy": 0.9407894736842105,
|
|
"kept_share": 0.059236165237724084,
|
|
"llm_calls_saved_share": 0.059236165237724084,
|
|
"margin_min": 0.2
|
|
},
|
|
{
|
|
"confidence_min": 0.9,
|
|
"hybrid_agreement": 0.9964925954793453,
|
|
"kept": 152,
|
|
"kept_accuracy": 0.9407894736842105,
|
|
"kept_share": 0.059236165237724084,
|
|
"llm_calls_saved_share": 0.059236165237724084,
|
|
"margin_min": 0.3
|
|
},
|
|
{
|
|
"confidence_min": 0.9,
|
|
"hybrid_agreement": 0.9964925954793453,
|
|
"kept": 152,
|
|
"kept_accuracy": 0.9407894736842105,
|
|
"kept_share": 0.059236165237724084,
|
|
"llm_calls_saved_share": 0.059236165237724084,
|
|
"margin_min": 0.4
|
|
}
|
|
],
|
|
"rows": 2637,
|
|
"rows_per_day": {
|
|
"2026-09-08": 71,
|
|
"2026-09-09": 36,
|
|
"2026-09-10": 215,
|
|
"2026-09-11": 205,
|
|
"2026-09-13": 5,
|
|
"2026-09-14": 266,
|
|
"2026-09-15": 217,
|
|
"2026-09-16": 320,
|
|
"2026-09-17": 280,
|
|
"2026-09-18": 341,
|
|
"2026-09-20": 26,
|
|
"2026-09-21": 367,
|
|
"2026-09-22": 288
|
|
},
|
|
"threshold": 0.05,
|
|
"train_scope": "all"
|
|
}
|