Harwinder Sekhon
01baef48ff
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked.
1176 lines
26 KiB
JSON
1176 lines
26 KiB
JSON
{
|
|
"days": [
|
|
"2026-09-08",
|
|
"2026-09-09",
|
|
"2026-09-10",
|
|
"2026-09-11",
|
|
"2026-09-14",
|
|
"2026-09-15",
|
|
"2026-09-16",
|
|
"2026-09-17"
|
|
],
|
|
"db": "/home/harwinder/Development/ForexBot2/logs/newsdata.db",
|
|
"folds": [
|
|
{
|
|
"day": "2026-09-10",
|
|
"fit_seconds_full": 0.019712364999918464,
|
|
"fit_seconds_head": 0.01026680799998303,
|
|
"full": {
|
|
"accuracy": 0.6206896551724138,
|
|
"confusion": [
|
|
[
|
|
17,
|
|
3,
|
|
11
|
|
],
|
|
[
|
|
1,
|
|
17,
|
|
6
|
|
],
|
|
[
|
|
1,
|
|
11,
|
|
20
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5873015873015873,
|
|
"directional_hits": 37,
|
|
"directional_n": 63,
|
|
"hits": 54,
|
|
"macro_f1": 0.6259639877031181,
|
|
"n": 87,
|
|
"sign_agreement": 0.7551020408163265,
|
|
"sign_hits": 37,
|
|
"sign_n": 49
|
|
},
|
|
"head": {
|
|
"accuracy": 0.5172413793103449,
|
|
"confusion": [
|
|
[
|
|
14,
|
|
6,
|
|
11
|
|
],
|
|
[
|
|
1,
|
|
15,
|
|
8
|
|
],
|
|
[
|
|
6,
|
|
10,
|
|
16
|
|
]
|
|
],
|
|
"directional_accuracy": 0.47619047619047616,
|
|
"directional_hits": 30,
|
|
"directional_n": 63,
|
|
"hits": 45,
|
|
"macro_f1": 0.5205093414048637,
|
|
"n": 87,
|
|
"sign_agreement": 0.6382978723404256,
|
|
"sign_hits": 30,
|
|
"sign_n": 47
|
|
},
|
|
"keyword": {
|
|
"accuracy": 0.47126436781609193,
|
|
"confusion": [
|
|
[
|
|
12,
|
|
19,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
24,
|
|
0
|
|
],
|
|
[
|
|
1,
|
|
26,
|
|
5
|
|
]
|
|
],
|
|
"directional_accuracy": 0.2698412698412698,
|
|
"directional_hits": 17,
|
|
"directional_n": 63,
|
|
"hits": 41,
|
|
"macro_f1": 0.44395128266096,
|
|
"n": 87,
|
|
"sign_agreement": 0.9444444444444444,
|
|
"sign_hits": 17,
|
|
"sign_n": 18
|
|
},
|
|
"n_test": 87,
|
|
"n_train": 49
|
|
},
|
|
{
|
|
"day": "2026-09-11",
|
|
"fit_seconds_full": 1.3148657310000544,
|
|
"fit_seconds_head": 2.299214097999993,
|
|
"full": {
|
|
"accuracy": 0.5274725274725275,
|
|
"confusion": [
|
|
[
|
|
17,
|
|
4,
|
|
10
|
|
],
|
|
[
|
|
4,
|
|
11,
|
|
6
|
|
],
|
|
[
|
|
10,
|
|
9,
|
|
20
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5285714285714286,
|
|
"directional_hits": 37,
|
|
"directional_n": 70,
|
|
"hits": 48,
|
|
"macro_f1": 0.5235364396654719,
|
|
"n": 91,
|
|
"sign_agreement": 0.6491228070175439,
|
|
"sign_hits": 37,
|
|
"sign_n": 57
|
|
},
|
|
"head": {
|
|
"accuracy": 0.5384615384615384,
|
|
"confusion": [
|
|
[
|
|
20,
|
|
5,
|
|
6
|
|
],
|
|
[
|
|
3,
|
|
9,
|
|
9
|
|
],
|
|
[
|
|
9,
|
|
10,
|
|
20
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5714285714285714,
|
|
"directional_hits": 40,
|
|
"directional_n": 70,
|
|
"hits": 49,
|
|
"macro_f1": 0.5251537251537252,
|
|
"n": 91,
|
|
"sign_agreement": 0.7272727272727273,
|
|
"sign_hits": 40,
|
|
"sign_n": 55
|
|
},
|
|
"keyword": {
|
|
"accuracy": 0.3956043956043956,
|
|
"confusion": [
|
|
[
|
|
12,
|
|
18,
|
|
1
|
|
],
|
|
[
|
|
0,
|
|
21,
|
|
0
|
|
],
|
|
[
|
|
5,
|
|
31,
|
|
3
|
|
]
|
|
],
|
|
"directional_accuracy": 0.21428571428571427,
|
|
"directional_hits": 15,
|
|
"directional_n": 70,
|
|
"hits": 36,
|
|
"macro_f1": 0.36702444841979726,
|
|
"n": 91,
|
|
"sign_agreement": 0.7142857142857143,
|
|
"sign_hits": 15,
|
|
"sign_n": 21
|
|
},
|
|
"n_test": 91,
|
|
"n_train": 136
|
|
},
|
|
{
|
|
"day": "2026-09-14",
|
|
"fit_seconds_full": 1.6045974749999914,
|
|
"fit_seconds_head": 0.9094852939999782,
|
|
"full": {
|
|
"accuracy": 0.7692307692307693,
|
|
"confusion": [
|
|
[
|
|
62,
|
|
7,
|
|
11
|
|
],
|
|
[
|
|
0,
|
|
24,
|
|
8
|
|
],
|
|
[
|
|
1,
|
|
3,
|
|
14
|
|
]
|
|
],
|
|
"directional_accuracy": 0.7755102040816326,
|
|
"directional_hits": 76,
|
|
"directional_n": 98,
|
|
"hits": 100,
|
|
"macro_f1": 0.7144750674162439,
|
|
"n": 130,
|
|
"sign_agreement": 0.8636363636363636,
|
|
"sign_hits": 76,
|
|
"sign_n": 88
|
|
},
|
|
"head": {
|
|
"accuracy": 0.7230769230769231,
|
|
"confusion": [
|
|
[
|
|
59,
|
|
10,
|
|
11
|
|
],
|
|
[
|
|
0,
|
|
24,
|
|
8
|
|
],
|
|
[
|
|
1,
|
|
6,
|
|
11
|
|
]
|
|
],
|
|
"directional_accuracy": 0.7142857142857143,
|
|
"directional_hits": 70,
|
|
"directional_n": 98,
|
|
"hits": 94,
|
|
"macro_f1": 0.655952380952381,
|
|
"n": 130,
|
|
"sign_agreement": 0.8536585365853658,
|
|
"sign_hits": 70,
|
|
"sign_n": 82
|
|
},
|
|
"keyword": {
|
|
"accuracy": 0.5846153846153846,
|
|
"confusion": [
|
|
[
|
|
43,
|
|
35,
|
|
2
|
|
],
|
|
[
|
|
0,
|
|
32,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
17,
|
|
1
|
|
]
|
|
],
|
|
"directional_accuracy": 0.4489795918367347,
|
|
"directional_hits": 44,
|
|
"directional_n": 98,
|
|
"hits": 76,
|
|
"macro_f1": 0.44871640834634946,
|
|
"n": 130,
|
|
"sign_agreement": 0.9565217391304348,
|
|
"sign_hits": 44,
|
|
"sign_n": 46
|
|
},
|
|
"n_test": 130,
|
|
"n_train": 227
|
|
},
|
|
{
|
|
"day": "2026-09-15",
|
|
"fit_seconds_full": 2.186163542999907,
|
|
"fit_seconds_head": 1.7867370169999504,
|
|
"full": {
|
|
"accuracy": 0.6904761904761905,
|
|
"confusion": [
|
|
[
|
|
34,
|
|
1,
|
|
9
|
|
],
|
|
[
|
|
3,
|
|
14,
|
|
2
|
|
],
|
|
[
|
|
5,
|
|
6,
|
|
10
|
|
]
|
|
],
|
|
"directional_accuracy": 0.676923076923077,
|
|
"directional_hits": 44,
|
|
"directional_n": 65,
|
|
"hits": 58,
|
|
"macro_f1": 0.6556293835363602,
|
|
"n": 84,
|
|
"sign_agreement": 0.7586206896551724,
|
|
"sign_hits": 44,
|
|
"sign_n": 58
|
|
},
|
|
"head": {
|
|
"accuracy": 0.6190476190476191,
|
|
"confusion": [
|
|
[
|
|
27,
|
|
2,
|
|
15
|
|
],
|
|
[
|
|
2,
|
|
15,
|
|
2
|
|
],
|
|
[
|
|
4,
|
|
7,
|
|
10
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5692307692307692,
|
|
"directional_hits": 37,
|
|
"directional_n": 65,
|
|
"hits": 52,
|
|
"macro_f1": 0.6052132621900064,
|
|
"n": 84,
|
|
"sign_agreement": 0.6607142857142857,
|
|
"sign_hits": 37,
|
|
"sign_n": 56
|
|
},
|
|
"keyword": {
|
|
"accuracy": 0.44047619047619047,
|
|
"confusion": [
|
|
[
|
|
14,
|
|
28,
|
|
2
|
|
],
|
|
[
|
|
0,
|
|
19,
|
|
0
|
|
],
|
|
[
|
|
1,
|
|
16,
|
|
4
|
|
]
|
|
],
|
|
"directional_accuracy": 0.27692307692307694,
|
|
"directional_hits": 18,
|
|
"directional_n": 65,
|
|
"hits": 37,
|
|
"macro_f1": 0.4114290672096928,
|
|
"n": 84,
|
|
"sign_agreement": 0.8571428571428571,
|
|
"sign_hits": 18,
|
|
"sign_n": 21
|
|
},
|
|
"n_test": 84,
|
|
"n_train": 357
|
|
},
|
|
{
|
|
"day": "2026-09-16",
|
|
"fit_seconds_full": 1.5508948169999712,
|
|
"fit_seconds_head": 1.5201307759999736,
|
|
"full": {
|
|
"accuracy": 0.5565217391304348,
|
|
"confusion": [
|
|
[
|
|
19,
|
|
4,
|
|
8
|
|
],
|
|
[
|
|
4,
|
|
12,
|
|
3
|
|
],
|
|
[
|
|
21,
|
|
11,
|
|
33
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5416666666666666,
|
|
"directional_hits": 52,
|
|
"directional_n": 96,
|
|
"hits": 64,
|
|
"macro_f1": 0.544636794752471,
|
|
"n": 115,
|
|
"sign_agreement": 0.6419753086419753,
|
|
"sign_hits": 52,
|
|
"sign_n": 81
|
|
},
|
|
"head": {
|
|
"accuracy": 0.5478260869565217,
|
|
"confusion": [
|
|
[
|
|
16,
|
|
4,
|
|
11
|
|
],
|
|
[
|
|
3,
|
|
13,
|
|
3
|
|
],
|
|
[
|
|
15,
|
|
16,
|
|
34
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5208333333333334,
|
|
"directional_hits": 50,
|
|
"directional_n": 96,
|
|
"hits": 63,
|
|
"macro_f1": 0.5313592012707057,
|
|
"n": 115,
|
|
"sign_agreement": 0.6578947368421053,
|
|
"sign_hits": 50,
|
|
"sign_n": 76
|
|
},
|
|
"keyword": {
|
|
"accuracy": 0.33043478260869563,
|
|
"confusion": [
|
|
[
|
|
11,
|
|
17,
|
|
3
|
|
],
|
|
[
|
|
0,
|
|
18,
|
|
1
|
|
],
|
|
[
|
|
3,
|
|
53,
|
|
9
|
|
]
|
|
],
|
|
"directional_accuracy": 0.20833333333333334,
|
|
"directional_hits": 20,
|
|
"directional_n": 96,
|
|
"hits": 38,
|
|
"macro_f1": 0.3520355725963203,
|
|
"n": 115,
|
|
"sign_agreement": 0.7692307692307693,
|
|
"sign_hits": 20,
|
|
"sign_n": 26
|
|
},
|
|
"n_test": 115,
|
|
"n_train": 441
|
|
},
|
|
{
|
|
"day": "2026-09-17",
|
|
"fit_seconds_full": 3.295582043999957,
|
|
"fit_seconds_head": 2.6792992629999617,
|
|
"full": {
|
|
"accuracy": 0.6833333333333333,
|
|
"confusion": [
|
|
[
|
|
16,
|
|
4,
|
|
6
|
|
],
|
|
[
|
|
1,
|
|
5,
|
|
0
|
|
],
|
|
[
|
|
2,
|
|
6,
|
|
20
|
|
]
|
|
],
|
|
"directional_accuracy": 0.6666666666666666,
|
|
"directional_hits": 36,
|
|
"directional_n": 54,
|
|
"hits": 41,
|
|
"macro_f1": 0.6426807760141093,
|
|
"n": 60,
|
|
"sign_agreement": 0.8181818181818182,
|
|
"sign_hits": 36,
|
|
"sign_n": 44
|
|
},
|
|
"head": {
|
|
"accuracy": 0.75,
|
|
"confusion": [
|
|
[
|
|
18,
|
|
4,
|
|
4
|
|
],
|
|
[
|
|
0,
|
|
5,
|
|
1
|
|
],
|
|
[
|
|
2,
|
|
4,
|
|
22
|
|
]
|
|
],
|
|
"directional_accuracy": 0.7407407407407407,
|
|
"directional_hits": 40,
|
|
"directional_n": 54,
|
|
"hits": 45,
|
|
"macro_f1": 0.7029748283752859,
|
|
"n": 60,
|
|
"sign_agreement": 0.8695652173913043,
|
|
"sign_hits": 40,
|
|
"sign_n": 46
|
|
},
|
|
"keyword": {
|
|
"accuracy": 0.4,
|
|
"confusion": [
|
|
[
|
|
9,
|
|
17,
|
|
0
|
|
],
|
|
[
|
|
0,
|
|
6,
|
|
0
|
|
],
|
|
[
|
|
2,
|
|
17,
|
|
9
|
|
]
|
|
],
|
|
"directional_accuracy": 0.3333333333333333,
|
|
"directional_hits": 18,
|
|
"directional_n": 54,
|
|
"hits": 24,
|
|
"macro_f1": 0.4112808460634548,
|
|
"n": 60,
|
|
"sign_agreement": 0.9,
|
|
"sign_hits": 18,
|
|
"sign_n": 20
|
|
},
|
|
"n_test": 60,
|
|
"n_train": 556
|
|
}
|
|
],
|
|
"generated_at": "20260917_2239",
|
|
"in_sample": {
|
|
"headline_description": {
|
|
"accuracy": 0.974025974025974,
|
|
"confusion": [
|
|
[
|
|
249,
|
|
4,
|
|
2
|
|
],
|
|
[
|
|
1,
|
|
134,
|
|
2
|
|
],
|
|
[
|
|
0,
|
|
7,
|
|
217
|
|
]
|
|
],
|
|
"directional_accuracy": 0.9728601252609603,
|
|
"directional_hits": 466,
|
|
"directional_n": 479,
|
|
"hits": 600,
|
|
"macro_f1": 0.9705913742222895,
|
|
"mae": 0.21260655681818183,
|
|
"n": 616,
|
|
"sign_agreement": 0.9957264957264957,
|
|
"sign_hits": 466,
|
|
"sign_n": 468,
|
|
"spearman": 0.9164145935732386
|
|
},
|
|
"headline_only": {
|
|
"accuracy": 0.9545454545454546,
|
|
"confusion": [
|
|
[
|
|
245,
|
|
4,
|
|
6
|
|
],
|
|
[
|
|
1,
|
|
132,
|
|
4
|
|
],
|
|
[
|
|
2,
|
|
11,
|
|
211
|
|
]
|
|
],
|
|
"directional_accuracy": 0.9519832985386222,
|
|
"directional_hits": 456,
|
|
"directional_n": 479,
|
|
"hits": 588,
|
|
"macro_f1": 0.9506823803709369,
|
|
"mae": 0.2395537012987013,
|
|
"n": 616,
|
|
"sign_agreement": 0.9827586206896551,
|
|
"sign_hits": 456,
|
|
"sign_n": 464,
|
|
"spearman": 0.906803357215335
|
|
}
|
|
},
|
|
"label_distribution": {
|
|
"bearish": 255,
|
|
"bullish": 224,
|
|
"neutral": 137
|
|
},
|
|
"model": {
|
|
"label_distribution": {
|
|
"bearish": 250,
|
|
"bullish": 221,
|
|
"neutral": 135
|
|
},
|
|
"model_path": "/home/harwinder/Development/ML-NEWS/models/xauusd_sentiment.joblib",
|
|
"model_type": "domain_augmented_tfidf_logistic_regression",
|
|
"neutral_threshold": 0.05,
|
|
"sklearn_version": "1.9.1",
|
|
"trained_at": "2026-09-17T18:54:46.714699+00:00",
|
|
"training_examples": 606
|
|
},
|
|
"pooled_oof": {
|
|
"full": {
|
|
"accuracy": 0.6437389770723104,
|
|
"confusion": [
|
|
[
|
|
165,
|
|
23,
|
|
55
|
|
],
|
|
[
|
|
13,
|
|
83,
|
|
25
|
|
],
|
|
[
|
|
40,
|
|
46,
|
|
117
|
|
]
|
|
],
|
|
"directional_accuracy": 0.6322869955156951,
|
|
"directional_hits": 282,
|
|
"directional_n": 446,
|
|
"hits": 365,
|
|
"macro_f1": 0.6362979163521462,
|
|
"mae": 0.32247548500881834,
|
|
"n": 567,
|
|
"sign_agreement": 0.7480106100795756,
|
|
"sign_hits": 282,
|
|
"sign_n": 377,
|
|
"spearman": 0.5385943703135879
|
|
},
|
|
"full_train_ge_300": {
|
|
"accuracy": 0.6293436293436293,
|
|
"confusion": [
|
|
[
|
|
69,
|
|
9,
|
|
23
|
|
],
|
|
[
|
|
8,
|
|
31,
|
|
5
|
|
],
|
|
[
|
|
28,
|
|
23,
|
|
63
|
|
]
|
|
],
|
|
"directional_accuracy": 0.6139534883720931,
|
|
"directional_hits": 132,
|
|
"directional_n": 215,
|
|
"hits": 163,
|
|
"macro_f1": 0.6213254370997571,
|
|
"mae": 0.32819111969111964,
|
|
"n": 259,
|
|
"sign_agreement": 0.7213114754098361,
|
|
"sign_hits": 132,
|
|
"sign_n": 183,
|
|
"spearman": 0.5313185456204952
|
|
},
|
|
"head": {
|
|
"accuracy": 0.6137566137566137,
|
|
"confusion": [
|
|
[
|
|
154,
|
|
31,
|
|
58
|
|
],
|
|
[
|
|
9,
|
|
81,
|
|
31
|
|
],
|
|
[
|
|
37,
|
|
53,
|
|
113
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5986547085201793,
|
|
"directional_hits": 267,
|
|
"directional_n": 446,
|
|
"hits": 348,
|
|
"macro_f1": 0.6065726171570165,
|
|
"mae": 0.3303056437389771,
|
|
"n": 567,
|
|
"sign_agreement": 0.7375690607734806,
|
|
"sign_hits": 267,
|
|
"sign_n": 362,
|
|
"spearman": 0.545541212678629
|
|
},
|
|
"keyword": {
|
|
"accuracy": 0.4444444444444444,
|
|
"confusion": [
|
|
[
|
|
101,
|
|
134,
|
|
8
|
|
],
|
|
[
|
|
0,
|
|
120,
|
|
1
|
|
],
|
|
[
|
|
12,
|
|
160,
|
|
31
|
|
]
|
|
],
|
|
"directional_accuracy": 0.29596412556053814,
|
|
"directional_hits": 132,
|
|
"directional_n": 446,
|
|
"hits": 252,
|
|
"macro_f1": 0.4237192980333369,
|
|
"n": 567,
|
|
"sign_agreement": 0.868421052631579,
|
|
"sign_hits": 132,
|
|
"sign_n": 152
|
|
}
|
|
},
|
|
"reprobe": {
|
|
"deepseek": {
|
|
"batch_ms_max": 1721.6889519999086,
|
|
"batch_ms_mean": 1605.3522224999597,
|
|
"cache_hits": 0,
|
|
"calls": 4,
|
|
"failed_calls": 0,
|
|
"headlines_sent": 100,
|
|
"model": "deepseek-v4-flash",
|
|
"per_headline_ms": 64.21408889999839,
|
|
"prompt_sha1": "d32e04c6a87d019f326f111b39c99b6dd8ce794d"
|
|
},
|
|
"identical_xau": 42,
|
|
"n_live": 100,
|
|
"oof_vs_archived": {
|
|
"accuracy": 0.6236559139784946,
|
|
"confusion": [
|
|
[
|
|
21,
|
|
5,
|
|
11
|
|
],
|
|
[
|
|
3,
|
|
20,
|
|
1
|
|
],
|
|
[
|
|
7,
|
|
8,
|
|
17
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5507246376811594,
|
|
"directional_hits": 38,
|
|
"directional_n": 69,
|
|
"hits": 58,
|
|
"macro_f1": 0.6255928313229232,
|
|
"n": 93,
|
|
"sign_agreement": 0.6785714285714286,
|
|
"sign_hits": 38,
|
|
"sign_n": 56
|
|
},
|
|
"oof_vs_today": {
|
|
"accuracy": 0.6344086021505376,
|
|
"confusion": [
|
|
[
|
|
21,
|
|
5,
|
|
10
|
|
],
|
|
[
|
|
3,
|
|
20,
|
|
1
|
|
],
|
|
[
|
|
7,
|
|
8,
|
|
18
|
|
]
|
|
],
|
|
"directional_accuracy": 0.5652173913043478,
|
|
"directional_hits": 39,
|
|
"directional_n": 69,
|
|
"hits": 59,
|
|
"macro_f1": 0.6364217396323419,
|
|
"n": 93,
|
|
"sign_agreement": 0.6964285714285714,
|
|
"sign_hits": 39,
|
|
"sign_n": 56
|
|
},
|
|
"same_prompt": 100,
|
|
"self_agreement": {
|
|
"accuracy": 0.9,
|
|
"confusion": [
|
|
[
|
|
38,
|
|
0,
|
|
1
|
|
],
|
|
[
|
|
0,
|
|
21,
|
|
5
|
|
],
|
|
[
|
|
0,
|
|
4,
|
|
31
|
|
]
|
|
],
|
|
"directional_accuracy": 0.9324324324324325,
|
|
"directional_hits": 69,
|
|
"directional_n": 74,
|
|
"hits": 90,
|
|
"macro_f1": 0.8905511699629347,
|
|
"mae": 0.08900000000000001,
|
|
"n": 100,
|
|
"sign_agreement": 0.9857142857142858,
|
|
"sign_hits": 69,
|
|
"sign_n": 70,
|
|
"spearman": 0.9518725584019171
|
|
}
|
|
},
|
|
"routing": [
|
|
{
|
|
"confidence_min": 0.0,
|
|
"hybrid_agreement": 0.6437389770723104,
|
|
"kept": 567,
|
|
"kept_accuracy": 0.6437389770723104,
|
|
"kept_share": 1.0,
|
|
"llm_calls_saved_share": 1.0,
|
|
"margin_min": 0.0
|
|
},
|
|
{
|
|
"confidence_min": 0.0,
|
|
"hybrid_agreement": 0.7336860670194003,
|
|
"kept": 464,
|
|
"kept_accuracy": 0.6745689655172413,
|
|
"kept_share": 0.818342151675485,
|
|
"llm_calls_saved_share": 0.818342151675485,
|
|
"margin_min": 0.1
|
|
},
|
|
{
|
|
"confidence_min": 0.0,
|
|
"hybrid_agreement": 0.8130511463844797,
|
|
"kept": 369,
|
|
"kept_accuracy": 0.7127371273712737,
|
|
"kept_share": 0.6507936507936508,
|
|
"llm_calls_saved_share": 0.6507936507936508,
|
|
"margin_min": 0.2
|
|
},
|
|
{
|
|
"confidence_min": 0.0,
|
|
"hybrid_agreement": 0.8818342151675485,
|
|
"kept": 287,
|
|
"kept_accuracy": 0.7665505226480837,
|
|
"kept_share": 0.5061728395061729,
|
|
"llm_calls_saved_share": 0.5061728395061729,
|
|
"margin_min": 0.3
|
|
},
|
|
{
|
|
"confidence_min": 0.0,
|
|
"hybrid_agreement": 0.9294532627865961,
|
|
"kept": 216,
|
|
"kept_accuracy": 0.8148148148148148,
|
|
"kept_share": 0.38095238095238093,
|
|
"llm_calls_saved_share": 0.38095238095238093,
|
|
"margin_min": 0.4
|
|
},
|
|
{
|
|
"confidence_min": 0.5,
|
|
"hybrid_agreement": 0.8042328042328042,
|
|
"kept": 393,
|
|
"kept_accuracy": 0.7175572519083969,
|
|
"kept_share": 0.6931216931216931,
|
|
"llm_calls_saved_share": 0.6931216931216931,
|
|
"margin_min": 0.0
|
|
},
|
|
{
|
|
"confidence_min": 0.5,
|
|
"hybrid_agreement": 0.8077601410934744,
|
|
"kept": 389,
|
|
"kept_accuracy": 0.7197943444730077,
|
|
"kept_share": 0.6860670194003528,
|
|
"llm_calls_saved_share": 0.6860670194003528,
|
|
"margin_min": 0.1
|
|
},
|
|
{
|
|
"confidence_min": 0.5,
|
|
"hybrid_agreement": 0.8253968253968254,
|
|
"kept": 360,
|
|
"kept_accuracy": 0.725,
|
|
"kept_share": 0.6349206349206349,
|
|
"llm_calls_saved_share": 0.6349206349206349,
|
|
"margin_min": 0.2
|
|
},
|
|
{
|
|
"confidence_min": 0.5,
|
|
"hybrid_agreement": 0.8818342151675485,
|
|
"kept": 287,
|
|
"kept_accuracy": 0.7665505226480837,
|
|
"kept_share": 0.5061728395061729,
|
|
"llm_calls_saved_share": 0.5061728395061729,
|
|
"margin_min": 0.3
|
|
},
|
|
{
|
|
"confidence_min": 0.5,
|
|
"hybrid_agreement": 0.9294532627865961,
|
|
"kept": 216,
|
|
"kept_accuracy": 0.8148148148148148,
|
|
"kept_share": 0.38095238095238093,
|
|
"llm_calls_saved_share": 0.38095238095238093,
|
|
"margin_min": 0.4
|
|
},
|
|
{
|
|
"confidence_min": 0.6,
|
|
"hybrid_agreement": 0.9153439153439153,
|
|
"kept": 251,
|
|
"kept_accuracy": 0.8087649402390438,
|
|
"kept_share": 0.4426807760141093,
|
|
"llm_calls_saved_share": 0.4426807760141093,
|
|
"margin_min": 0.0
|
|
},
|
|
{
|
|
"confidence_min": 0.6,
|
|
"hybrid_agreement": 0.9153439153439153,
|
|
"kept": 251,
|
|
"kept_accuracy": 0.8087649402390438,
|
|
"kept_share": 0.4426807760141093,
|
|
"llm_calls_saved_share": 0.4426807760141093,
|
|
"margin_min": 0.1
|
|
},
|
|
{
|
|
"confidence_min": 0.6,
|
|
"hybrid_agreement": 0.9153439153439153,
|
|
"kept": 251,
|
|
"kept_accuracy": 0.8087649402390438,
|
|
"kept_share": 0.4426807760141093,
|
|
"llm_calls_saved_share": 0.4426807760141093,
|
|
"margin_min": 0.2
|
|
},
|
|
{
|
|
"confidence_min": 0.6,
|
|
"hybrid_agreement": 0.9153439153439153,
|
|
"kept": 248,
|
|
"kept_accuracy": 0.8064516129032258,
|
|
"kept_share": 0.43738977072310403,
|
|
"llm_calls_saved_share": 0.43738977072310403,
|
|
"margin_min": 0.3
|
|
},
|
|
{
|
|
"confidence_min": 0.6,
|
|
"hybrid_agreement": 0.9294532627865961,
|
|
"kept": 216,
|
|
"kept_accuracy": 0.8148148148148148,
|
|
"kept_share": 0.38095238095238093,
|
|
"llm_calls_saved_share": 0.38095238095238093,
|
|
"margin_min": 0.4
|
|
},
|
|
{
|
|
"confidence_min": 0.7,
|
|
"hybrid_agreement": 0.9647266313932981,
|
|
"kept": 159,
|
|
"kept_accuracy": 0.8742138364779874,
|
|
"kept_share": 0.2804232804232804,
|
|
"llm_calls_saved_share": 0.2804232804232804,
|
|
"margin_min": 0.0
|
|
},
|
|
{
|
|
"confidence_min": 0.7,
|
|
"hybrid_agreement": 0.9647266313932981,
|
|
"kept": 159,
|
|
"kept_accuracy": 0.8742138364779874,
|
|
"kept_share": 0.2804232804232804,
|
|
"llm_calls_saved_share": 0.2804232804232804,
|
|
"margin_min": 0.1
|
|
},
|
|
{
|
|
"confidence_min": 0.7,
|
|
"hybrid_agreement": 0.9647266313932981,
|
|
"kept": 159,
|
|
"kept_accuracy": 0.8742138364779874,
|
|
"kept_share": 0.2804232804232804,
|
|
"llm_calls_saved_share": 0.2804232804232804,
|
|
"margin_min": 0.2
|
|
},
|
|
{
|
|
"confidence_min": 0.7,
|
|
"hybrid_agreement": 0.9647266313932981,
|
|
"kept": 159,
|
|
"kept_accuracy": 0.8742138364779874,
|
|
"kept_share": 0.2804232804232804,
|
|
"llm_calls_saved_share": 0.2804232804232804,
|
|
"margin_min": 0.3
|
|
},
|
|
{
|
|
"confidence_min": 0.7,
|
|
"hybrid_agreement": 0.9647266313932981,
|
|
"kept": 159,
|
|
"kept_accuracy": 0.8742138364779874,
|
|
"kept_share": 0.2804232804232804,
|
|
"llm_calls_saved_share": 0.2804232804232804,
|
|
"margin_min": 0.4
|
|
},
|
|
{
|
|
"confidence_min": 0.8,
|
|
"hybrid_agreement": 0.9929453262786596,
|
|
"kept": 96,
|
|
"kept_accuracy": 0.9583333333333334,
|
|
"kept_share": 0.1693121693121693,
|
|
"llm_calls_saved_share": 0.1693121693121693,
|
|
"margin_min": 0.0
|
|
},
|
|
{
|
|
"confidence_min": 0.8,
|
|
"hybrid_agreement": 0.9929453262786596,
|
|
"kept": 96,
|
|
"kept_accuracy": 0.9583333333333334,
|
|
"kept_share": 0.1693121693121693,
|
|
"llm_calls_saved_share": 0.1693121693121693,
|
|
"margin_min": 0.1
|
|
},
|
|
{
|
|
"confidence_min": 0.8,
|
|
"hybrid_agreement": 0.9929453262786596,
|
|
"kept": 96,
|
|
"kept_accuracy": 0.9583333333333334,
|
|
"kept_share": 0.1693121693121693,
|
|
"llm_calls_saved_share": 0.1693121693121693,
|
|
"margin_min": 0.2
|
|
},
|
|
{
|
|
"confidence_min": 0.8,
|
|
"hybrid_agreement": 0.9929453262786596,
|
|
"kept": 96,
|
|
"kept_accuracy": 0.9583333333333334,
|
|
"kept_share": 0.1693121693121693,
|
|
"llm_calls_saved_share": 0.1693121693121693,
|
|
"margin_min": 0.3
|
|
},
|
|
{
|
|
"confidence_min": 0.8,
|
|
"hybrid_agreement": 0.9929453262786596,
|
|
"kept": 96,
|
|
"kept_accuracy": 0.9583333333333334,
|
|
"kept_share": 0.1693121693121693,
|
|
"llm_calls_saved_share": 0.1693121693121693,
|
|
"margin_min": 0.4
|
|
},
|
|
{
|
|
"confidence_min": 0.9,
|
|
"hybrid_agreement": 1.0,
|
|
"kept": 37,
|
|
"kept_accuracy": 1.0,
|
|
"kept_share": 0.06525573192239859,
|
|
"llm_calls_saved_share": 0.06525573192239859,
|
|
"margin_min": 0.0
|
|
},
|
|
{
|
|
"confidence_min": 0.9,
|
|
"hybrid_agreement": 1.0,
|
|
"kept": 37,
|
|
"kept_accuracy": 1.0,
|
|
"kept_share": 0.06525573192239859,
|
|
"llm_calls_saved_share": 0.06525573192239859,
|
|
"margin_min": 0.1
|
|
},
|
|
{
|
|
"confidence_min": 0.9,
|
|
"hybrid_agreement": 1.0,
|
|
"kept": 37,
|
|
"kept_accuracy": 1.0,
|
|
"kept_share": 0.06525573192239859,
|
|
"llm_calls_saved_share": 0.06525573192239859,
|
|
"margin_min": 0.2
|
|
},
|
|
{
|
|
"confidence_min": 0.9,
|
|
"hybrid_agreement": 1.0,
|
|
"kept": 37,
|
|
"kept_accuracy": 1.0,
|
|
"kept_share": 0.06525573192239859,
|
|
"llm_calls_saved_share": 0.06525573192239859,
|
|
"margin_min": 0.3
|
|
},
|
|
{
|
|
"confidence_min": 0.9,
|
|
"hybrid_agreement": 1.0,
|
|
"kept": 37,
|
|
"kept_accuracy": 1.0,
|
|
"kept_share": 0.06525573192239859,
|
|
"llm_calls_saved_share": 0.06525573192239859,
|
|
"margin_min": 0.4
|
|
}
|
|
],
|
|
"rows": 616,
|
|
"rows_per_day": {
|
|
"2026-09-08": 31,
|
|
"2026-09-09": 18,
|
|
"2026-09-10": 87,
|
|
"2026-09-11": 91,
|
|
"2026-09-14": 130,
|
|
"2026-09-15": 84,
|
|
"2026-09-16": 115,
|
|
"2026-09-17": 60
|
|
},
|
|
"threshold": 0.05
|
|
}
|