Initial import: XAUUSD news sentiment models, evals, and data tooling
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Harwinder Sekhon
Binary file not shown.
@@ -0,0 +1,169 @@
|
||||
{
|
||||
"evaluation": {
|
||||
"stratified_holdout": {
|
||||
"accuracy": 0.7295081967213115,
|
||||
"balanced_accuracy": 0.7079012345679012,
|
||||
"classification_report": {
|
||||
"accuracy": 0.7295081967213115,
|
||||
"bearish": {
|
||||
"f1-score": 0.82,
|
||||
"precision": 0.82,
|
||||
"recall": 0.82,
|
||||
"support": 50.0
|
||||
},
|
||||
"bullish": {
|
||||
"f1-score": 0.7032967032967034,
|
||||
"precision": 0.6956521739130435,
|
||||
"recall": 0.7111111111111111,
|
||||
"support": 45.0
|
||||
},
|
||||
"macro avg": {
|
||||
"f1-score": 0.7090234294007879,
|
||||
"precision": 0.7103455964325529,
|
||||
"recall": 0.7079012345679012,
|
||||
"support": 122.0
|
||||
},
|
||||
"neutral": {
|
||||
"f1-score": 0.6037735849056604,
|
||||
"precision": 0.6153846153846154,
|
||||
"recall": 0.5925925925925926,
|
||||
"support": 27.0
|
||||
},
|
||||
"weighted avg": {
|
||||
"f1-score": 0.7291003150885613,
|
||||
"precision": 0.7288502659137014,
|
||||
"recall": 0.7295081967213115,
|
||||
"support": 122.0
|
||||
}
|
||||
},
|
||||
"confusion_matrix": {
|
||||
"labels": [
|
||||
"bearish",
|
||||
"neutral",
|
||||
"bullish"
|
||||
],
|
||||
"matrix": [
|
||||
[
|
||||
41,
|
||||
2,
|
||||
7
|
||||
],
|
||||
[
|
||||
4,
|
||||
16,
|
||||
7
|
||||
],
|
||||
[
|
||||
5,
|
||||
8,
|
||||
32
|
||||
]
|
||||
]
|
||||
},
|
||||
"macro_f1": 0.7090234294007879,
|
||||
"score_mae": 0.2818823212040985,
|
||||
"test_examples": 122
|
||||
},
|
||||
"temporal_holdout": {
|
||||
"accuracy": 0.6967213114754098,
|
||||
"balanced_accuracy": 0.6945570783661773,
|
||||
"classification_report": {
|
||||
"accuracy": 0.6967213114754098,
|
||||
"bearish": {
|
||||
"f1-score": 0.7380952380952381,
|
||||
"precision": 0.775,
|
||||
"recall": 0.7045454545454546,
|
||||
"support": 44.0
|
||||
},
|
||||
"bullish": {
|
||||
"f1-score": 0.7321428571428571,
|
||||
"precision": 0.7735849056603774,
|
||||
"recall": 0.6949152542372882,
|
||||
"support": 59.0
|
||||
},
|
||||
"macro avg": {
|
||||
"f1-score": 0.6706349206349206,
|
||||
"precision": 0.665620255909781,
|
||||
"recall": 0.6945570783661773,
|
||||
"support": 122.0
|
||||
},
|
||||
"neutral": {
|
||||
"f1-score": 0.5416666666666666,
|
||||
"precision": 0.4482758620689655,
|
||||
"recall": 0.6842105263157895,
|
||||
"support": 19.0
|
||||
},
|
||||
"weighted avg": {
|
||||
"f1-score": 0.7046252927400469,
|
||||
"precision": 0.7234323837153492,
|
||||
"recall": 0.6967213114754098,
|
||||
"support": 122.0
|
||||
}
|
||||
},
|
||||
"confusion_matrix": {
|
||||
"labels": [
|
||||
"bearish",
|
||||
"neutral",
|
||||
"bullish"
|
||||
],
|
||||
"matrix": [
|
||||
[
|
||||
31,
|
||||
5,
|
||||
8
|
||||
],
|
||||
[
|
||||
2,
|
||||
13,
|
||||
4
|
||||
],
|
||||
[
|
||||
7,
|
||||
11,
|
||||
41
|
||||
]
|
||||
]
|
||||
},
|
||||
"macro_f1": 0.6706349206349206,
|
||||
"score_mae": 0.32407588182906516,
|
||||
"test_examples": 122,
|
||||
"test_from": "2026-09-16T08:27:46+00:00",
|
||||
"train_through": "2026-09-16T08:25:31+00:00"
|
||||
}
|
||||
},
|
||||
"format_version": 3,
|
||||
"model_path": "models/xauusd_sentiment.joblib",
|
||||
"model_type": "domain_augmented_tfidf_logistic_regression",
|
||||
"score_definition": "P(XAUUSD bullish) - P(XAUUSD bearish)",
|
||||
"sklearn_version": "1.9.1",
|
||||
"target": "XAUUSD",
|
||||
"trained_at": "2026-09-17T18:54:46.714699+00:00",
|
||||
"training": {
|
||||
"database": "newsdata.db",
|
||||
"domain_features": [
|
||||
"explicit_xau_direction",
|
||||
"usd_direction",
|
||||
"treasury_yield_direction",
|
||||
"hawkish_or_dovish_policy"
|
||||
],
|
||||
"duplicates_collapsed": 10,
|
||||
"examples": 606,
|
||||
"label_distribution": {
|
||||
"bearish": 250,
|
||||
"bullish": 221,
|
||||
"neutral": 135
|
||||
},
|
||||
"max_features": 50000,
|
||||
"min_df": 2,
|
||||
"neutral_threshold": 0.05,
|
||||
"random_state": 42,
|
||||
"rows_without_target_label": 930,
|
||||
"score_source": "llm",
|
||||
"skipped_rows": 0,
|
||||
"source_rows": 1546,
|
||||
"target": "XAUUSD",
|
||||
"target_labeled_rows": 616,
|
||||
"target_score_key": "XAU",
|
||||
"test_size": 0.2
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,186 @@
|
||||
{
|
||||
"evaluation": {
|
||||
"stratified_holdout": {
|
||||
"accuracy": 0.6927480916030534,
|
||||
"balanced_accuracy": 0.689958210237739,
|
||||
"classification_report": {
|
||||
"accuracy": 0.6927480916030534,
|
||||
"bearish": {
|
||||
"f1-score": 0.752,
|
||||
"precision": 0.7305699481865285,
|
||||
"recall": 0.7747252747252747,
|
||||
"support": 182.0
|
||||
},
|
||||
"bullish": {
|
||||
"f1-score": 0.6230529595015576,
|
||||
"precision": 0.6369426751592356,
|
||||
"recall": 0.6097560975609756,
|
||||
"support": 164.0
|
||||
},
|
||||
"macro avg": {
|
||||
"f1-score": 0.6894115925611253,
|
||||
"precision": 0.6895540162110402,
|
||||
"recall": 0.689958210237739,
|
||||
"support": 524.0
|
||||
},
|
||||
"neutral": {
|
||||
"f1-score": 0.6931818181818182,
|
||||
"precision": 0.7011494252873564,
|
||||
"recall": 0.6853932584269663,
|
||||
"support": 178.0
|
||||
},
|
||||
"weighted avg": {
|
||||
"f1-score": 0.6916623072416396,
|
||||
"precision": 0.6912727614450616,
|
||||
"recall": 0.6927480916030534,
|
||||
"support": 524.0
|
||||
}
|
||||
},
|
||||
"confusion_matrix": {
|
||||
"labels": [
|
||||
"bearish",
|
||||
"neutral",
|
||||
"bullish"
|
||||
],
|
||||
"matrix": [
|
||||
[
|
||||
141,
|
||||
18,
|
||||
23
|
||||
],
|
||||
[
|
||||
22,
|
||||
122,
|
||||
34
|
||||
],
|
||||
[
|
||||
30,
|
||||
34,
|
||||
100
|
||||
]
|
||||
]
|
||||
},
|
||||
"macro_f1": 0.6894115925611253,
|
||||
"score_mae": 0.2644602373963052,
|
||||
"test_examples": 524
|
||||
},
|
||||
"temporal_holdout": {
|
||||
"accuracy": 0.666030534351145,
|
||||
"balanced_accuracy": 0.6572494770227411,
|
||||
"classification_report": {
|
||||
"accuracy": 0.666030534351145,
|
||||
"bearish": {
|
||||
"f1-score": 0.7040498442367601,
|
||||
"precision": 0.773972602739726,
|
||||
"recall": 0.6457142857142857,
|
||||
"support": 175.0
|
||||
},
|
||||
"bullish": {
|
||||
"f1-score": 0.5647840531561462,
|
||||
"precision": 0.5483870967741935,
|
||||
"recall": 0.5821917808219178,
|
||||
"support": 146.0
|
||||
},
|
||||
"macro avg": {
|
||||
"f1-score": 0.6592513617287779,
|
||||
"precision": 0.6664965814523229,
|
||||
"recall": 0.6572494770227411,
|
||||
"support": 524.0
|
||||
},
|
||||
"neutral": {
|
||||
"f1-score": 0.7089201877934272,
|
||||
"precision": 0.6771300448430493,
|
||||
"recall": 0.7438423645320197,
|
||||
"support": 203.0
|
||||
},
|
||||
"weighted avg": {
|
||||
"f1-score": 0.6671335737104888,
|
||||
"precision": 0.6736013754038612,
|
||||
"recall": 0.666030534351145,
|
||||
"support": 524.0
|
||||
}
|
||||
},
|
||||
"confusion_matrix": {
|
||||
"labels": [
|
||||
"bearish",
|
||||
"neutral",
|
||||
"bullish"
|
||||
],
|
||||
"matrix": [
|
||||
[
|
||||
113,
|
||||
31,
|
||||
31
|
||||
],
|
||||
[
|
||||
13,
|
||||
151,
|
||||
39
|
||||
],
|
||||
[
|
||||
20,
|
||||
41,
|
||||
85
|
||||
]
|
||||
]
|
||||
},
|
||||
"macro_f1": 0.6592513617287779,
|
||||
"score_mae": 0.24771861749332214,
|
||||
"test_examples": 524,
|
||||
"test_from": "2026-09-21T07:34:39+00:00",
|
||||
"train_through": "2026-09-21T07:33:06+00:00"
|
||||
}
|
||||
},
|
||||
"format_version": 3,
|
||||
"model_path": "models/candidates/xau_all_20260922.joblib",
|
||||
"model_type": "domain_augmented_tfidf_logistic_regression",
|
||||
"score_definition": "P(XAUUSD bullish) - P(XAUUSD bearish)",
|
||||
"sklearn_version": "1.9.1",
|
||||
"target": "XAUUSD",
|
||||
"trained_at": "2026-09-23T00:35:47.838884+00:00",
|
||||
"training": {
|
||||
"database": "newsdata.db",
|
||||
"domain_features": [
|
||||
"explicit_xau_direction",
|
||||
"usd_direction",
|
||||
"treasury_yield_direction",
|
||||
"hawkish_or_dovish_policy"
|
||||
],
|
||||
"duplicates_collapsed": 17,
|
||||
"examples": 2620,
|
||||
"extra_labels": "eval/results/backfill_labels.jsonl",
|
||||
"label_distribution": {
|
||||
"bearish": 909,
|
||||
"bullish": 818,
|
||||
"neutral": 893
|
||||
},
|
||||
"label_priority": [
|
||||
"xau_score",
|
||||
"json",
|
||||
"extra_labels"
|
||||
],
|
||||
"label_sources": {
|
||||
"extra_labels": 933,
|
||||
"json": 640,
|
||||
"xau_score": 1064
|
||||
},
|
||||
"max_features": 50000,
|
||||
"min_df": 2,
|
||||
"neutral_threshold": 0.05,
|
||||
"published_range": [
|
||||
"2026-09-08T13:35:37+00:00",
|
||||
"2026-09-22T18:13:29+00:00"
|
||||
],
|
||||
"random_state": 42,
|
||||
"rows_without_target_label": 24,
|
||||
"scope": "all",
|
||||
"score_source": "llm",
|
||||
"skipped_rows": 0,
|
||||
"source_rows": 2661,
|
||||
"target": "XAUUSD",
|
||||
"target_labeled_rows": 2637,
|
||||
"target_score_key": "XAU",
|
||||
"test_feed_rows": 0,
|
||||
"test_size": 0.2
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,185 @@
|
||||
{
|
||||
"evaluation": {
|
||||
"stratified_holdout": {
|
||||
"accuracy": 0.8042328042328042,
|
||||
"balanced_accuracy": 0.7887686387686387,
|
||||
"classification_report": {
|
||||
"accuracy": 0.8042328042328042,
|
||||
"bearish": {
|
||||
"f1-score": 0.8435374149659864,
|
||||
"precision": 0.8266666666666667,
|
||||
"recall": 0.8611111111111112,
|
||||
"support": 72.0
|
||||
},
|
||||
"bullish": {
|
||||
"f1-score": 0.8,
|
||||
"precision": 0.7948717948717948,
|
||||
"recall": 0.8051948051948052,
|
||||
"support": 77.0
|
||||
},
|
||||
"macro avg": {
|
||||
"f1-score": 0.7934598400763816,
|
||||
"precision": 0.7997720797720796,
|
||||
"recall": 0.7887686387686387,
|
||||
"support": 189.0
|
||||
},
|
||||
"neutral": {
|
||||
"f1-score": 0.7368421052631579,
|
||||
"precision": 0.7777777777777778,
|
||||
"recall": 0.7,
|
||||
"support": 40.0
|
||||
},
|
||||
"weighted avg": {
|
||||
"f1-score": 0.8032189316829489,
|
||||
"precision": 0.8033663455885677,
|
||||
"recall": 0.8042328042328042,
|
||||
"support": 189.0
|
||||
}
|
||||
},
|
||||
"confusion_matrix": {
|
||||
"labels": [
|
||||
"bearish",
|
||||
"neutral",
|
||||
"bullish"
|
||||
],
|
||||
"matrix": [
|
||||
[
|
||||
62,
|
||||
3,
|
||||
7
|
||||
],
|
||||
[
|
||||
3,
|
||||
28,
|
||||
9
|
||||
],
|
||||
[
|
||||
10,
|
||||
5,
|
||||
62
|
||||
]
|
||||
]
|
||||
},
|
||||
"macro_f1": 0.7934598400763816,
|
||||
"score_mae": 0.24237639972402897,
|
||||
"test_examples": 189
|
||||
},
|
||||
"temporal_holdout": {
|
||||
"accuracy": 0.746031746031746,
|
||||
"balanced_accuracy": 0.7388071895424838,
|
||||
"classification_report": {
|
||||
"accuracy": 0.746031746031746,
|
||||
"bearish": {
|
||||
"f1-score": 0.8070175438596491,
|
||||
"precision": 0.8518518518518519,
|
||||
"recall": 0.7666666666666667,
|
||||
"support": 90.0
|
||||
},
|
||||
"bullish": {
|
||||
"f1-score": 0.6833333333333333,
|
||||
"precision": 0.5942028985507246,
|
||||
"recall": 0.803921568627451,
|
||||
"support": 51.0
|
||||
},
|
||||
"macro avg": {
|
||||
"f1-score": 0.7343315184513006,
|
||||
"precision": 0.7469755150914571,
|
||||
"recall": 0.7388071895424838,
|
||||
"support": 189.0
|
||||
},
|
||||
"neutral": {
|
||||
"f1-score": 0.7126436781609196,
|
||||
"precision": 0.7948717948717948,
|
||||
"recall": 0.6458333333333334,
|
||||
"support": 48.0
|
||||
},
|
||||
"weighted avg": {
|
||||
"f1-score": 0.7496744735401722,
|
||||
"precision": 0.7678564055375648,
|
||||
"recall": 0.746031746031746,
|
||||
"support": 189.0
|
||||
}
|
||||
},
|
||||
"confusion_matrix": {
|
||||
"labels": [
|
||||
"bearish",
|
||||
"neutral",
|
||||
"bullish"
|
||||
],
|
||||
"matrix": [
|
||||
[
|
||||
69,
|
||||
5,
|
||||
16
|
||||
],
|
||||
[
|
||||
5,
|
||||
31,
|
||||
12
|
||||
],
|
||||
[
|
||||
7,
|
||||
3,
|
||||
41
|
||||
]
|
||||
]
|
||||
},
|
||||
"macro_f1": 0.7343315184513006,
|
||||
"score_mae": 0.27303151729050784,
|
||||
"test_examples": 189,
|
||||
"test_from": "2026-09-21T02:41:59+00:00",
|
||||
"train_through": "2026-09-21T02:41:59+00:00"
|
||||
}
|
||||
},
|
||||
"format_version": 3,
|
||||
"model_path": "models/candidates/xau_gold_20260922.joblib",
|
||||
"model_type": "domain_augmented_tfidf_logistic_regression",
|
||||
"score_definition": "P(XAUUSD bullish) - P(XAUUSD bearish)",
|
||||
"sklearn_version": "1.9.1",
|
||||
"target": "XAUUSD",
|
||||
"trained_at": "2026-09-23T00:35:57.482787+00:00",
|
||||
"training": {
|
||||
"database": "newsdata.db",
|
||||
"domain_features": [
|
||||
"explicit_xau_direction",
|
||||
"usd_direction",
|
||||
"treasury_yield_direction",
|
||||
"hawkish_or_dovish_policy"
|
||||
],
|
||||
"duplicates_collapsed": 14,
|
||||
"examples": 945,
|
||||
"extra_labels": null,
|
||||
"label_distribution": {
|
||||
"bearish": 359,
|
||||
"bullish": 384,
|
||||
"neutral": 202
|
||||
},
|
||||
"label_priority": [
|
||||
"xau_score",
|
||||
"json",
|
||||
"extra_labels"
|
||||
],
|
||||
"label_sources": {
|
||||
"json": 640,
|
||||
"xau_score": 319
|
||||
},
|
||||
"max_features": 50000,
|
||||
"min_df": 2,
|
||||
"neutral_threshold": 0.05,
|
||||
"published_range": [
|
||||
"2026-09-08T13:46:00+00:00",
|
||||
"2026-09-22T18:13:29+00:00"
|
||||
],
|
||||
"random_state": 42,
|
||||
"rows_without_target_label": 1702,
|
||||
"scope": "gold",
|
||||
"score_source": "llm",
|
||||
"skipped_rows": 0,
|
||||
"source_rows": 2661,
|
||||
"target": "XAUUSD",
|
||||
"target_labeled_rows": 959,
|
||||
"target_score_key": "XAU",
|
||||
"test_feed_rows": 0,
|
||||
"test_size": 0.2
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
{
|
||||
"base_model": "microsoft/deberta-v3-small",
|
||||
"trained_at": "2026-09-27T23:15:30.403442+00:00",
|
||||
"final": true,
|
||||
"max_length": 64,
|
||||
"relevant_min_abs_xau": 0.25,
|
||||
"irrelevant_max_abs_xau": 0.1,
|
||||
"relevance_threshold": 0.2,
|
||||
"best_epoch": 4,
|
||||
"hyperparameters": {
|
||||
"base": "microsoft/deberta-v3-small",
|
||||
"epochs": 4,
|
||||
"batch_size": 64,
|
||||
"lr": 5e-05,
|
||||
"warmup": 0.06,
|
||||
"weight_decay": 0.01,
|
||||
"val_from": "2026-09-18",
|
||||
"seed": 42,
|
||||
"final": true
|
||||
},
|
||||
"data": {
|
||||
"train": {
|
||||
"n": 15416,
|
||||
"relevant": 4746,
|
||||
"irrelevant": 8996,
|
||||
"grey": 1674,
|
||||
"bullish": 3080,
|
||||
"bearish": 1666,
|
||||
"sources": {
|
||||
"history": 13893,
|
||||
"newsdata": 1523
|
||||
},
|
||||
"years": {
|
||||
"2009": 10,
|
||||
"2010": 10,
|
||||
"2011": 9,
|
||||
"2012": 2,
|
||||
"2015": 11,
|
||||
"2016": 6,
|
||||
"2017": 32,
|
||||
"2018": 19,
|
||||
"2019": 949,
|
||||
"2020": 1144,
|
||||
"2021": 1197,
|
||||
"2022": 1467,
|
||||
"2023": 1619,
|
||||
"2024": 1938,
|
||||
"2025": 2922,
|
||||
"2026": 4081
|
||||
}
|
||||
},
|
||||
"dev": {
|
||||
"n": 811,
|
||||
"relevant": 251,
|
||||
"irrelevant": 476,
|
||||
"grey": 84,
|
||||
"bullish": 164,
|
||||
"bearish": 87,
|
||||
"sources": {
|
||||
"history": 732,
|
||||
"newsdata": 79
|
||||
},
|
||||
"years": {
|
||||
"2011": 1,
|
||||
"2015": 2,
|
||||
"2017": 1,
|
||||
"2019": 54,
|
||||
"2020": 67,
|
||||
"2021": 67,
|
||||
"2022": 69,
|
||||
"2023": 72,
|
||||
"2024": 109,
|
||||
"2025": 149,
|
||||
"2026": 220
|
||||
}
|
||||
},
|
||||
"val": {
|
||||
"n": 1018,
|
||||
"relevant": 378,
|
||||
"irrelevant": 501,
|
||||
"grey": 139,
|
||||
"bullish": 171,
|
||||
"bearish": 207,
|
||||
"sources": {
|
||||
"newsdata": 1018
|
||||
},
|
||||
"years": {
|
||||
"2026": 1018
|
||||
}
|
||||
}
|
||||
},
|
||||
"history": [
|
||||
{
|
||||
"epoch": 1,
|
||||
"train_loss": 0.9252183866832664,
|
||||
"seconds": 17
|
||||
},
|
||||
{
|
||||
"epoch": 2,
|
||||
"train_loss": 0.5151546665778123,
|
||||
"seconds": 34
|
||||
},
|
||||
{
|
||||
"epoch": 3,
|
||||
"train_loss": 0.3269671004340074,
|
||||
"seconds": 50
|
||||
},
|
||||
{
|
||||
"epoch": 4,
|
||||
"train_loss": 0.2127824436748985,
|
||||
"seconds": 67
|
||||
}
|
||||
],
|
||||
"evaluation": {
|
||||
"dev": {
|
||||
"relevance_threshold": 0.2,
|
||||
"relevance_precision": 0.7660256410256411,
|
||||
"relevance_recall": 0.952191235059761,
|
||||
"relevance_f1": 0.8490230905861458,
|
||||
"relevance_accuracy": 0.8830811554332875,
|
||||
"direction_accuracy_on_relevant": 0.8884462151394422,
|
||||
"three_class_vs_deepseek_0.05": 0.7891491985203453,
|
||||
"three_class_macro_f1": 0.7551648489586519,
|
||||
"n": 811
|
||||
},
|
||||
"val": {
|
||||
"relevance_threshold": 0.2,
|
||||
"relevance_precision": 0.7237113402061855,
|
||||
"relevance_recall": 0.9285714285714286,
|
||||
"relevance_f1": 0.8134414831981461,
|
||||
"relevance_accuracy": 0.8168373151308305,
|
||||
"direction_accuracy_on_relevant": 0.8941798941798942,
|
||||
"three_class_vs_deepseek_0.05": 0.712180746561886,
|
||||
"three_class_macro_f1": 0.7103112215543884,
|
||||
"n": 1018
|
||||
}
|
||||
},
|
||||
"evaluation_note": "final model trained on val too; evaluation is from the non-final run",
|
||||
"labels": "DeepSeek deepseek-v4-flash live prompt XAU score",
|
||||
"onnx_file": "model.onnx",
|
||||
"quantization": "int8 weight-only embedding table; fp32 compute",
|
||||
"pad_token": "[PAD]",
|
||||
"pad_token_id": 0,
|
||||
"exported_from": "models/xau_transformer/torch_final",
|
||||
"export_checks": {
|
||||
"rows": 500,
|
||||
"tokenizer_mismatches": 0,
|
||||
"p_relevant_max_abs_diff": 0.06070525050163267,
|
||||
"p_bullish_max_abs_diff": 0.07697186813354495,
|
||||
"decision_flips": 2,
|
||||
"latency_ms_per_headline": {
|
||||
"1": 18.84,
|
||||
"16": 9.55
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,184 @@
|
||||
{
|
||||
"base_model": "microsoft/deberta-v3-small",
|
||||
"trained_at": "2026-09-27T23:12:37.970737+00:00",
|
||||
"final": false,
|
||||
"max_length": 64,
|
||||
"relevant_min_abs_xau": 0.25,
|
||||
"irrelevant_max_abs_xau": 0.1,
|
||||
"relevance_threshold": 0.2,
|
||||
"best_epoch": 4,
|
||||
"hyperparameters": {
|
||||
"base": "microsoft/deberta-v3-small",
|
||||
"epochs": 4,
|
||||
"batch_size": 64,
|
||||
"lr": 5e-05,
|
||||
"warmup": 0.06,
|
||||
"weight_decay": 0.01,
|
||||
"val_from": "2026-09-18",
|
||||
"seed": 42,
|
||||
"final": false
|
||||
},
|
||||
"data": {
|
||||
"train": {
|
||||
"n": 15416,
|
||||
"relevant": 4746,
|
||||
"irrelevant": 8996,
|
||||
"grey": 1674,
|
||||
"bullish": 3080,
|
||||
"bearish": 1666,
|
||||
"sources": {
|
||||
"history": 13893,
|
||||
"newsdata": 1523
|
||||
},
|
||||
"years": {
|
||||
"2009": 10,
|
||||
"2010": 10,
|
||||
"2011": 9,
|
||||
"2012": 2,
|
||||
"2015": 11,
|
||||
"2016": 6,
|
||||
"2017": 32,
|
||||
"2018": 19,
|
||||
"2019": 949,
|
||||
"2020": 1144,
|
||||
"2021": 1197,
|
||||
"2022": 1467,
|
||||
"2023": 1619,
|
||||
"2024": 1938,
|
||||
"2025": 2922,
|
||||
"2026": 4081
|
||||
}
|
||||
},
|
||||
"dev": {
|
||||
"n": 811,
|
||||
"relevant": 251,
|
||||
"irrelevant": 476,
|
||||
"grey": 84,
|
||||
"bullish": 164,
|
||||
"bearish": 87,
|
||||
"sources": {
|
||||
"history": 732,
|
||||
"newsdata": 79
|
||||
},
|
||||
"years": {
|
||||
"2011": 1,
|
||||
"2015": 2,
|
||||
"2017": 1,
|
||||
"2019": 54,
|
||||
"2020": 67,
|
||||
"2021": 67,
|
||||
"2022": 69,
|
||||
"2023": 72,
|
||||
"2024": 109,
|
||||
"2025": 149,
|
||||
"2026": 220
|
||||
}
|
||||
},
|
||||
"val": {
|
||||
"n": 1018,
|
||||
"relevant": 378,
|
||||
"irrelevant": 501,
|
||||
"grey": 139,
|
||||
"bullish": 171,
|
||||
"bearish": 207,
|
||||
"sources": {
|
||||
"newsdata": 1018
|
||||
},
|
||||
"years": {
|
||||
"2026": 1018
|
||||
}
|
||||
}
|
||||
},
|
||||
"history": [
|
||||
{
|
||||
"epoch": 1,
|
||||
"train_loss": 0.9298006843951433,
|
||||
"seconds": 15,
|
||||
"dev": {
|
||||
"relevance_threshold": 0.45,
|
||||
"relevance_precision": 0.7483870967741936,
|
||||
"relevance_recall": 0.9243027888446215,
|
||||
"relevance_f1": 0.8270944741532976,
|
||||
"relevance_accuracy": 0.8665749656121046,
|
||||
"direction_accuracy_on_relevant": 0.7888446215139442,
|
||||
"three_class_vs_deepseek_0.05": 0.7287299630086314,
|
||||
"three_class_macro_f1": 0.6827620410967894,
|
||||
"n": 811
|
||||
}
|
||||
},
|
||||
{
|
||||
"epoch": 2,
|
||||
"train_loss": 0.5256601697617884,
|
||||
"seconds": 30,
|
||||
"dev": {
|
||||
"relevance_threshold": 0.2,
|
||||
"relevance_precision": 0.7347560975609756,
|
||||
"relevance_recall": 0.9601593625498008,
|
||||
"relevance_f1": 0.8324697754749568,
|
||||
"relevance_accuracy": 0.8665749656121046,
|
||||
"direction_accuracy_on_relevant": 0.8884462151394422,
|
||||
"three_class_vs_deepseek_0.05": 0.7780517879161529,
|
||||
"three_class_macro_f1": 0.7441922940968541,
|
||||
"n": 811
|
||||
}
|
||||
},
|
||||
{
|
||||
"epoch": 3,
|
||||
"train_loss": 0.3365446340097438,
|
||||
"seconds": 45,
|
||||
"dev": {
|
||||
"relevance_threshold": 0.2,
|
||||
"relevance_precision": 0.7807308970099668,
|
||||
"relevance_recall": 0.9362549800796812,
|
||||
"relevance_f1": 0.8514492753623188,
|
||||
"relevance_accuracy": 0.8872077028885832,
|
||||
"direction_accuracy_on_relevant": 0.8924302788844621,
|
||||
"three_class_vs_deepseek_0.05": 0.7792848335388409,
|
||||
"three_class_macro_f1": 0.7430898597339434,
|
||||
"n": 811
|
||||
}
|
||||
},
|
||||
{
|
||||
"epoch": 4,
|
||||
"train_loss": 0.22059692797353989,
|
||||
"seconds": 60,
|
||||
"dev": {
|
||||
"relevance_threshold": 0.2,
|
||||
"relevance_precision": 0.7660256410256411,
|
||||
"relevance_recall": 0.952191235059761,
|
||||
"relevance_f1": 0.8490230905861458,
|
||||
"relevance_accuracy": 0.8830811554332875,
|
||||
"direction_accuracy_on_relevant": 0.8884462151394422,
|
||||
"three_class_vs_deepseek_0.05": 0.7891491985203453,
|
||||
"three_class_macro_f1": 0.7551648489586519,
|
||||
"n": 811
|
||||
}
|
||||
}
|
||||
],
|
||||
"evaluation": {
|
||||
"dev": {
|
||||
"relevance_threshold": 0.2,
|
||||
"relevance_precision": 0.7660256410256411,
|
||||
"relevance_recall": 0.952191235059761,
|
||||
"relevance_f1": 0.8490230905861458,
|
||||
"relevance_accuracy": 0.8830811554332875,
|
||||
"direction_accuracy_on_relevant": 0.8884462151394422,
|
||||
"three_class_vs_deepseek_0.05": 0.7891491985203453,
|
||||
"three_class_macro_f1": 0.7551648489586519,
|
||||
"n": 811
|
||||
},
|
||||
"val": {
|
||||
"relevance_threshold": 0.2,
|
||||
"relevance_precision": 0.7237113402061855,
|
||||
"relevance_recall": 0.9285714285714286,
|
||||
"relevance_f1": 0.8134414831981461,
|
||||
"relevance_accuracy": 0.8168373151308305,
|
||||
"direction_accuracy_on_relevant": 0.8941798941798942,
|
||||
"three_class_vs_deepseek_0.05": 0.712180746561886,
|
||||
"three_class_macro_f1": 0.7103112215543884,
|
||||
"n": 1018
|
||||
}
|
||||
},
|
||||
"evaluation_note": "val never used for selection",
|
||||
"labels": "DeepSeek deepseek-v4-flash live prompt XAU score"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"add_prefix_space": true,
|
||||
"backend": "tokenizers",
|
||||
"bos_token": "[CLS]",
|
||||
"cls_token": "[CLS]",
|
||||
"do_lower_case": false,
|
||||
"eos_token": "[SEP]",
|
||||
"extra_special_tokens": [
|
||||
"[PAD]",
|
||||
"[CLS]",
|
||||
"[SEP]"
|
||||
],
|
||||
"is_local": false,
|
||||
"local_files_only": false,
|
||||
"mask_token": "[MASK]",
|
||||
"model_max_length": 1000000000000000019884624838656,
|
||||
"pad_token": "[PAD]",
|
||||
"sep_token": "[SEP]",
|
||||
"split_by_punct": false,
|
||||
"tokenizer_class": "DebertaV2Tokenizer",
|
||||
"unk_id": 3,
|
||||
"unk_token": "[UNK]",
|
||||
"vocab_type": "spm"
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
{
|
||||
"base_model": "microsoft/deberta-v3-small",
|
||||
"trained_at": "2026-09-27T23:15:30.403442+00:00",
|
||||
"final": true,
|
||||
"max_length": 64,
|
||||
"relevant_min_abs_xau": 0.25,
|
||||
"irrelevant_max_abs_xau": 0.1,
|
||||
"relevance_threshold": 0.2,
|
||||
"best_epoch": 4,
|
||||
"hyperparameters": {
|
||||
"base": "microsoft/deberta-v3-small",
|
||||
"epochs": 4,
|
||||
"batch_size": 64,
|
||||
"lr": 5e-05,
|
||||
"warmup": 0.06,
|
||||
"weight_decay": 0.01,
|
||||
"val_from": "2026-09-18",
|
||||
"seed": 42,
|
||||
"final": true
|
||||
},
|
||||
"data": {
|
||||
"train": {
|
||||
"n": 15416,
|
||||
"relevant": 4746,
|
||||
"irrelevant": 8996,
|
||||
"grey": 1674,
|
||||
"bullish": 3080,
|
||||
"bearish": 1666,
|
||||
"sources": {
|
||||
"history": 13893,
|
||||
"newsdata": 1523
|
||||
},
|
||||
"years": {
|
||||
"2009": 10,
|
||||
"2010": 10,
|
||||
"2011": 9,
|
||||
"2012": 2,
|
||||
"2015": 11,
|
||||
"2016": 6,
|
||||
"2017": 32,
|
||||
"2018": 19,
|
||||
"2019": 949,
|
||||
"2020": 1144,
|
||||
"2021": 1197,
|
||||
"2022": 1467,
|
||||
"2023": 1619,
|
||||
"2024": 1938,
|
||||
"2025": 2922,
|
||||
"2026": 4081
|
||||
}
|
||||
},
|
||||
"dev": {
|
||||
"n": 811,
|
||||
"relevant": 251,
|
||||
"irrelevant": 476,
|
||||
"grey": 84,
|
||||
"bullish": 164,
|
||||
"bearish": 87,
|
||||
"sources": {
|
||||
"history": 732,
|
||||
"newsdata": 79
|
||||
},
|
||||
"years": {
|
||||
"2011": 1,
|
||||
"2015": 2,
|
||||
"2017": 1,
|
||||
"2019": 54,
|
||||
"2020": 67,
|
||||
"2021": 67,
|
||||
"2022": 69,
|
||||
"2023": 72,
|
||||
"2024": 109,
|
||||
"2025": 149,
|
||||
"2026": 220
|
||||
}
|
||||
},
|
||||
"val": {
|
||||
"n": 1018,
|
||||
"relevant": 378,
|
||||
"irrelevant": 501,
|
||||
"grey": 139,
|
||||
"bullish": 171,
|
||||
"bearish": 207,
|
||||
"sources": {
|
||||
"newsdata": 1018
|
||||
},
|
||||
"years": {
|
||||
"2026": 1018
|
||||
}
|
||||
}
|
||||
},
|
||||
"history": [
|
||||
{
|
||||
"epoch": 1,
|
||||
"train_loss": 0.9252183866832664,
|
||||
"seconds": 17
|
||||
},
|
||||
{
|
||||
"epoch": 2,
|
||||
"train_loss": 0.5151546665778123,
|
||||
"seconds": 34
|
||||
},
|
||||
{
|
||||
"epoch": 3,
|
||||
"train_loss": 0.3269671004340074,
|
||||
"seconds": 50
|
||||
},
|
||||
{
|
||||
"epoch": 4,
|
||||
"train_loss": 0.2127824436748985,
|
||||
"seconds": 67
|
||||
}
|
||||
],
|
||||
"evaluation": {
|
||||
"dev": {
|
||||
"relevance_threshold": 0.2,
|
||||
"relevance_precision": 0.7660256410256411,
|
||||
"relevance_recall": 0.952191235059761,
|
||||
"relevance_f1": 0.8490230905861458,
|
||||
"relevance_accuracy": 0.8830811554332875,
|
||||
"direction_accuracy_on_relevant": 0.8884462151394422,
|
||||
"three_class_vs_deepseek_0.05": 0.7891491985203453,
|
||||
"three_class_macro_f1": 0.7551648489586519,
|
||||
"n": 811
|
||||
},
|
||||
"val": {
|
||||
"relevance_threshold": 0.2,
|
||||
"relevance_precision": 0.7237113402061855,
|
||||
"relevance_recall": 0.9285714285714286,
|
||||
"relevance_f1": 0.8134414831981461,
|
||||
"relevance_accuracy": 0.8168373151308305,
|
||||
"direction_accuracy_on_relevant": 0.8941798941798942,
|
||||
"three_class_vs_deepseek_0.05": 0.712180746561886,
|
||||
"three_class_macro_f1": 0.7103112215543884,
|
||||
"n": 1018
|
||||
}
|
||||
},
|
||||
"evaluation_note": "final model trained on val too; evaluation is from the non-final run",
|
||||
"labels": "DeepSeek deepseek-v4-flash live prompt XAU score"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"add_prefix_space": true,
|
||||
"backend": "tokenizers",
|
||||
"bos_token": "[CLS]",
|
||||
"cls_token": "[CLS]",
|
||||
"do_lower_case": false,
|
||||
"eos_token": "[SEP]",
|
||||
"extra_special_tokens": [
|
||||
"[PAD]",
|
||||
"[CLS]",
|
||||
"[SEP]"
|
||||
],
|
||||
"is_local": false,
|
||||
"local_files_only": false,
|
||||
"mask_token": "[MASK]",
|
||||
"model_max_length": 1000000000000000019884624838656,
|
||||
"pad_token": "[PAD]",
|
||||
"sep_token": "[SEP]",
|
||||
"split_by_punct": false,
|
||||
"tokenizer_class": "DebertaV2Tokenizer",
|
||||
"unk_id": 3,
|
||||
"unk_token": "[UNK]",
|
||||
"vocab_type": "spm"
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,186 @@
|
||||
{
|
||||
"evaluation": {
|
||||
"stratified_holdout": {
|
||||
"accuracy": 0.6927480916030534,
|
||||
"balanced_accuracy": 0.689958210237739,
|
||||
"classification_report": {
|
||||
"accuracy": 0.6927480916030534,
|
||||
"bearish": {
|
||||
"f1-score": 0.752,
|
||||
"precision": 0.7305699481865285,
|
||||
"recall": 0.7747252747252747,
|
||||
"support": 182.0
|
||||
},
|
||||
"bullish": {
|
||||
"f1-score": 0.6230529595015576,
|
||||
"precision": 0.6369426751592356,
|
||||
"recall": 0.6097560975609756,
|
||||
"support": 164.0
|
||||
},
|
||||
"macro avg": {
|
||||
"f1-score": 0.6894115925611253,
|
||||
"precision": 0.6895540162110402,
|
||||
"recall": 0.689958210237739,
|
||||
"support": 524.0
|
||||
},
|
||||
"neutral": {
|
||||
"f1-score": 0.6931818181818182,
|
||||
"precision": 0.7011494252873564,
|
||||
"recall": 0.6853932584269663,
|
||||
"support": 178.0
|
||||
},
|
||||
"weighted avg": {
|
||||
"f1-score": 0.6916623072416396,
|
||||
"precision": 0.6912727614450616,
|
||||
"recall": 0.6927480916030534,
|
||||
"support": 524.0
|
||||
}
|
||||
},
|
||||
"confusion_matrix": {
|
||||
"labels": [
|
||||
"bearish",
|
||||
"neutral",
|
||||
"bullish"
|
||||
],
|
||||
"matrix": [
|
||||
[
|
||||
141,
|
||||
18,
|
||||
23
|
||||
],
|
||||
[
|
||||
22,
|
||||
122,
|
||||
34
|
||||
],
|
||||
[
|
||||
30,
|
||||
34,
|
||||
100
|
||||
]
|
||||
]
|
||||
},
|
||||
"macro_f1": 0.6894115925611253,
|
||||
"score_mae": 0.2644602373963052,
|
||||
"test_examples": 524
|
||||
},
|
||||
"temporal_holdout": {
|
||||
"accuracy": 0.666030534351145,
|
||||
"balanced_accuracy": 0.6572494770227411,
|
||||
"classification_report": {
|
||||
"accuracy": 0.666030534351145,
|
||||
"bearish": {
|
||||
"f1-score": 0.7040498442367601,
|
||||
"precision": 0.773972602739726,
|
||||
"recall": 0.6457142857142857,
|
||||
"support": 175.0
|
||||
},
|
||||
"bullish": {
|
||||
"f1-score": 0.5647840531561462,
|
||||
"precision": 0.5483870967741935,
|
||||
"recall": 0.5821917808219178,
|
||||
"support": 146.0
|
||||
},
|
||||
"macro avg": {
|
||||
"f1-score": 0.6592513617287779,
|
||||
"precision": 0.6664965814523229,
|
||||
"recall": 0.6572494770227411,
|
||||
"support": 524.0
|
||||
},
|
||||
"neutral": {
|
||||
"f1-score": 0.7089201877934272,
|
||||
"precision": 0.6771300448430493,
|
||||
"recall": 0.7438423645320197,
|
||||
"support": 203.0
|
||||
},
|
||||
"weighted avg": {
|
||||
"f1-score": 0.6671335737104888,
|
||||
"precision": 0.6736013754038612,
|
||||
"recall": 0.666030534351145,
|
||||
"support": 524.0
|
||||
}
|
||||
},
|
||||
"confusion_matrix": {
|
||||
"labels": [
|
||||
"bearish",
|
||||
"neutral",
|
||||
"bullish"
|
||||
],
|
||||
"matrix": [
|
||||
[
|
||||
113,
|
||||
31,
|
||||
31
|
||||
],
|
||||
[
|
||||
13,
|
||||
151,
|
||||
39
|
||||
],
|
||||
[
|
||||
20,
|
||||
41,
|
||||
85
|
||||
]
|
||||
]
|
||||
},
|
||||
"macro_f1": 0.6592513617287779,
|
||||
"score_mae": 0.24771861749332214,
|
||||
"test_examples": 524,
|
||||
"test_from": "2026-09-21T07:34:39+00:00",
|
||||
"train_through": "2026-09-21T07:33:06+00:00"
|
||||
}
|
||||
},
|
||||
"format_version": 3,
|
||||
"model_path": "models/xauusd_sentiment.joblib",
|
||||
"model_type": "domain_augmented_tfidf_logistic_regression",
|
||||
"score_definition": "P(XAUUSD bullish) - P(XAUUSD bearish)",
|
||||
"sklearn_version": "1.9.1",
|
||||
"target": "XAUUSD",
|
||||
"trained_at": "2026-09-23T00:40:10.241293+00:00",
|
||||
"training": {
|
||||
"database": "newsdata.db",
|
||||
"domain_features": [
|
||||
"explicit_xau_direction",
|
||||
"usd_direction",
|
||||
"treasury_yield_direction",
|
||||
"hawkish_or_dovish_policy"
|
||||
],
|
||||
"duplicates_collapsed": 17,
|
||||
"examples": 2620,
|
||||
"extra_labels": "eval/results/backfill_labels.jsonl",
|
||||
"label_distribution": {
|
||||
"bearish": 909,
|
||||
"bullish": 818,
|
||||
"neutral": 893
|
||||
},
|
||||
"label_priority": [
|
||||
"xau_score",
|
||||
"json",
|
||||
"extra_labels"
|
||||
],
|
||||
"label_sources": {
|
||||
"extra_labels": 933,
|
||||
"json": 640,
|
||||
"xau_score": 1064
|
||||
},
|
||||
"max_features": 50000,
|
||||
"min_df": 2,
|
||||
"neutral_threshold": 0.05,
|
||||
"published_range": [
|
||||
"2026-09-08T13:35:37+00:00",
|
||||
"2026-09-22T18:13:29+00:00"
|
||||
],
|
||||
"random_state": 42,
|
||||
"rows_without_target_label": 24,
|
||||
"scope": "all",
|
||||
"score_source": "llm",
|
||||
"skipped_rows": 0,
|
||||
"source_rows": 2661,
|
||||
"target": "XAUUSD",
|
||||
"target_labeled_rows": 2637,
|
||||
"target_score_key": "XAU",
|
||||
"test_feed_rows": 0,
|
||||
"test_size": 0.2
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user