Harwinder Sekhon
1d72e4006a
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
185 lines
4.8 KiB
JSON
185 lines
4.8 KiB
JSON
{
|
|
"base_model": "microsoft/deberta-v3-small",
|
|
"trained_at": "2026-09-27T23:12:37.970737+00:00",
|
|
"final": false,
|
|
"max_length": 64,
|
|
"relevant_min_abs_xau": 0.25,
|
|
"irrelevant_max_abs_xau": 0.1,
|
|
"relevance_threshold": 0.2,
|
|
"best_epoch": 4,
|
|
"hyperparameters": {
|
|
"base": "microsoft/deberta-v3-small",
|
|
"epochs": 4,
|
|
"batch_size": 64,
|
|
"lr": 5e-05,
|
|
"warmup": 0.06,
|
|
"weight_decay": 0.01,
|
|
"val_from": "2026-09-18",
|
|
"seed": 42,
|
|
"final": false
|
|
},
|
|
"data": {
|
|
"train": {
|
|
"n": 15416,
|
|
"relevant": 4746,
|
|
"irrelevant": 8996,
|
|
"grey": 1674,
|
|
"bullish": 3080,
|
|
"bearish": 1666,
|
|
"sources": {
|
|
"history": 13893,
|
|
"newsdata": 1523
|
|
},
|
|
"years": {
|
|
"2009": 10,
|
|
"2010": 10,
|
|
"2011": 9,
|
|
"2012": 2,
|
|
"2015": 11,
|
|
"2016": 6,
|
|
"2017": 32,
|
|
"2018": 19,
|
|
"2019": 949,
|
|
"2020": 1144,
|
|
"2021": 1197,
|
|
"2022": 1467,
|
|
"2023": 1619,
|
|
"2024": 1938,
|
|
"2025": 2922,
|
|
"2026": 4081
|
|
}
|
|
},
|
|
"dev": {
|
|
"n": 811,
|
|
"relevant": 251,
|
|
"irrelevant": 476,
|
|
"grey": 84,
|
|
"bullish": 164,
|
|
"bearish": 87,
|
|
"sources": {
|
|
"history": 732,
|
|
"newsdata": 79
|
|
},
|
|
"years": {
|
|
"2011": 1,
|
|
"2015": 2,
|
|
"2017": 1,
|
|
"2019": 54,
|
|
"2020": 67,
|
|
"2021": 67,
|
|
"2022": 69,
|
|
"2023": 72,
|
|
"2024": 109,
|
|
"2025": 149,
|
|
"2026": 220
|
|
}
|
|
},
|
|
"val": {
|
|
"n": 1018,
|
|
"relevant": 378,
|
|
"irrelevant": 501,
|
|
"grey": 139,
|
|
"bullish": 171,
|
|
"bearish": 207,
|
|
"sources": {
|
|
"newsdata": 1018
|
|
},
|
|
"years": {
|
|
"2026": 1018
|
|
}
|
|
}
|
|
},
|
|
"history": [
|
|
{
|
|
"epoch": 1,
|
|
"train_loss": 0.9298006843951433,
|
|
"seconds": 15,
|
|
"dev": {
|
|
"relevance_threshold": 0.45,
|
|
"relevance_precision": 0.7483870967741936,
|
|
"relevance_recall": 0.9243027888446215,
|
|
"relevance_f1": 0.8270944741532976,
|
|
"relevance_accuracy": 0.8665749656121046,
|
|
"direction_accuracy_on_relevant": 0.7888446215139442,
|
|
"three_class_vs_deepseek_0.05": 0.7287299630086314,
|
|
"three_class_macro_f1": 0.6827620410967894,
|
|
"n": 811
|
|
}
|
|
},
|
|
{
|
|
"epoch": 2,
|
|
"train_loss": 0.5256601697617884,
|
|
"seconds": 30,
|
|
"dev": {
|
|
"relevance_threshold": 0.2,
|
|
"relevance_precision": 0.7347560975609756,
|
|
"relevance_recall": 0.9601593625498008,
|
|
"relevance_f1": 0.8324697754749568,
|
|
"relevance_accuracy": 0.8665749656121046,
|
|
"direction_accuracy_on_relevant": 0.8884462151394422,
|
|
"three_class_vs_deepseek_0.05": 0.7780517879161529,
|
|
"three_class_macro_f1": 0.7441922940968541,
|
|
"n": 811
|
|
}
|
|
},
|
|
{
|
|
"epoch": 3,
|
|
"train_loss": 0.3365446340097438,
|
|
"seconds": 45,
|
|
"dev": {
|
|
"relevance_threshold": 0.2,
|
|
"relevance_precision": 0.7807308970099668,
|
|
"relevance_recall": 0.9362549800796812,
|
|
"relevance_f1": 0.8514492753623188,
|
|
"relevance_accuracy": 0.8872077028885832,
|
|
"direction_accuracy_on_relevant": 0.8924302788844621,
|
|
"three_class_vs_deepseek_0.05": 0.7792848335388409,
|
|
"three_class_macro_f1": 0.7430898597339434,
|
|
"n": 811
|
|
}
|
|
},
|
|
{
|
|
"epoch": 4,
|
|
"train_loss": 0.22059692797353989,
|
|
"seconds": 60,
|
|
"dev": {
|
|
"relevance_threshold": 0.2,
|
|
"relevance_precision": 0.7660256410256411,
|
|
"relevance_recall": 0.952191235059761,
|
|
"relevance_f1": 0.8490230905861458,
|
|
"relevance_accuracy": 0.8830811554332875,
|
|
"direction_accuracy_on_relevant": 0.8884462151394422,
|
|
"three_class_vs_deepseek_0.05": 0.7891491985203453,
|
|
"three_class_macro_f1": 0.7551648489586519,
|
|
"n": 811
|
|
}
|
|
}
|
|
],
|
|
"evaluation": {
|
|
"dev": {
|
|
"relevance_threshold": 0.2,
|
|
"relevance_precision": 0.7660256410256411,
|
|
"relevance_recall": 0.952191235059761,
|
|
"relevance_f1": 0.8490230905861458,
|
|
"relevance_accuracy": 0.8830811554332875,
|
|
"direction_accuracy_on_relevant": 0.8884462151394422,
|
|
"three_class_vs_deepseek_0.05": 0.7891491985203453,
|
|
"three_class_macro_f1": 0.7551648489586519,
|
|
"n": 811
|
|
},
|
|
"val": {
|
|
"relevance_threshold": 0.2,
|
|
"relevance_precision": 0.7237113402061855,
|
|
"relevance_recall": 0.9285714285714286,
|
|
"relevance_f1": 0.8134414831981461,
|
|
"relevance_accuracy": 0.8168373151308305,
|
|
"direction_accuracy_on_relevant": 0.8941798941798942,
|
|
"three_class_vs_deepseek_0.05": 0.712180746561886,
|
|
"three_class_macro_f1": 0.7103112215543884,
|
|
"n": 1018
|
|
}
|
|
},
|
|
"evaluation_note": "val never used for selection",
|
|
"labels": "DeepSeek deepseek-v4-flash live prompt XAU score"
|
|
}
|