Harwinder Sekhon
1d72e4006a
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
187 lines
4.6 KiB
JSON
187 lines
4.6 KiB
JSON
{
|
|
"evaluation": {
|
|
"stratified_holdout": {
|
|
"accuracy": 0.6927480916030534,
|
|
"balanced_accuracy": 0.689958210237739,
|
|
"classification_report": {
|
|
"accuracy": 0.6927480916030534,
|
|
"bearish": {
|
|
"f1-score": 0.752,
|
|
"precision": 0.7305699481865285,
|
|
"recall": 0.7747252747252747,
|
|
"support": 182.0
|
|
},
|
|
"bullish": {
|
|
"f1-score": 0.6230529595015576,
|
|
"precision": 0.6369426751592356,
|
|
"recall": 0.6097560975609756,
|
|
"support": 164.0
|
|
},
|
|
"macro avg": {
|
|
"f1-score": 0.6894115925611253,
|
|
"precision": 0.6895540162110402,
|
|
"recall": 0.689958210237739,
|
|
"support": 524.0
|
|
},
|
|
"neutral": {
|
|
"f1-score": 0.6931818181818182,
|
|
"precision": 0.7011494252873564,
|
|
"recall": 0.6853932584269663,
|
|
"support": 178.0
|
|
},
|
|
"weighted avg": {
|
|
"f1-score": 0.6916623072416396,
|
|
"precision": 0.6912727614450616,
|
|
"recall": 0.6927480916030534,
|
|
"support": 524.0
|
|
}
|
|
},
|
|
"confusion_matrix": {
|
|
"labels": [
|
|
"bearish",
|
|
"neutral",
|
|
"bullish"
|
|
],
|
|
"matrix": [
|
|
[
|
|
141,
|
|
18,
|
|
23
|
|
],
|
|
[
|
|
22,
|
|
122,
|
|
34
|
|
],
|
|
[
|
|
30,
|
|
34,
|
|
100
|
|
]
|
|
]
|
|
},
|
|
"macro_f1": 0.6894115925611253,
|
|
"score_mae": 0.2644602373963052,
|
|
"test_examples": 524
|
|
},
|
|
"temporal_holdout": {
|
|
"accuracy": 0.666030534351145,
|
|
"balanced_accuracy": 0.6572494770227411,
|
|
"classification_report": {
|
|
"accuracy": 0.666030534351145,
|
|
"bearish": {
|
|
"f1-score": 0.7040498442367601,
|
|
"precision": 0.773972602739726,
|
|
"recall": 0.6457142857142857,
|
|
"support": 175.0
|
|
},
|
|
"bullish": {
|
|
"f1-score": 0.5647840531561462,
|
|
"precision": 0.5483870967741935,
|
|
"recall": 0.5821917808219178,
|
|
"support": 146.0
|
|
},
|
|
"macro avg": {
|
|
"f1-score": 0.6592513617287779,
|
|
"precision": 0.6664965814523229,
|
|
"recall": 0.6572494770227411,
|
|
"support": 524.0
|
|
},
|
|
"neutral": {
|
|
"f1-score": 0.7089201877934272,
|
|
"precision": 0.6771300448430493,
|
|
"recall": 0.7438423645320197,
|
|
"support": 203.0
|
|
},
|
|
"weighted avg": {
|
|
"f1-score": 0.6671335737104888,
|
|
"precision": 0.6736013754038612,
|
|
"recall": 0.666030534351145,
|
|
"support": 524.0
|
|
}
|
|
},
|
|
"confusion_matrix": {
|
|
"labels": [
|
|
"bearish",
|
|
"neutral",
|
|
"bullish"
|
|
],
|
|
"matrix": [
|
|
[
|
|
113,
|
|
31,
|
|
31
|
|
],
|
|
[
|
|
13,
|
|
151,
|
|
39
|
|
],
|
|
[
|
|
20,
|
|
41,
|
|
85
|
|
]
|
|
]
|
|
},
|
|
"macro_f1": 0.6592513617287779,
|
|
"score_mae": 0.24771861749332214,
|
|
"test_examples": 524,
|
|
"test_from": "2026-09-21T07:34:39+00:00",
|
|
"train_through": "2026-09-21T07:33:06+00:00"
|
|
}
|
|
},
|
|
"format_version": 3,
|
|
"model_path": "models/xauusd_sentiment.joblib",
|
|
"model_type": "domain_augmented_tfidf_logistic_regression",
|
|
"score_definition": "P(XAUUSD bullish) - P(XAUUSD bearish)",
|
|
"sklearn_version": "1.9.1",
|
|
"target": "XAUUSD",
|
|
"trained_at": "2026-09-23T00:40:10.241293+00:00",
|
|
"training": {
|
|
"database": "newsdata.db",
|
|
"domain_features": [
|
|
"explicit_xau_direction",
|
|
"usd_direction",
|
|
"treasury_yield_direction",
|
|
"hawkish_or_dovish_policy"
|
|
],
|
|
"duplicates_collapsed": 17,
|
|
"examples": 2620,
|
|
"extra_labels": "eval/results/backfill_labels.jsonl",
|
|
"label_distribution": {
|
|
"bearish": 909,
|
|
"bullish": 818,
|
|
"neutral": 893
|
|
},
|
|
"label_priority": [
|
|
"xau_score",
|
|
"json",
|
|
"extra_labels"
|
|
],
|
|
"label_sources": {
|
|
"extra_labels": 933,
|
|
"json": 640,
|
|
"xau_score": 1064
|
|
},
|
|
"max_features": 50000,
|
|
"min_df": 2,
|
|
"neutral_threshold": 0.05,
|
|
"published_range": [
|
|
"2026-09-08T13:35:37+00:00",
|
|
"2026-09-22T18:13:29+00:00"
|
|
],
|
|
"random_state": 42,
|
|
"rows_without_target_label": 24,
|
|
"scope": "all",
|
|
"score_source": "llm",
|
|
"skipped_rows": 0,
|
|
"source_rows": 2661,
|
|
"target": "XAUUSD",
|
|
"target_labeled_rows": 2637,
|
|
"target_score_key": "XAU",
|
|
"test_feed_rows": 0,
|
|
"test_size": 0.2
|
|
}
|
|
}
|