Initial import: XAUUSD news sentiment models, evals, and data tooling
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked.
This commit is contained in:
Harwinder Sekhon
@@ -0,0 +1,186 @@
|
||||
{
|
||||
"evaluation": {
|
||||
"stratified_holdout": {
|
||||
"accuracy": 0.6927480916030534,
|
||||
"balanced_accuracy": 0.689958210237739,
|
||||
"classification_report": {
|
||||
"accuracy": 0.6927480916030534,
|
||||
"bearish": {
|
||||
"f1-score": 0.752,
|
||||
"precision": 0.7305699481865285,
|
||||
"recall": 0.7747252747252747,
|
||||
"support": 182.0
|
||||
},
|
||||
"bullish": {
|
||||
"f1-score": 0.6230529595015576,
|
||||
"precision": 0.6369426751592356,
|
||||
"recall": 0.6097560975609756,
|
||||
"support": 164.0
|
||||
},
|
||||
"macro avg": {
|
||||
"f1-score": 0.6894115925611253,
|
||||
"precision": 0.6895540162110402,
|
||||
"recall": 0.689958210237739,
|
||||
"support": 524.0
|
||||
},
|
||||
"neutral": {
|
||||
"f1-score": 0.6931818181818182,
|
||||
"precision": 0.7011494252873564,
|
||||
"recall": 0.6853932584269663,
|
||||
"support": 178.0
|
||||
},
|
||||
"weighted avg": {
|
||||
"f1-score": 0.6916623072416396,
|
||||
"precision": 0.6912727614450616,
|
||||
"recall": 0.6927480916030534,
|
||||
"support": 524.0
|
||||
}
|
||||
},
|
||||
"confusion_matrix": {
|
||||
"labels": [
|
||||
"bearish",
|
||||
"neutral",
|
||||
"bullish"
|
||||
],
|
||||
"matrix": [
|
||||
[
|
||||
141,
|
||||
18,
|
||||
23
|
||||
],
|
||||
[
|
||||
22,
|
||||
122,
|
||||
34
|
||||
],
|
||||
[
|
||||
30,
|
||||
34,
|
||||
100
|
||||
]
|
||||
]
|
||||
},
|
||||
"macro_f1": 0.6894115925611253,
|
||||
"score_mae": 0.2644602373963052,
|
||||
"test_examples": 524
|
||||
},
|
||||
"temporal_holdout": {
|
||||
"accuracy": 0.666030534351145,
|
||||
"balanced_accuracy": 0.6572494770227411,
|
||||
"classification_report": {
|
||||
"accuracy": 0.666030534351145,
|
||||
"bearish": {
|
||||
"f1-score": 0.7040498442367601,
|
||||
"precision": 0.773972602739726,
|
||||
"recall": 0.6457142857142857,
|
||||
"support": 175.0
|
||||
},
|
||||
"bullish": {
|
||||
"f1-score": 0.5647840531561462,
|
||||
"precision": 0.5483870967741935,
|
||||
"recall": 0.5821917808219178,
|
||||
"support": 146.0
|
||||
},
|
||||
"macro avg": {
|
||||
"f1-score": 0.6592513617287779,
|
||||
"precision": 0.6664965814523229,
|
||||
"recall": 0.6572494770227411,
|
||||
"support": 524.0
|
||||
},
|
||||
"neutral": {
|
||||
"f1-score": 0.7089201877934272,
|
||||
"precision": 0.6771300448430493,
|
||||
"recall": 0.7438423645320197,
|
||||
"support": 203.0
|
||||
},
|
||||
"weighted avg": {
|
||||
"f1-score": 0.6671335737104888,
|
||||
"precision": 0.6736013754038612,
|
||||
"recall": 0.666030534351145,
|
||||
"support": 524.0
|
||||
}
|
||||
},
|
||||
"confusion_matrix": {
|
||||
"labels": [
|
||||
"bearish",
|
||||
"neutral",
|
||||
"bullish"
|
||||
],
|
||||
"matrix": [
|
||||
[
|
||||
113,
|
||||
31,
|
||||
31
|
||||
],
|
||||
[
|
||||
13,
|
||||
151,
|
||||
39
|
||||
],
|
||||
[
|
||||
20,
|
||||
41,
|
||||
85
|
||||
]
|
||||
]
|
||||
},
|
||||
"macro_f1": 0.6592513617287779,
|
||||
"score_mae": 0.24771861749332214,
|
||||
"test_examples": 524,
|
||||
"test_from": "2026-09-21T07:34:39+00:00",
|
||||
"train_through": "2026-09-21T07:33:06+00:00"
|
||||
}
|
||||
},
|
||||
"format_version": 3,
|
||||
"model_path": "models/xauusd_sentiment.joblib",
|
||||
"model_type": "domain_augmented_tfidf_logistic_regression",
|
||||
"score_definition": "P(XAUUSD bullish) - P(XAUUSD bearish)",
|
||||
"sklearn_version": "1.9.1",
|
||||
"target": "XAUUSD",
|
||||
"trained_at": "2026-09-23T00:40:10.241293+00:00",
|
||||
"training": {
|
||||
"database": "newsdata.db",
|
||||
"domain_features": [
|
||||
"explicit_xau_direction",
|
||||
"usd_direction",
|
||||
"treasury_yield_direction",
|
||||
"hawkish_or_dovish_policy"
|
||||
],
|
||||
"duplicates_collapsed": 17,
|
||||
"examples": 2620,
|
||||
"extra_labels": "eval/results/backfill_labels.jsonl",
|
||||
"label_distribution": {
|
||||
"bearish": 909,
|
||||
"bullish": 818,
|
||||
"neutral": 893
|
||||
},
|
||||
"label_priority": [
|
||||
"xau_score",
|
||||
"json",
|
||||
"extra_labels"
|
||||
],
|
||||
"label_sources": {
|
||||
"extra_labels": 933,
|
||||
"json": 640,
|
||||
"xau_score": 1064
|
||||
},
|
||||
"max_features": 50000,
|
||||
"min_df": 2,
|
||||
"neutral_threshold": 0.05,
|
||||
"published_range": [
|
||||
"2026-09-08T13:35:37+00:00",
|
||||
"2026-09-22T18:13:29+00:00"
|
||||
],
|
||||
"random_state": 42,
|
||||
"rows_without_target_label": 24,
|
||||
"scope": "all",
|
||||
"score_source": "llm",
|
||||
"skipped_rows": 0,
|
||||
"source_rows": 2661,
|
||||
"target": "XAUUSD",
|
||||
"target_labeled_rows": 2637,
|
||||
"target_score_key": "XAU",
|
||||
"test_feed_rows": 0,
|
||||
"test_size": 0.2
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user