Harwinder Sekhon
1d72e4006a
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
186 lines
4.5 KiB
JSON
186 lines
4.5 KiB
JSON
{
|
|
"evaluation": {
|
|
"stratified_holdout": {
|
|
"accuracy": 0.8042328042328042,
|
|
"balanced_accuracy": 0.7887686387686387,
|
|
"classification_report": {
|
|
"accuracy": 0.8042328042328042,
|
|
"bearish": {
|
|
"f1-score": 0.8435374149659864,
|
|
"precision": 0.8266666666666667,
|
|
"recall": 0.8611111111111112,
|
|
"support": 72.0
|
|
},
|
|
"bullish": {
|
|
"f1-score": 0.8,
|
|
"precision": 0.7948717948717948,
|
|
"recall": 0.8051948051948052,
|
|
"support": 77.0
|
|
},
|
|
"macro avg": {
|
|
"f1-score": 0.7934598400763816,
|
|
"precision": 0.7997720797720796,
|
|
"recall": 0.7887686387686387,
|
|
"support": 189.0
|
|
},
|
|
"neutral": {
|
|
"f1-score": 0.7368421052631579,
|
|
"precision": 0.7777777777777778,
|
|
"recall": 0.7,
|
|
"support": 40.0
|
|
},
|
|
"weighted avg": {
|
|
"f1-score": 0.8032189316829489,
|
|
"precision": 0.8033663455885677,
|
|
"recall": 0.8042328042328042,
|
|
"support": 189.0
|
|
}
|
|
},
|
|
"confusion_matrix": {
|
|
"labels": [
|
|
"bearish",
|
|
"neutral",
|
|
"bullish"
|
|
],
|
|
"matrix": [
|
|
[
|
|
62,
|
|
3,
|
|
7
|
|
],
|
|
[
|
|
3,
|
|
28,
|
|
9
|
|
],
|
|
[
|
|
10,
|
|
5,
|
|
62
|
|
]
|
|
]
|
|
},
|
|
"macro_f1": 0.7934598400763816,
|
|
"score_mae": 0.24237639972402897,
|
|
"test_examples": 189
|
|
},
|
|
"temporal_holdout": {
|
|
"accuracy": 0.746031746031746,
|
|
"balanced_accuracy": 0.7388071895424838,
|
|
"classification_report": {
|
|
"accuracy": 0.746031746031746,
|
|
"bearish": {
|
|
"f1-score": 0.8070175438596491,
|
|
"precision": 0.8518518518518519,
|
|
"recall": 0.7666666666666667,
|
|
"support": 90.0
|
|
},
|
|
"bullish": {
|
|
"f1-score": 0.6833333333333333,
|
|
"precision": 0.5942028985507246,
|
|
"recall": 0.803921568627451,
|
|
"support": 51.0
|
|
},
|
|
"macro avg": {
|
|
"f1-score": 0.7343315184513006,
|
|
"precision": 0.7469755150914571,
|
|
"recall": 0.7388071895424838,
|
|
"support": 189.0
|
|
},
|
|
"neutral": {
|
|
"f1-score": 0.7126436781609196,
|
|
"precision": 0.7948717948717948,
|
|
"recall": 0.6458333333333334,
|
|
"support": 48.0
|
|
},
|
|
"weighted avg": {
|
|
"f1-score": 0.7496744735401722,
|
|
"precision": 0.7678564055375648,
|
|
"recall": 0.746031746031746,
|
|
"support": 189.0
|
|
}
|
|
},
|
|
"confusion_matrix": {
|
|
"labels": [
|
|
"bearish",
|
|
"neutral",
|
|
"bullish"
|
|
],
|
|
"matrix": [
|
|
[
|
|
69,
|
|
5,
|
|
16
|
|
],
|
|
[
|
|
5,
|
|
31,
|
|
12
|
|
],
|
|
[
|
|
7,
|
|
3,
|
|
41
|
|
]
|
|
]
|
|
},
|
|
"macro_f1": 0.7343315184513006,
|
|
"score_mae": 0.27303151729050784,
|
|
"test_examples": 189,
|
|
"test_from": "2026-09-21T02:41:59+00:00",
|
|
"train_through": "2026-09-21T02:41:59+00:00"
|
|
}
|
|
},
|
|
"format_version": 3,
|
|
"model_path": "models/candidates/xau_gold_20260922.joblib",
|
|
"model_type": "domain_augmented_tfidf_logistic_regression",
|
|
"score_definition": "P(XAUUSD bullish) - P(XAUUSD bearish)",
|
|
"sklearn_version": "1.9.1",
|
|
"target": "XAUUSD",
|
|
"trained_at": "2026-09-23T00:35:57.482787+00:00",
|
|
"training": {
|
|
"database": "newsdata.db",
|
|
"domain_features": [
|
|
"explicit_xau_direction",
|
|
"usd_direction",
|
|
"treasury_yield_direction",
|
|
"hawkish_or_dovish_policy"
|
|
],
|
|
"duplicates_collapsed": 14,
|
|
"examples": 945,
|
|
"extra_labels": null,
|
|
"label_distribution": {
|
|
"bearish": 359,
|
|
"bullish": 384,
|
|
"neutral": 202
|
|
},
|
|
"label_priority": [
|
|
"xau_score",
|
|
"json",
|
|
"extra_labels"
|
|
],
|
|
"label_sources": {
|
|
"json": 640,
|
|
"xau_score": 319
|
|
},
|
|
"max_features": 50000,
|
|
"min_df": 2,
|
|
"neutral_threshold": 0.05,
|
|
"published_range": [
|
|
"2026-09-08T13:46:00+00:00",
|
|
"2026-09-22T18:13:29+00:00"
|
|
],
|
|
"random_state": 42,
|
|
"rows_without_target_label": 1702,
|
|
"scope": "gold",
|
|
"score_source": "llm",
|
|
"skipped_rows": 0,
|
|
"source_rows": 2661,
|
|
"target": "XAUUSD",
|
|
"target_labeled_rows": 959,
|
|
"target_score_key": "XAU",
|
|
"test_feed_rows": 0,
|
|
"test_size": 0.2
|
|
}
|
|
}
|