harry/News-ML-Decoder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial import: XAUUSD news sentiment models, evals, and data tooling

Browse Source 
TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the
two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py)
with its Google News crawler and DeepSeek labeller (data_collect/), the
golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27
results and progress notes.

Transformer weights (model.pt, model.onnx) and the crawl database are not
tracked; rebuild with transformer/train.py and transformer/export_onnx.py.
The DeepSeek label cache in data/ is tracked.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Harwinder Sekhon
2026-09-27 19:22:30 -04:00
commit 1d72e4006a
98 changed files with 1719757 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
eval/results/golden_20260923_0036_xauusd_sentiment.json
+277
View File
@@ -0,0 +1,277 @@
{
"band": 0.15,
"band_sweep": {
"0.05": {
"hits": 37,
"n": 78,
"neutral_predicted": 5
},
"0.1": {
"hits": 35,
"n": 78,
"neutral_predicted": 11
},
"0.15": {
"hits": 36,
"n": 78,
"neutral_predicted": 21
},
"0.2": {
"hits": 33,
"n": 78,
"neutral_predicted": 26
},
"0.3": {
"hits": 27,
"n": 78,
"neutral_predicted": 42
},
"0.5": {
"hits": 17,
"n": 78,
"neutral_predicted": 65
}
},
"blocks": {
"DeepSeek live (xau through band \u00b10.15)": {
"accuracy": 1.0,
"confusion": [
[
32,
0,
0
],
[
0,
8,
0
],
[
0,
0,
38
]
],
"directional_accuracy": 1.0,
"directional_hits": 70,
"directional_n": 70,
"hits": 78,
"macro_f1": 1.0,
"n": 78,
"sign_agreement": 1.0,
"sign_hits": 70,
"sign_n": 70
},
"keyword fallback (band \u00b10.15)": {
"accuracy": 0.3076923076923077,
"confusion": [
[
6,
15,
11
],
[
0,
8,
0
],
[
12,
16,
10
]
],
"directional_accuracy": 0.22857142857142856,
"directional_hits": 16,
"directional_n": 70,
"hits": 24,
"macro_f1": 0.30646952758745044,
"n": 78,
"sign_agreement": 0.41025641025641024,
"sign_hits": 16,
"sign_n": 39
},
"model (native argmax label)": {
"accuracy": 0.5,
"confusion": [
[
13,
2,
17
],
[
2,
1,
5
],
[
12,
1,
25
]
],
"directional_accuracy": 0.5428571428571428,
"directional_hits": 38,
"directional_n": 70,
"hits": 39,
"macro_f1": 0.3985266422953362,
"n": 78,
"sign_agreement": 0.5671641791044776,
"sign_hits": 38,
"sign_n": 67
},
"model (score through band \u00b10.15)": {
"accuracy": 0.46153846153846156,
"confusion": [
[
11,
7,
14
],
[
2,
4,
2
],
[
7,
10,
21
]
],
"directional_accuracy": 0.45714285714285713,
"directional_hits": 32,
"directional_n": 70,
"hits": 36,
"macro_f1": 0.4196463306808134,
"n": 78,
"sign_agreement": 0.6037735849056604,
"sign_hits": 32,
"sign_n": 53
}
},
"deepseek": {
"batch_ms_max": null,
"batch_ms_mean": null,
"cache_hits": 78,
"calls": 0,
"failed_calls": 0,
"headlines_sent": 0,
"model": "deepseek-v4-flash",
"per_headline_ms": null,
"prompt_sha1": "d32e04c6a87d019f326f111b39c99b6dd8ce794d"
},
"deepseek_scored": 78,
"generated_at": "20260923_0036_xauusd_sentiment",
"mcnemar": {
"model_vs_deepseek": {
"deepseek_only_right": 39,
"model_only_right": 0,
"p": 3.637978807091713e-12
},
"model_vs_keyword": {
"keyword_only_right": 13,
"model_only_right": 28,
"p": 0.027533155829587486
}
},
"model": {
"label_distribution": {
"bearish": 250,
"bullish": 221,
"neutral": 135
},
"model_path": "../models/xauusd_sentiment.joblib",
"model_type": "domain_augmented_tfidf_logistic_regression",
"neutral_threshold": 0.05,
"sklearn_version": "1.9.1",
"trained_at": "2026-09-17T18:54:46.714699+00:00",
"training_examples": 606
},
"model_latency_ms": {
"max": 6.73,
"mean": 1.0356410256410256,
"median": 0.94
},
"n": 78,
"per_category": {
"distractors": {
"deepseek": 8,
"deepseek_n": 8,
"keyword": 8,
"model": 1,
"n": 8
},
"dovish_fed": {
"deepseek": 8,
"deepseek_n": 8,
"keyword": 7,
"model": 6,
"n": 8
},
"gold_direct": {
"deepseek": 8,
"deepseek_n": 8,
"keyword": 4,
"model": 8,
"n": 8
},
"hawkish_fed": {
"deepseek": 8,
"deepseek_n": 8,
"keyword": 2,
"model": 3,
"n": 8
},
"inverted_metrics": {
"deepseek": 6,
"deepseek_n": 6,
"keyword": 0,
"model": 3,
"n": 6
},
"negation_traps": {
"deepseek": 8,
"deepseek_n": 8,
"keyword": 2,
"model": 0,
"n": 8
},
"real_yields": {
"deepseek": 6,
"deepseek_n": 6,
"keyword": 1,
"model": 3,
"n": 6
},
"risk_on": {
"deepseek": 4,
"deepseek_n": 4,
"keyword": 0,
"model": 1,
"n": 4
},
"safe_haven": {
"deepseek": 8,
"deepseek_n": 8,
"keyword": 0,
"model": 8,
"n": 8
},
"strong_usd": {
"deepseek": 7,
"deepseek_n": 7,
"keyword": 0,
"model": 2,
"n": 7
},
"weak_usd": {
"deepseek": 7,
"deepseek_n": 7,
"keyword": 0,
"model": 4,
"n": 7
}
},
"spearman_model_vs_deepseek": 0.30475630790202185
}