Initial import: XAUUSD news sentiment models, evals, and data tooling

TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked.
2026-09-27 19:22:30 -04:00
commit 01baef48ff
98 changed files with 1719757 additions and 0 deletions
@@ -0,0 +1,145 @@
+{
+  "db": "../newsdata.db",
+  "generated_at": "20260927_1907_transformer",
+  "in_sample_or_mixed": {
+    "headline": {
+      "accuracy": 0.7375805839969662,
+      "confusion": [
+        [
+          774,
+          84,
+          58
+        ],
+        [
+          64,
+          715,
+          117
+        ],
+        [
+          109,
+          260,
+          456
+        ]
+      ],
+      "directional_accuracy": 0.7064905226881103,
+      "directional_hits": 1230,
+      "directional_n": 1741,
+      "hits": 1945,
+      "macro_f1": 0.7295831004270902,
+      "mae": 0.30792290481607887,
+      "n": 2637,
+      "sign_agreement": 0.8804581245526127,
+      "sign_hits": 1230,
+      "sign_n": 1397,
+      "spearman": 0.7678844230729466
+    }
+  },
+  "model": {
+    "final": false,
+    "model_path": "/home/harwinder/Development/ML-NEWS/models/xau_transformer",
+    "model_type": "two_head_transformer:microsoft/deberta-v3-small",
+    "oos_from": "2026-09-18",
+    "relevance_threshold": 0.2,
+    "trained_at": "2026-09-27T19:03:25.366953+00:00",
+    "training_examples": 2402
+  },
+  "out_of_sample": {
+    "all": {
+      "accuracy": 0.6565557729941291,
+      "confusion": [
+        [
+          227,
+          36,
+          44
+        ],
+        [
+          26,
+          266,
+          74
+        ],
+        [
+          53,
+          118,
+          178
+        ]
+      ],
+      "directional_accuracy": 0.6173780487804879,
+      "directional_hits": 405,
+      "directional_n": 656,
+      "hits": 671,
+      "macro_f1": 0.6564675567772849,
+      "n": 1022,
+      "sign_agreement": 0.8067729083665338,
+      "sign_hits": 405,
+      "sign_n": 502
+    },
+    "gold": {
+      "accuracy": 0.7507987220447284,
+      "confusion": [
+        [
+          91,
+          4,
+          14
+        ],
+        [
+          9,
+          31,
+          19
+        ],
+        [
+          10,
+          22,
+          113
+        ]
+      ],
+      "directional_accuracy": 0.8031496062992126,
+      "directional_hits": 204,
+      "directional_n": 254,
+      "hits": 235,
+      "macro_f1": 0.7140550964455633,
+      "n": 313,
+      "sign_agreement": 0.8947368421052632,
+      "sign_hits": 204,
+      "sign_n": 228
+    },
+    "macro": {
+      "accuracy": 0.614950634696756,
+      "confusion": [
+        [
+          136,
+          32,
+          30
+        ],
+        [
+          17,
+          235,
+          55
+        ],
+        [
+          43,
+          96,
+          65
+        ]
+      ],
+      "directional_accuracy": 0.5,
+      "directional_hits": 201,
+      "directional_n": 402,
+      "hits": 436,
+      "macro_f1": 0.5863598352269169,
+      "n": 709,
+      "sign_agreement": 0.7335766423357665,
+      "sign_hits": 201,
+      "sign_n": 274
+    }
+  },
+  "relevance_out_of_sample": {
+    "direction_accuracy": 0.8976377952755905,
+    "direction_n": 381,
+    "f1": 0.775881683731513,
+    "n": 881,
+    "precision": 0.6847389558232931,
+    "recall": 0.89501312335958
+  },
+  "rows": 2637,
+  "threshold": 0.05
+}