Initial import: XAUUSD news sentiment models, evals, and data tooling

TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-27 19:22:30 -04:00
commit 1d72e4006a
98 changed files with 1719757 additions and 0 deletions
@@ -0,0 +1,145 @@
+{
+  "db": "../newsdata.db",
+  "generated_at": "20260927_1947_transformer",
+  "in_sample_or_mixed": {
+    "headline": {
+      "accuracy": 0.7724687144482366,
+      "confusion": [
+        [
+          773,
+          97,
+          46
+        ],
+        [
+          49,
+          784,
+          63
+        ],
+        [
+          78,
+          267,
+          480
+        ]
+      ],
+      "directional_accuracy": 0.7197013210798392,
+      "directional_hits": 1253,
+      "directional_n": 1741,
+      "hits": 2037,
+      "macro_f1": 0.7657899696449985,
+      "mae": 0.2876833522942738,
+      "n": 2637,
+      "sign_agreement": 0.9099491648511256,
+      "sign_hits": 1253,
+      "sign_n": 1377,
+      "spearman": 0.8074118582323724
+    }
+  },
+  "model": {
+    "final": false,
+    "model_path": "/home/harwinder/Development/ML-NEWS/models/xau_transformer",
+    "model_type": "two_head_transformer:microsoft/deberta-v3-small",
+    "oos_from": "2026-09-18",
+    "relevance_threshold": 0.2,
+    "trained_at": "2026-09-27T19:46:15.509042+00:00",
+    "training_examples": 10690
+  },
+  "out_of_sample": {
+    "all": {
+      "accuracy": 0.7181996086105675,
+      "confusion": [
+        [
+          229,
+          45,
+          33
+        ],
+        [
+          21,
+          308,
+          37
+        ],
+        [
+          38,
+          114,
+          197
+        ]
+      ],
+      "directional_accuracy": 0.649390243902439,
+      "directional_hits": 426,
+      "directional_n": 656,
+      "hits": 734,
+      "macro_f1": 0.7162846956964604,
+      "n": 1022,
+      "sign_agreement": 0.8571428571428571,
+      "sign_hits": 426,
+      "sign_n": 497
+    },
+    "gold": {
+      "accuracy": 0.8306709265175719,
+      "confusion": [
+        [
+          100,
+          1,
+          8
+        ],
+        [
+          9,
+          40,
+          10
+        ],
+        [
+          13,
+          12,
+          120
+        ]
+      ],
+      "directional_accuracy": 0.8661417322834646,
+      "directional_hits": 220,
+      "directional_n": 254,
+      "hits": 260,
+      "macro_f1": 0.8093810390630178,
+      "n": 313,
+      "sign_agreement": 0.9128630705394191,
+      "sign_hits": 220,
+      "sign_n": 241
+    },
+    "macro": {
+      "accuracy": 0.6685472496473907,
+      "confusion": [
+        [
+          129,
+          44,
+          25
+        ],
+        [
+          12,
+          268,
+          27
+        ],
+        [
+          25,
+          102,
+          77
+        ]
+      ],
+      "directional_accuracy": 0.5124378109452736,
+      "directional_hits": 206,
+      "directional_n": 402,
+      "hits": 474,
+      "macro_f1": 0.6382218663772061,
+      "n": 709,
+      "sign_agreement": 0.8046875,
+      "sign_hits": 206,
+      "sign_n": 256
+    }
+  },
+  "relevance_out_of_sample": {
+    "direction_accuracy": 0.931758530183727,
+    "direction_n": 381,
+    "f1": 0.8374851720047449,
+    "n": 881,
+    "precision": 0.7640692640692641,
+    "recall": 0.926509186351706
+  },
+  "rows": 2637,
+  "threshold": 0.05
+}