Initial import: XAUUSD news sentiment models, evals, and data tooling

TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked.
2026-09-27 19:22:30 -04:00
commit 01baef48ff
98 changed files with 1719757 additions and 0 deletions
@@ -0,0 +1,145 @@
+{
+  "db": "../newsdata.db",
+  "generated_at": "20260927_1927_transformer",
+  "in_sample_or_mixed": {
+    "headline": {
+      "accuracy": 0.7682973075464543,
+      "confusion": [
+        [
+          764,
+          117,
+          35
+        ],
+        [
+          42,
+          821,
+          33
+        ],
+        [
+          82,
+          302,
+          441
+        ]
+      ],
+      "directional_accuracy": 0.6921309592188397,
+      "directional_hits": 1205,
+      "directional_n": 1741,
+      "hits": 2026,
+      "macro_f1": 0.7589675529791181,
+      "mae": 0.2953307546454304,
+      "n": 2637,
+      "sign_agreement": 0.9114977307110439,
+      "sign_hits": 1205,
+      "sign_n": 1322,
+      "spearman": 0.7978166855256467
+    }
+  },
+  "model": {
+    "final": false,
+    "model_path": "/home/harwinder/Development/ML-NEWS/models/xau_transformer",
+    "model_type": "two_head_transformer:microsoft/deberta-v3-small",
+    "oos_from": "2026-09-18",
+    "relevance_threshold": 0.6,
+    "trained_at": "2026-09-27T19:22:03.286521+00:00",
+    "training_examples": 7460
+  },
+  "out_of_sample": {
+    "all": {
+      "accuracy": 0.7191780821917808,
+      "confusion": [
+        [
+          231,
+          52,
+          24
+        ],
+        [
+          18,
+          327,
+          21
+        ],
+        [
+          37,
+          135,
+          177
+        ]
+      ],
+      "directional_accuracy": 0.6219512195121951,
+      "directional_hits": 408,
+      "directional_n": 656,
+      "hits": 735,
+      "macro_f1": 0.7140787226553597,
+      "n": 1022,
+      "sign_agreement": 0.8699360341151386,
+      "sign_hits": 408,
+      "sign_n": 469
+    },
+    "gold": {
+      "accuracy": 0.8370607028753994,
+      "confusion": [
+        [
+          102,
+          2,
+          5
+        ],
+        [
+          8,
+          46,
+          5
+        ],
+        [
+          9,
+          22,
+          114
+        ]
+      ],
+      "directional_accuracy": 0.8503937007874016,
+      "directional_hits": 216,
+      "directional_n": 254,
+      "hits": 262,
+      "macro_f1": 0.8184995932671942,
+      "n": 313,
+      "sign_agreement": 0.9391304347826087,
+      "sign_hits": 216,
+      "sign_n": 230
+    },
+    "macro": {
+      "accuracy": 0.6671368124118476,
+      "confusion": [
+        [
+          129,
+          50,
+          19
+        ],
+        [
+          10,
+          281,
+          16
+        ],
+        [
+          28,
+          113,
+          63
+        ]
+      ],
+      "directional_accuracy": 0.47761194029850745,
+      "directional_hits": 192,
+      "directional_n": 402,
+      "hits": 473,
+      "macro_f1": 0.6241344702371295,
+      "n": 709,
+      "sign_agreement": 0.803347280334728,
+      "sign_hits": 192,
+      "sign_n": 239
+    }
+  },
+  "relevance_out_of_sample": {
+    "direction_accuracy": 0.9422572178477691,
+    "direction_n": 381,
+    "f1": 0.8414481897627966,
+    "n": 881,
+    "precision": 0.8023809523809524,
+    "recall": 0.884514435695538
+  },
+  "rows": 2637,
+  "threshold": 0.05
+}