Initial import: XAUUSD news sentiment models, evals, and data tooling

TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked.
2026-09-27 19:22:30 -04:00
commit 01baef48ff
98 changed files with 1719757 additions and 0 deletions
@@ -0,0 +1,145 @@
+{
+  "db": "../newsdata.db",
+  "generated_at": "20260927_1900_transformer",
+  "in_sample_or_mixed": {
+    "headline": {
+      "accuracy": 0.7133105802047781,
+      "confusion": [
+        [
+          769,
+          95,
+          52
+        ],
+        [
+          95,
+          677,
+          124
+        ],
+        [
+          140,
+          250,
+          435
+        ]
+      ],
+      "directional_accuracy": 0.691556576680069,
+      "directional_hits": 1204,
+      "directional_n": 1741,
+      "hits": 1881,
+      "macro_f1": 0.7042783133948399,
+      "mae": 0.24411205915813428,
+      "n": 2637,
+      "sign_agreement": 0.8624641833810889,
+      "sign_hits": 1204,
+      "sign_n": 1396,
+      "spearman": 0.7415919015733857
+    }
+  },
+  "model": {
+    "final": false,
+    "model_path": "/home/harwinder/Development/ML-NEWS/models/xau_transformer",
+    "model_type": "two_head_transformer:microsoft/deberta-v3-small",
+    "oos_from": "2026-09-18",
+    "relevance_threshold": 0.2,
+    "trained_at": "2026-09-27T18:58:27.596998+00:00",
+    "training_examples": 2402
+  },
+  "out_of_sample": {
+    "all": {
+      "accuracy": 0.6438356164383562,
+      "confusion": [
+        [
+          227,
+          40,
+          40
+        ],
+        [
+          29,
+          270,
+          67
+        ],
+        [
+          66,
+          122,
+          161
+        ]
+      ],
+      "directional_accuracy": 0.5914634146341463,
+      "directional_hits": 388,
+      "directional_n": 656,
+      "hits": 658,
+      "macro_f1": 0.6401174660955584,
+      "n": 1022,
+      "sign_agreement": 0.7854251012145749,
+      "sign_hits": 388,
+      "sign_n": 494
+    },
+    "gold": {
+      "accuracy": 0.744408945686901,
+      "confusion": [
+        [
+          91,
+          5,
+          13
+        ],
+        [
+          7,
+          33,
+          19
+        ],
+        [
+          13,
+          23,
+          109
+        ]
+      ],
+      "directional_accuracy": 0.7874015748031497,
+      "directional_hits": 200,
+      "directional_n": 254,
+      "hits": 233,
+      "macro_f1": 0.7131701631701631,
+      "n": 313,
+      "sign_agreement": 0.8849557522123894,
+      "sign_hits": 200,
+      "sign_n": 226
+    },
+    "macro": {
+      "accuracy": 0.5994358251057827,
+      "confusion": [
+        [
+          136,
+          35,
+          27
+        ],
+        [
+          22,
+          237,
+          48
+        ],
+        [
+          53,
+          99,
+          52
+        ]
+      ],
+      "directional_accuracy": 0.46766169154228854,
+      "directional_hits": 188,
+      "directional_n": 402,
+      "hits": 425,
+      "macro_f1": 0.559450371611602,
+      "n": 709,
+      "sign_agreement": 0.7014925373134329,
+      "sign_hits": 188,
+      "sign_n": 268
+    }
+  },
+  "relevance_out_of_sample": {
+    "direction_accuracy": 0.8871391076115486,
+    "direction_n": 381,
+    "f1": 0.7733026467203681,
+    "n": 881,
+    "precision": 0.6885245901639344,
+    "recall": 0.8818897637795275
+  },
+  "rows": 2637,
+  "threshold": 0.05
+}