Initial import: XAUUSD news sentiment models, evals, and data tooling

TF-IDF baseline (news_sentiment.py, models/xauusd_sentiment.joblib) and the two-head DeBERTa-v3-small transformer (transformer/, news_transformer.py) with its Google News crawler and DeepSeek labeller (data_collect/), the golden-set and archive evals (eval/), and the 2026-09-22 and 2026-09-27 results and progress notes. Transformer weights (model.pt, model.onnx) and the crawl database are not tracked; rebuild with transformer/train.py and transformer/export_onnx.py. The DeepSeek label cache in data/ is tracked. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-27 19:22:30 -04:00
commit 1d72e4006a
98 changed files with 1719757 additions and 0 deletions
@@ -0,0 +1,145 @@
+{
+  "db": "../newsdata.db",
+  "generated_at": "20260927_2314_transformer",
+  "in_sample_or_mixed": {
+    "headline": {
+      "accuracy": 0.7671596511186954,
+      "confusion": [
+        [
+          777,
+          86,
+          53
+        ],
+        [
+          73,
+          754,
+          69
+        ],
+        [
+          87,
+          246,
+          492
+        ]
+      ],
+      "directional_accuracy": 0.7288914417001723,
+      "directional_hits": 1269,
+      "directional_n": 1741,
+      "hits": 2023,
+      "macro_f1": 0.7610986239900465,
+      "mae": 0.2995589685248388,
+      "n": 2637,
+      "sign_agreement": 0.9006387508871541,
+      "sign_hits": 1269,
+      "sign_n": 1409,
+      "spearman": 0.802027157757526
+    }
+  },
+  "model": {
+    "final": false,
+    "model_path": "/home/harwinder/Development/ML-NEWS/models/xau_transformer",
+    "model_type": "two_head_transformer:microsoft/deberta-v3-small",
+    "oos_from": "2026-09-18",
+    "relevance_threshold": 0.2,
+    "trained_at": "2026-09-27T23:12:37.970737+00:00",
+    "training_examples": 15416
+  },
+  "out_of_sample": {
+    "all": {
+      "accuracy": 0.7152641878669276,
+      "confusion": [
+        [
+          236,
+          37,
+          34
+        ],
+        [
+          42,
+          292,
+          32
+        ],
+        [
+          42,
+          104,
+          203
+        ]
+      ],
+      "directional_accuracy": 0.6692073170731707,
+      "directional_hits": 439,
+      "directional_n": 656,
+      "hits": 731,
+      "macro_f1": 0.7135542131452324,
+      "n": 1022,
+      "sign_agreement": 0.8524271844660194,
+      "sign_hits": 439,
+      "sign_n": 515
+    },
+    "gold": {
+      "accuracy": 0.7859424920127795,
+      "confusion": [
+        [
+          99,
+          1,
+          9
+        ],
+        [
+          13,
+          32,
+          14
+        ],
+        [
+          19,
+          11,
+          115
+        ]
+      ],
+      "directional_accuracy": 0.84251968503937,
+      "directional_hits": 214,
+      "directional_n": 254,
+      "hits": 246,
+      "macro_f1": 0.7530266904525025,
+      "n": 313,
+      "sign_agreement": 0.8842975206611571,
+      "sign_hits": 214,
+      "sign_n": 242
+    },
+    "macro": {
+      "accuracy": 0.6840620592383639,
+      "confusion": [
+        [
+          137,
+          36,
+          25
+        ],
+        [
+          29,
+          260,
+          18
+        ],
+        [
+          23,
+          93,
+          88
+        ]
+      ],
+      "directional_accuracy": 0.5597014925373134,
+      "directional_hits": 225,
+      "directional_n": 402,
+      "hits": 485,
+      "macro_f1": 0.6601699690090934,
+      "n": 709,
+      "sign_agreement": 0.8241758241758241,
+      "sign_hits": 225,
+      "sign_n": 273
+    }
+  },
+  "relevance_out_of_sample": {
+    "direction_accuracy": 0.8923884514435696,
+    "direction_n": 381,
+    "f1": 0.8143021914648213,
+    "n": 881,
+    "precision": 0.7263374485596708,
+    "recall": 0.926509186351706
+  },
+  "rows": 2637,
+  "threshold": 0.05
+}