{
  "db": "../newsdata.db",
  "generated_at": "20260927_1947_transformer",
  "in_sample_or_mixed": {
    "headline": {
      "accuracy": 0.7724687144482366,
      "confusion": [
        [
          773,
          97,
          46
        ],
        [
          49,
          784,
          63
        ],
        [
          78,
          267,
          480
        ]
      ],
      "directional_accuracy": 0.7197013210798392,
      "directional_hits": 1253,
      "directional_n": 1741,
      "hits": 2037,
      "macro_f1": 0.7657899696449985,
      "mae": 0.2876833522942738,
      "n": 2637,
      "sign_agreement": 0.9099491648511256,
      "sign_hits": 1253,
      "sign_n": 1377,
      "spearman": 0.8074118582323724
    }
  },
  "model": {
    "final": false,
    "model_path": "/home/harwinder/Development/ML-NEWS/models/xau_transformer",
    "model_type": "two_head_transformer:microsoft/deberta-v3-small",
    "oos_from": "2026-09-18",
    "relevance_threshold": 0.2,
    "trained_at": "2026-09-27T19:46:15.509042+00:00",
    "training_examples": 10690
  },
  "out_of_sample": {
    "all": {
      "accuracy": 0.7181996086105675,
      "confusion": [
        [
          229,
          45,
          33
        ],
        [
          21,
          308,
          37
        ],
        [
          38,
          114,
          197
        ]
      ],
      "directional_accuracy": 0.649390243902439,
      "directional_hits": 426,
      "directional_n": 656,
      "hits": 734,
      "macro_f1": 0.7162846956964604,
      "n": 1022,
      "sign_agreement": 0.8571428571428571,
      "sign_hits": 426,
      "sign_n": 497
    },
    "gold": {
      "accuracy": 0.8306709265175719,
      "confusion": [
        [
          100,
          1,
          8
        ],
        [
          9,
          40,
          10
        ],
        [
          13,
          12,
          120
        ]
      ],
      "directional_accuracy": 0.8661417322834646,
      "directional_hits": 220,
      "directional_n": 254,
      "hits": 260,
      "macro_f1": 0.8093810390630178,
      "n": 313,
      "sign_agreement": 0.9128630705394191,
      "sign_hits": 220,
      "sign_n": 241
    },
    "macro": {
      "accuracy": 0.6685472496473907,
      "confusion": [
        [
          129,
          44,
          25
        ],
        [
          12,
          268,
          27
        ],
        [
          25,
          102,
          77
        ]
      ],
      "directional_accuracy": 0.5124378109452736,
      "directional_hits": 206,
      "directional_n": 402,
      "hits": 474,
      "macro_f1": 0.6382218663772061,
      "n": 709,
      "sign_agreement": 0.8046875,
      "sign_hits": 206,
      "sign_n": 256
    }
  },
  "relevance_out_of_sample": {
    "direction_accuracy": 0.931758530183727,
    "direction_n": 381,
    "f1": 0.8374851720047449,
    "n": 881,
    "precision": 0.7640692640692641,
    "recall": 0.926509186351706
  },
  "rows": 2637,
  "threshold": 0.05
}