{
  "db": "../newsdata.db",
  "generated_at": "20260927_2314_transformer",
  "in_sample_or_mixed": {
    "headline": {
      "accuracy": 0.7671596511186954,
      "confusion": [
        [
          777,
          86,
          53
        ],
        [
          73,
          754,
          69
        ],
        [
          87,
          246,
          492
        ]
      ],
      "directional_accuracy": 0.7288914417001723,
      "directional_hits": 1269,
      "directional_n": 1741,
      "hits": 2023,
      "macro_f1": 0.7610986239900465,
      "mae": 0.2995589685248388,
      "n": 2637,
      "sign_agreement": 0.9006387508871541,
      "sign_hits": 1269,
      "sign_n": 1409,
      "spearman": 0.802027157757526
    }
  },
  "model": {
    "final": false,
    "model_path": "/home/harwinder/Development/ML-NEWS/models/xau_transformer",
    "model_type": "two_head_transformer:microsoft/deberta-v3-small",
    "oos_from": "2026-09-18",
    "relevance_threshold": 0.2,
    "trained_at": "2026-09-27T23:12:37.970737+00:00",
    "training_examples": 15416
  },
  "out_of_sample": {
    "all": {
      "accuracy": 0.7152641878669276,
      "confusion": [
        [
          236,
          37,
          34
        ],
        [
          42,
          292,
          32
        ],
        [
          42,
          104,
          203
        ]
      ],
      "directional_accuracy": 0.6692073170731707,
      "directional_hits": 439,
      "directional_n": 656,
      "hits": 731,
      "macro_f1": 0.7135542131452324,
      "n": 1022,
      "sign_agreement": 0.8524271844660194,
      "sign_hits": 439,
      "sign_n": 515
    },
    "gold": {
      "accuracy": 0.7859424920127795,
      "confusion": [
        [
          99,
          1,
          9
        ],
        [
          13,
          32,
          14
        ],
        [
          19,
          11,
          115
        ]
      ],
      "directional_accuracy": 0.84251968503937,
      "directional_hits": 214,
      "directional_n": 254,
      "hits": 246,
      "macro_f1": 0.7530266904525025,
      "n": 313,
      "sign_agreement": 0.8842975206611571,
      "sign_hits": 214,
      "sign_n": 242
    },
    "macro": {
      "accuracy": 0.6840620592383639,
      "confusion": [
        [
          137,
          36,
          25
        ],
        [
          29,
          260,
          18
        ],
        [
          23,
          93,
          88
        ]
      ],
      "directional_accuracy": 0.5597014925373134,
      "directional_hits": 225,
      "directional_n": 402,
      "hits": 485,
      "macro_f1": 0.6601699690090934,
      "n": 709,
      "sign_agreement": 0.8241758241758241,
      "sign_hits": 225,
      "sign_n": 273
    }
  },
  "relevance_out_of_sample": {
    "direction_accuracy": 0.8923884514435696,
    "direction_n": 381,
    "f1": 0.8143021914648213,
    "n": 881,
    "precision": 0.7263374485596708,
    "recall": 0.926509186351706
  },
  "rows": 2637,
  "threshold": 0.05
}