{
  "db": "../newsdata.db",
  "generated_at": "20260927_1907_transformer",
  "in_sample_or_mixed": {
    "headline": {
      "accuracy": 0.7375805839969662,
      "confusion": [
        [
          774,
          84,
          58
        ],
        [
          64,
          715,
          117
        ],
        [
          109,
          260,
          456
        ]
      ],
      "directional_accuracy": 0.7064905226881103,
      "directional_hits": 1230,
      "directional_n": 1741,
      "hits": 1945,
      "macro_f1": 0.7295831004270902,
      "mae": 0.30792290481607887,
      "n": 2637,
      "sign_agreement": 0.8804581245526127,
      "sign_hits": 1230,
      "sign_n": 1397,
      "spearman": 0.7678844230729466
    }
  },
  "model": {
    "final": false,
    "model_path": "/home/harwinder/Development/ML-NEWS/models/xau_transformer",
    "model_type": "two_head_transformer:microsoft/deberta-v3-small",
    "oos_from": "2026-09-18",
    "relevance_threshold": 0.2,
    "trained_at": "2026-09-27T19:03:25.366953+00:00",
    "training_examples": 2402
  },
  "out_of_sample": {
    "all": {
      "accuracy": 0.6565557729941291,
      "confusion": [
        [
          227,
          36,
          44
        ],
        [
          26,
          266,
          74
        ],
        [
          53,
          118,
          178
        ]
      ],
      "directional_accuracy": 0.6173780487804879,
      "directional_hits": 405,
      "directional_n": 656,
      "hits": 671,
      "macro_f1": 0.6564675567772849,
      "n": 1022,
      "sign_agreement": 0.8067729083665338,
      "sign_hits": 405,
      "sign_n": 502
    },
    "gold": {
      "accuracy": 0.7507987220447284,
      "confusion": [
        [
          91,
          4,
          14
        ],
        [
          9,
          31,
          19
        ],
        [
          10,
          22,
          113
        ]
      ],
      "directional_accuracy": 0.8031496062992126,
      "directional_hits": 204,
      "directional_n": 254,
      "hits": 235,
      "macro_f1": 0.7140550964455633,
      "n": 313,
      "sign_agreement": 0.8947368421052632,
      "sign_hits": 204,
      "sign_n": 228
    },
    "macro": {
      "accuracy": 0.614950634696756,
      "confusion": [
        [
          136,
          32,
          30
        ],
        [
          17,
          235,
          55
        ],
        [
          43,
          96,
          65
        ]
      ],
      "directional_accuracy": 0.5,
      "directional_hits": 201,
      "directional_n": 402,
      "hits": 436,
      "macro_f1": 0.5863598352269169,
      "n": 709,
      "sign_agreement": 0.7335766423357665,
      "sign_hits": 201,
      "sign_n": 274
    }
  },
  "relevance_out_of_sample": {
    "direction_accuracy": 0.8976377952755905,
    "direction_n": 381,
    "f1": 0.775881683731513,
    "n": 881,
    "precision": 0.6847389558232931,
    "recall": 0.89501312335958
  },
  "rows": 2637,
  "threshold": 0.05
}