{
  "db": "../newsdata.db",
  "generated_at": "20260927_1927_transformer",
  "in_sample_or_mixed": {
    "headline": {
      "accuracy": 0.7682973075464543,
      "confusion": [
        [
          764,
          117,
          35
        ],
        [
          42,
          821,
          33
        ],
        [
          82,
          302,
          441
        ]
      ],
      "directional_accuracy": 0.6921309592188397,
      "directional_hits": 1205,
      "directional_n": 1741,
      "hits": 2026,
      "macro_f1": 0.7589675529791181,
      "mae": 0.2953307546454304,
      "n": 2637,
      "sign_agreement": 0.9114977307110439,
      "sign_hits": 1205,
      "sign_n": 1322,
      "spearman": 0.7978166855256467
    }
  },
  "model": {
    "final": false,
    "model_path": "/home/harwinder/Development/ML-NEWS/models/xau_transformer",
    "model_type": "two_head_transformer:microsoft/deberta-v3-small",
    "oos_from": "2026-09-18",
    "relevance_threshold": 0.6,
    "trained_at": "2026-09-27T19:22:03.286521+00:00",
    "training_examples": 7460
  },
  "out_of_sample": {
    "all": {
      "accuracy": 0.7191780821917808,
      "confusion": [
        [
          231,
          52,
          24
        ],
        [
          18,
          327,
          21
        ],
        [
          37,
          135,
          177
        ]
      ],
      "directional_accuracy": 0.6219512195121951,
      "directional_hits": 408,
      "directional_n": 656,
      "hits": 735,
      "macro_f1": 0.7140787226553597,
      "n": 1022,
      "sign_agreement": 0.8699360341151386,
      "sign_hits": 408,
      "sign_n": 469
    },
    "gold": {
      "accuracy": 0.8370607028753994,
      "confusion": [
        [
          102,
          2,
          5
        ],
        [
          8,
          46,
          5
        ],
        [
          9,
          22,
          114
        ]
      ],
      "directional_accuracy": 0.8503937007874016,
      "directional_hits": 216,
      "directional_n": 254,
      "hits": 262,
      "macro_f1": 0.8184995932671942,
      "n": 313,
      "sign_agreement": 0.9391304347826087,
      "sign_hits": 216,
      "sign_n": 230
    },
    "macro": {
      "accuracy": 0.6671368124118476,
      "confusion": [
        [
          129,
          50,
          19
        ],
        [
          10,
          281,
          16
        ],
        [
          28,
          113,
          63
        ]
      ],
      "directional_accuracy": 0.47761194029850745,
      "directional_hits": 192,
      "directional_n": 402,
      "hits": 473,
      "macro_f1": 0.6241344702371295,
      "n": 709,
      "sign_agreement": 0.803347280334728,
      "sign_hits": 192,
      "sign_n": 239
    }
  },
  "relevance_out_of_sample": {
    "direction_accuracy": 0.9422572178477691,
    "direction_n": 381,
    "f1": 0.8414481897627966,
    "n": 881,
    "precision": 0.8023809523809524,
    "recall": 0.884514435695538
  },
  "rows": 2637,
  "threshold": 0.05
}