{
  "db": "../newsdata.db",
  "generated_at": "20260927_1900_transformer",
  "in_sample_or_mixed": {
    "headline": {
      "accuracy": 0.7133105802047781,
      "confusion": [
        [
          769,
          95,
          52
        ],
        [
          95,
          677,
          124
        ],
        [
          140,
          250,
          435
        ]
      ],
      "directional_accuracy": 0.691556576680069,
      "directional_hits": 1204,
      "directional_n": 1741,
      "hits": 1881,
      "macro_f1": 0.7042783133948399,
      "mae": 0.24411205915813428,
      "n": 2637,
      "sign_agreement": 0.8624641833810889,
      "sign_hits": 1204,
      "sign_n": 1396,
      "spearman": 0.7415919015733857
    }
  },
  "model": {
    "final": false,
    "model_path": "/home/harwinder/Development/ML-NEWS/models/xau_transformer",
    "model_type": "two_head_transformer:microsoft/deberta-v3-small",
    "oos_from": "2026-09-18",
    "relevance_threshold": 0.2,
    "trained_at": "2026-09-27T18:58:27.596998+00:00",
    "training_examples": 2402
  },
  "out_of_sample": {
    "all": {
      "accuracy": 0.6438356164383562,
      "confusion": [
        [
          227,
          40,
          40
        ],
        [
          29,
          270,
          67
        ],
        [
          66,
          122,
          161
        ]
      ],
      "directional_accuracy": 0.5914634146341463,
      "directional_hits": 388,
      "directional_n": 656,
      "hits": 658,
      "macro_f1": 0.6401174660955584,
      "n": 1022,
      "sign_agreement": 0.7854251012145749,
      "sign_hits": 388,
      "sign_n": 494
    },
    "gold": {
      "accuracy": 0.744408945686901,
      "confusion": [
        [
          91,
          5,
          13
        ],
        [
          7,
          33,
          19
        ],
        [
          13,
          23,
          109
        ]
      ],
      "directional_accuracy": 0.7874015748031497,
      "directional_hits": 200,
      "directional_n": 254,
      "hits": 233,
      "macro_f1": 0.7131701631701631,
      "n": 313,
      "sign_agreement": 0.8849557522123894,
      "sign_hits": 200,
      "sign_n": 226
    },
    "macro": {
      "accuracy": 0.5994358251057827,
      "confusion": [
        [
          136,
          35,
          27
        ],
        [
          22,
          237,
          48
        ],
        [
          53,
          99,
          52
        ]
      ],
      "directional_accuracy": 0.46766169154228854,
      "directional_hits": 188,
      "directional_n": 402,
      "hits": 425,
      "macro_f1": 0.559450371611602,
      "n": 709,
      "sign_agreement": 0.7014925373134329,
      "sign_hits": 188,
      "sign_n": 268
    }
  },
  "relevance_out_of_sample": {
    "direction_accuracy": 0.8871391076115486,
    "direction_n": 381,
    "f1": 0.7733026467203681,
    "n": 881,
    "precision": 0.6885245901639344,
    "recall": 0.8818897637795275
  },
  "rows": 2637,
  "threshold": 0.05
}