{
  "deepseek_self_agreement": {
    "accuracy": 0.7,
    "confusion": [
      [
        4,
        0,
        1
      ],
      [
        0,
        0,
        0
      ],
      [
        1,
        1,
        3
      ]
    ],
    "directional_accuracy": 0.7,
    "directional_hits": 7,
    "directional_n": 10,
    "hits": 7,
    "identical": 2,
    "macro_f1": 0.48888888888888893,
    "mae": 0.25,
    "n": 10,
    "sign_agreement": 0.7777777777777778,
    "sign_hits": 7,
    "sign_n": 9,
    "spearman": 0.5569257132640542
  },
  "generated_at": "20260917_2241",
  "latency": {
    "deepseek_ms_per_headline": 56.414902748947796,
    "model_ms_mean": 1.0686842105263157
  },
  "log_rows": 76,
  "n": 76,
  "subsets": {
    "all headlines": {
      "keyword": {
        "accuracy": 0.3815789473684211,
        "confusion": [
          [
            9,
            17,
            0
          ],
          [
            0,
            16,
            0
          ],
          [
            2,
            28,
            4
          ]
        ],
        "directional_accuracy": 0.21666666666666667,
        "directional_hits": 13,
        "directional_n": 60,
        "hits": 29,
        "macro_f1": 0.37086573928679195,
        "n": 76,
        "sign_agreement": 0.8666666666666667,
        "sign_hits": 13,
        "sign_n": 15
      },
      "model_full": {
        "accuracy": 0.6052631578947368,
        "confusion": [
          [
            16,
            3,
            7
          ],
          [
            5,
            6,
            5
          ],
          [
            7,
            3,
            24
          ]
        ],
        "directional_accuracy": 0.6666666666666666,
        "directional_hits": 40,
        "directional_n": 60,
        "hits": 46,
        "macro_f1": 0.5689594356261023,
        "mae": 0.33246710526315787,
        "n": 76,
        "sign_agreement": 0.7407407407407407,
        "sign_hits": 40,
        "sign_n": 54,
        "spearman": 0.4808996156923549
      },
      "model_head": {
        "accuracy": 0.6842105263157895,
        "confusion": [
          [
            20,
            0,
            6
          ],
          [
            2,
            6,
            8
          ],
          [
            6,
            2,
            26
          ]
        ],
        "directional_accuracy": 0.7666666666666667,
        "directional_hits": 46,
        "directional_n": 60,
        "hits": 52,
        "macro_f1": 0.6478144811478145,
        "mae": 0.3404934210526316,
        "n": 76,
        "sign_agreement": 0.7931034482758621,
        "sign_hits": 46,
        "sign_n": 58,
        "spearman": 0.5639368037865334
      },
      "n": 76
    },
    "mentions gold (XAU keyword)": {
      "keyword": {
        "accuracy": 0.3404255319148936,
        "confusion": [
          [
            5,
            6,
            0
          ],
          [
            0,
            9,
            0
          ],
          [
            0,
            25,
            2
          ]
        ],
        "directional_accuracy": 0.18421052631578946,
        "directional_hits": 7,
        "directional_n": 38,
        "hits": 16,
        "macro_f1": 0.37675932441942295,
        "n": 47,
        "sign_agreement": 1.0,
        "sign_hits": 7,
        "sign_n": 7
      },
      "model_full": {
        "accuracy": 0.723404255319149,
        "confusion": [
          [
            10,
            0,
            1
          ],
          [
            2,
            4,
            3
          ],
          [
            4,
            3,
            20
          ]
        ],
        "directional_accuracy": 0.7894736842105263,
        "directional_hits": 30,
        "directional_n": 38,
        "hits": 34,
        "macro_f1": 0.6750181554103122,
        "mae": 0.2999148936170213,
        "n": 47,
        "sign_agreement": 0.8571428571428571,
        "sign_hits": 30,
        "sign_n": 35,
        "spearman": 0.5816073836387256
      },
      "model_head": {
        "accuracy": 0.7659574468085106,
        "confusion": [
          [
            10,
            0,
            1
          ],
          [
            2,
            4,
            3
          ],
          [
            3,
            2,
            22
          ]
        ],
        "directional_accuracy": 0.8421052631578947,
        "directional_hits": 32,
        "directional_n": 38,
        "hits": 36,
        "macro_f1": 0.7109175939364617,
        "mae": 0.302268085106383,
        "n": 47,
        "sign_agreement": 0.8888888888888888,
        "sign_hits": 32,
        "sign_n": 36,
        "spearman": 0.6912522895796331
      },
      "n": 47
    },
    "not in training & mentions gold": {
      "keyword": {
        "accuracy": 0.35135135135135137,
        "confusion": [
          [
            3,
            3,
            0
          ],
          [
            0,
            8,
            0
          ],
          [
            0,
            21,
            2
          ]
        ],
        "directional_accuracy": 0.1724137931034483,
        "directional_hits": 5,
        "directional_n": 29,
        "hits": 13,
        "macro_f1": 0.40888888888888886,
        "n": 37,
        "sign_agreement": 1.0,
        "sign_hits": 5,
        "sign_n": 5
      },
      "model_full": {
        "accuracy": 0.7027027027027027,
        "confusion": [
          [
            6,
            0,
            0
          ],
          [
            2,
            4,
            2
          ],
          [
            4,
            3,
            16
          ]
        ],
        "directional_accuracy": 0.7586206896551724,
        "directional_hits": 22,
        "directional_n": 29,
        "hits": 26,
        "macro_f1": 0.6601626016260163,
        "mae": 0.25864864864864867,
        "n": 37,
        "sign_agreement": 0.8461538461538461,
        "sign_hits": 22,
        "sign_n": 26,
        "spearman": 0.5732166454312866
      },
      "model_head": {
        "accuracy": 0.7567567567567568,
        "confusion": [
          [
            6,
            0,
            0
          ],
          [
            2,
            4,
            2
          ],
          [
            3,
            2,
            18
          ]
        ],
        "directional_accuracy": 0.8275862068965517,
        "directional_hits": 24,
        "directional_n": 29,
        "hits": 28,
        "macro_f1": 0.7048400755651097,
        "mae": 0.2563864864864865,
        "n": 37,
        "sign_agreement": 0.8888888888888888,
        "sign_hits": 24,
        "sign_n": 27,
        "spearman": 0.6927913852069925
      },
      "n": 37
    },
    "not in training set": {
      "keyword": {
        "accuracy": 0.3939393939393939,
        "confusion": [
          [
            7,
            14,
            0
          ],
          [
            0,
            15,
            0
          ],
          [
            2,
            24,
            4
          ]
        ],
        "directional_accuracy": 0.21568627450980393,
        "directional_hits": 11,
        "directional_n": 51,
        "hits": 26,
        "macro_f1": 0.3810457516339869,
        "n": 66,
        "sign_agreement": 0.8461538461538461,
        "sign_hits": 11,
        "sign_n": 13
      },
      "model_full": {
        "accuracy": 0.5757575757575758,
        "confusion": [
          [
            12,
            3,
            6
          ],
          [
            5,
            6,
            4
          ],
          [
            7,
            3,
            20
          ]
        ],
        "directional_accuracy": 0.6274509803921569,
        "directional_hits": 32,
        "directional_n": 51,
        "hits": 38,
        "macro_f1": 0.5481481481481482,
        "mae": 0.3142651515151515,
        "n": 66,
        "sign_agreement": 0.7111111111111111,
        "sign_hits": 32,
        "sign_n": 45,
        "spearman": 0.4561157459616647
      },
      "model_head": {
        "accuracy": 0.6666666666666666,
        "confusion": [
          [
            16,
            0,
            5
          ],
          [
            2,
            6,
            7
          ],
          [
            6,
            2,
            22
          ]
        ],
        "directional_accuracy": 0.7450980392156863,
        "directional_hits": 38,
        "directional_n": 51,
        "hits": 44,
        "macro_f1": 0.6401167471819647,
        "mae": 0.3205636363636364,
        "n": 66,
        "sign_agreement": 0.7755102040816326,
        "sign_hits": 38,
        "sign_n": 49,
        "spearman": 0.5415716725239331
      },
      "n": 66
    }
  },
  "threshold": 0.05,
  "usd_only_display_vs_xau": {
    "accuracy": 0.875,
    "confusion": [
      [
        9,
        0,
        0
      ],
      [
        0,
        3,
        0
      ],
      [
        2,
        0,
        2
      ]
    ],
    "directional_accuracy": 0.8461538461538461,
    "directional_hits": 11,
    "directional_n": 13,
    "hits": 14,
    "macro_f1": 0.8555555555555555,
    "n": 16,
    "sign_agreement": 0.8461538461538461,
    "sign_hits": 11,
    "sign_n": 13
  }
}