{
  "agreements": 146769,
  "cohen_kappa": 0.9781300729494184,
  "confusion": {
    "environment": {
      "environment": 17757,
      "goal": 0,
      "integration": 0,
      "memory": 0,
      "normal": 209,
      "reasoning": 2,
      "reward": 0
    },
    "goal": {
      "environment": 8,
      "goal": 26657,
      "integration": 0,
      "memory": 0,
      "normal": 289,
      "reasoning": 3,
      "reward": 0
    },
    "integration": {
      "environment": 2,
      "goal": 0,
      "integration": 21721,
      "memory": 0,
      "normal": 684,
      "reasoning": 4,
      "reward": 0
    },
    "memory": {
      "environment": 0,
      "goal": 0,
      "integration": 0,
      "memory": 14946,
      "normal": 45,
      "reasoning": 0,
      "reward": 0
    },
    "normal": {
      "environment": 37,
      "goal": 1,
      "integration": 7,
      "memory": 110,
      "normal": 14702,
      "reasoning": 137,
      "reward": 0
    },
    "reasoning": {
      "environment": 16,
      "goal": 0,
      "integration": 0,
      "memory": 0,
      "normal": 1072,
      "reasoning": 36147,
      "reward": 0
    },
    "reward": {
      "environment": 3,
      "goal": 0,
      "integration": 0,
      "memory": 0,
      "normal": 116,
      "reasoning": 1,
      "reward": 14839
    }
  },
  "disagreements": 2746,
  "distribution_audit": {
    "environment": {
      "deviation_percentage_points": 0.0014135783535813573,
      "observed_percent": 12.001413578353581,
      "planned_percent": 12.0,
      "rows": 17999
    },
    "goal": {
      "deviation_percentage_points": 0.0011201941669902737,
      "observed_percent": 18.00112019416699,
      "planned_percent": 18.0,
      "rows": 26997
    },
    "integration": {
      "deviation_percentage_points": 0.002600450744795424,
      "observed_percent": 15.002600450744795,
      "planned_percent": 15.0,
      "rows": 22500
    },
    "memory": {
      "deviation_percentage_points": 0.0017336338298630238,
      "observed_percent": 10.001733633829863,
      "planned_percent": 10.0,
      "rows": 15000
    },
    "normal": {
      "deviation_percentage_points": 0.0017336338298630238,
      "observed_percent": 10.001733633829863,
      "planned_percent": 10.0,
      "rows": 15000
    },
    "reasoning": {
      "deviation_percentage_points": -0.009001560270448294,
      "observed_percent": 24.99099843972955,
      "planned_percent": 25.0,
      "rows": 37480
    },
    "reward": {
      "deviation_percentage_points": 0.00040006934535341543,
      "observed_percent": 10.000400069345353,
      "planned_percent": 10.0,
      "rows": 14998
    }
  },
  "exclusion_counts": {
    "missing_or_invalid_raw_teacher_tier": 459
  },
  "expected_agreement": 0.16021438084377937,
  "inputs": [
    "semantic_150k/train.jsonl",
    "semantic_150k/val.jsonl",
    "semantic_150k/test.jsonl"
  ],
  "labels": [
    "goal",
    "reasoning",
    "environment",
    "integration",
    "memory",
    "reward",
    "normal"
  ],
  "maximum_absolute_distribution_deviation_percentage_points": 0.009001560270448294,
  "method": "Cohen's kappa between the independently assigned target_tier and the tier in metadata.raw_teacher_text.label, before any target-tier overwrite",
  "observed_agreement": 0.981633949770926,
  "planned_distribution_percent": {
    "environment": 12.0,
    "goal": 18.0,
    "integration": 15.0,
    "memory": 10.0,
    "normal": 10.0,
    "reasoning": 25.0,
    "reward": 10.0
  },
  "raw_teacher_counts_valid_rows": {
    "environment": 17823,
    "goal": 26658,
    "integration": 21728,
    "memory": 15056,
    "normal": 17117,
    "reasoning": 36294,
    "reward": 14839
  },
  "rows_excluded_from_kappa": 459,
  "rows_total": 149974,
  "rows_valid_for_kappa": 149515,
  "schema_version": "judge6.label-consistency-audit.v1",
  "target_counts_all_rows": {
    "environment": 17999,
    "goal": 26997,
    "integration": 22500,
    "memory": 15000,
    "normal": 15000,
    "reasoning": 37480,
    "reward": 14998
  },
  "target_counts_valid_rows": {
    "environment": 17968,
    "goal": 26957,
    "integration": 22411,
    "memory": 14991,
    "normal": 14994,
    "reasoning": 37235,
    "reward": 14959
  },
  "redaction": {
    "removed": [
      "absolute filesystem paths"
    ],
    "internal_evidence_retained": true
  }
}