| { |
| "agreements": 146769, |
| "cohen_kappa": 0.9781300729494184, |
| "confusion": { |
| "environment": { |
| "environment": 17757, |
| "goal": 0, |
| "integration": 0, |
| "memory": 0, |
| "normal": 209, |
| "reasoning": 2, |
| "reward": 0 |
| }, |
| "goal": { |
| "environment": 8, |
| "goal": 26657, |
| "integration": 0, |
| "memory": 0, |
| "normal": 289, |
| "reasoning": 3, |
| "reward": 0 |
| }, |
| "integration": { |
| "environment": 2, |
| "goal": 0, |
| "integration": 21721, |
| "memory": 0, |
| "normal": 684, |
| "reasoning": 4, |
| "reward": 0 |
| }, |
| "memory": { |
| "environment": 0, |
| "goal": 0, |
| "integration": 0, |
| "memory": 14946, |
| "normal": 45, |
| "reasoning": 0, |
| "reward": 0 |
| }, |
| "normal": { |
| "environment": 37, |
| "goal": 1, |
| "integration": 7, |
| "memory": 110, |
| "normal": 14702, |
| "reasoning": 137, |
| "reward": 0 |
| }, |
| "reasoning": { |
| "environment": 16, |
| "goal": 0, |
| "integration": 0, |
| "memory": 0, |
| "normal": 1072, |
| "reasoning": 36147, |
| "reward": 0 |
| }, |
| "reward": { |
| "environment": 3, |
| "goal": 0, |
| "integration": 0, |
| "memory": 0, |
| "normal": 116, |
| "reasoning": 1, |
| "reward": 14839 |
| } |
| }, |
| "disagreements": 2746, |
| "distribution_audit": { |
| "environment": { |
| "deviation_percentage_points": 0.0014135783535813573, |
| "observed_percent": 12.001413578353581, |
| "planned_percent": 12.0, |
| "rows": 17999 |
| }, |
| "goal": { |
| "deviation_percentage_points": 0.0011201941669902737, |
| "observed_percent": 18.00112019416699, |
| "planned_percent": 18.0, |
| "rows": 26997 |
| }, |
| "integration": { |
| "deviation_percentage_points": 0.002600450744795424, |
| "observed_percent": 15.002600450744795, |
| "planned_percent": 15.0, |
| "rows": 22500 |
| }, |
| "memory": { |
| "deviation_percentage_points": 0.0017336338298630238, |
| "observed_percent": 10.001733633829863, |
| "planned_percent": 10.0, |
| "rows": 15000 |
| }, |
| "normal": { |
| "deviation_percentage_points": 0.0017336338298630238, |
| "observed_percent": 10.001733633829863, |
| "planned_percent": 10.0, |
| "rows": 15000 |
| }, |
| "reasoning": { |
| "deviation_percentage_points": -0.009001560270448294, |
| "observed_percent": 24.99099843972955, |
| "planned_percent": 25.0, |
| "rows": 37480 |
| }, |
| "reward": { |
| "deviation_percentage_points": 0.00040006934535341543, |
| "observed_percent": 10.000400069345353, |
| "planned_percent": 10.0, |
| "rows": 14998 |
| } |
| }, |
| "exclusion_counts": { |
| "missing_or_invalid_raw_teacher_tier": 459 |
| }, |
| "expected_agreement": 0.16021438084377937, |
| "inputs": [ |
| "semantic_150k/train.jsonl", |
| "semantic_150k/val.jsonl", |
| "semantic_150k/test.jsonl" |
| ], |
| "labels": [ |
| "goal", |
| "reasoning", |
| "environment", |
| "integration", |
| "memory", |
| "reward", |
| "normal" |
| ], |
| "maximum_absolute_distribution_deviation_percentage_points": 0.009001560270448294, |
| "method": "Cohen's kappa between the independently assigned target_tier and the tier in metadata.raw_teacher_text.label, before any target-tier overwrite", |
| "observed_agreement": 0.981633949770926, |
| "planned_distribution_percent": { |
| "environment": 12.0, |
| "goal": 18.0, |
| "integration": 15.0, |
| "memory": 10.0, |
| "normal": 10.0, |
| "reasoning": 25.0, |
| "reward": 10.0 |
| }, |
| "raw_teacher_counts_valid_rows": { |
| "environment": 17823, |
| "goal": 26658, |
| "integration": 21728, |
| "memory": 15056, |
| "normal": 17117, |
| "reasoning": 36294, |
| "reward": 14839 |
| }, |
| "rows_excluded_from_kappa": 459, |
| "rows_total": 149974, |
| "rows_valid_for_kappa": 149515, |
| "schema_version": "judge6.label-consistency-audit.v1", |
| "target_counts_all_rows": { |
| "environment": 17999, |
| "goal": 26997, |
| "integration": 22500, |
| "memory": 15000, |
| "normal": 15000, |
| "reasoning": 37480, |
| "reward": 14998 |
| }, |
| "target_counts_valid_rows": { |
| "environment": 17968, |
| "goal": 26957, |
| "integration": 22411, |
| "memory": 14991, |
| "normal": 14994, |
| "reasoning": 37235, |
| "reward": 14959 |
| }, |
| "redaction": { |
| "removed": [ |
| "absolute filesystem paths" |
| ], |
| "internal_evidence_retained": true |
| } |
| } |
|
|
