{
  "complete": true,
  "evaluation_complete": true,
  "requested_tasks": [
    "agi_eval_lsat_ar",
    "arc_challenge",
    "arc_easy",
    "bigbench_cs_algorithms",
    "bigbench_dyck_languages",
    "bigbench_language_identification",
    "bigbench_operators",
    "bigbench_qa_wikidata",
    "bigbench_repeat_copy_logic",
    "boolq",
    "commonsense_qa",
    "copa",
    "coqa",
    "hellaswag",
    "hellaswag_zeroshot",
    "jeopardy",
    "lambada_openai",
    "openbook_qa",
    "piqa",
    "squad",
    "winograd",
    "winogrande"
  ],
  "claim_status": "provisional_pending_prompt_review",
  "scores": {
    "hellaswag_zeroshot": 0.2410874366760254,
    "jeopardy": 0.0,
    "bigbench_qa_wikidata": 0.0,
    "arc_easy": 0.27146464586257935,
    "arc_challenge": 0.20989760756492615,
    "copa": 0.4699999988079071,
    "commonsense_qa": 0.1957411915063858,
    "piqa": 0.5174102187156677,
    "openbook_qa": 0.23399999737739563,
    "lambada_openai": 0.0,
    "hellaswag": 0.24158534407615662,
    "winograd": 0.5274725556373596,
    "winogrande": 0.519336998462677,
    "bigbench_dyck_languages": 0.0,
    "agi_eval_lsat_ar": 0.2347826063632965,
    "bigbench_cs_algorithms": 0.0060606058686971664,
    "bigbench_operators": 0.02857142873108387,
    "bigbench_repeat_copy_logic": 0.0,
    "squad": 0.0,
    "coqa": 0.014155079610645771,
    "boolq": 0.3782874643802643,
    "bigbench_language_identification": 0.2500999867916107
  },
  "aggregate": {
    "Core_v2": -0.04343042332051612,
    "centered_tasks": {
      "hellaswag_zeroshot": -0.011883417765299479,
      "jeopardy": 0.0,
      "bigbench_qa_wikidata": 0.0,
      "arc_easy": 0.02861952781677246,
      "arc_challenge": -0.05346985658009847,
      "copa": -0.06000000238418579,
      "commonsense_qa": -0.3471671834063889,
      "piqa": 0.03482043743133545,
      "openbook_qa": -0.02133333683013916,
      "lambada_openai": 0.0,
      "hellaswag": -0.011219541231791178,
      "winograd": 0.05494511127471924,
      "winogrande": 0.038673996925354004,
      "bigbench_dyck_languages": 0.0,
      "agi_eval_lsat_ar": -0.02028985818227132,
      "bigbench_cs_algorithms": 0.0060606058686971664,
      "bigbench_operators": 0.02857142873108387,
      "bigbench_repeat_copy_logic": 0.0,
      "squad": 0.0,
      "coqa": 0.014155079610645771,
      "boolq": -0.636085620051936,
      "bigbench_language_identification": 0.0001333157221476237
    },
    "protocol_id": "dclm-core-v2-361714bd-context2048-proposed",
    "required_tasks": 22
  },
  "lineage": {
    "counters": {
      "updates": 191,
      "tokens": 50000000
    },
    "manifests": {
      "recipe": "3fea67860037c274f5f1e5a5cb48349b4a5ef9345466084e8400bc398547826f",
      "data": "0c6c0d7bba82974895e1439a84e3f7515d0fadb54d2a18c9d05beb49efe0d641",
      "tokenizer": "d316298eef9ad7963ac948139d2284b35fc4e8c0a33776062f101a22b296a4fe",
      "code": "39ef30aa46adb8f8284e49097e0b7ae253aee5d7c4a2211e03b8abb1f4431ab6",
      "torch": "2.11.0+cu130"
    },
    "schedule": {
      "warmup_tokens": 5000000,
      "peak_lr": 0.001,
      "fork_tokens": null,
      "end_tokens": null,
      "floor_ratio": 0.05
    },
    "checkpoint_sha256": "5f8daa094efc741e6f3c73ae6a91a8a76c8630fe6cd610e86c737fe8cf37d24e",
    "branch": null
  },
  "prompt_audit_sha256": "6a895f22f85246bc077ae1ba0ebb13d35b8ae0c69002be136ffd01e538e5fef9",
  "item_ledger_sha256": "cebc0106e9725cfa95d42184543ab2dae6419406bf8b2340d7012584d7e2fcf6",
  "item_identity": "task, source file SHA256, zero-based source row, raw source row SHA256",
  "protocol_sha256": "f420fd3c2ab47c852633f60824e1f33a77f24a49baa841c0ff20610238bba345",
  "data_manifest_sha256": "0a750ef0ccb2e118cd467251faf07b9dc3d774662b9c1fdea807d9251c34c7a5",
  "tokenizer_policy": "BOS postprocessor; native Foundry context/answer tokenization",
  "precision": "fp32",
  "tf32": false,
  "elapsed_seconds": 22724.797448242083,
  "peak_allocated_bytes": 3860806656,
  "launch_manifest": {
    "scope": "experimental full E01 evaluation with per-example outcomes; 50M training targets and incomplete training decontamination",
    "source_snapshot": "/data/rfc005/full-core-e01-v1/src; byte-identical E01 source snapshot",
    "source_sha256": {
      "fabryka_track.english_base.icl": "c496f571bd78118f306479c62528c67d025c4ff1eca5594198388555230ee1b0",
      "fabryka_track.english_base.reference": "a8f75bd0134858c4c4cb16d789bfba69316afe480d29960c55fa9fd3ecfac2e6",
      "fabryka_track.english_base.prompt_audit": "7a891094a73bb9268c4d4bd2ce349d921917196447b57deae11ce2e351fc2af5",
      "fabryka_track.english_base.scores": "fa4226f90415195f008b7188b3df535aded2dc4039d0b44335ca7a35aa47fd4d",
      "fabryka_track.english_base.item_ledger": "352120820d0fd6333ad4897f966a57dda17e2b40742bdcf13aab9c7519e6e675"
    },
    "protocol_sha256": "f420fd3c2ab47c852633f60824e1f33a77f24a49baa841c0ff20610238bba345",
    "data_manifest_sha256": "0a750ef0ccb2e118cd467251faf07b9dc3d774662b9c1fdea807d9251c34c7a5",
    "model_bundle_sha256": "ee046f48c84be5cad1af96958fc26792f6ebc2aad99605da13e4aedd87eebd90",
    "harness_sha256": "74aebbcd97945ca64688c56e1e51cd5a6d595f3f791441df2afa00227b3be41a",
    "precision": "fp32; TF32 disabled",
    "gpu": "NVIDIA GeForce RTX 3090",
    "torch": "2.3.1+cu121"
  },
  "prompt_audit": {
    "hellaswag_zeroshot": {
      "trim_calls": 40168,
      "truncated_calls": 0,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 0,
        "partially_removed": 0,
        "affected_continuations": 0
      },
      "status": "measured; requires protocol review"
    },
    "jeopardy": {
      "trim_calls": 2117,
      "truncated_calls": 0,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 0,
        "partially_removed": 0,
        "affected_continuations": 0
      },
      "status": "measured; requires protocol review"
    },
    "bigbench_qa_wikidata": {
      "trim_calls": 20321,
      "truncated_calls": 0,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 0,
        "partially_removed": 0,
        "affected_continuations": 0
      },
      "status": "measured; requires protocol review"
    },
    "arc_easy": {
      "trim_calls": 9504,
      "truncated_calls": 0,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 0,
        "partially_removed": 0,
        "affected_continuations": 0
      },
      "status": "measured; requires protocol review"
    },
    "arc_challenge": {
      "trim_calls": 4688,
      "truncated_calls": 0,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 0,
        "partially_removed": 0,
        "affected_continuations": 0
      },
      "status": "measured; requires protocol review"
    },
    "copa": {
      "trim_calls": 200,
      "truncated_calls": 0,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 0,
        "partially_removed": 0,
        "affected_continuations": 0
      },
      "status": "measured; requires protocol review"
    },
    "commonsense_qa": {
      "trim_calls": 4884,
      "truncated_calls": 0,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 0,
        "partially_removed": 0,
        "affected_continuations": 0
      },
      "status": "measured; requires protocol review"
    },
    "piqa": {
      "trim_calls": 3676,
      "truncated_calls": 0,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 0,
        "partially_removed": 0,
        "affected_continuations": 0
      },
      "status": "measured; requires protocol review"
    },
    "openbook_qa": {
      "trim_calls": 2000,
      "truncated_calls": 0,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 0,
        "partially_removed": 0,
        "affected_continuations": 0
      },
      "status": "measured; requires protocol review"
    },
    "lambada_openai": {
      "trim_calls": 5153,
      "truncated_calls": 0,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 0,
        "partially_removed": 0,
        "affected_continuations": 0
      },
      "status": "measured; requires protocol review"
    },
    "hellaswag": {
      "trim_calls": 40168,
      "truncated_calls": 0,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 0,
        "partially_removed": 0,
        "affected_continuations": 0
      },
      "status": "measured; requires protocol review"
    },
    "winograd": {
      "trim_calls": 546,
      "truncated_calls": 0,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 0,
        "partially_removed": 0,
        "affected_continuations": 0
      },
      "status": "measured; requires protocol review"
    },
    "winogrande": {
      "trim_calls": 2534,
      "truncated_calls": 0,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 0,
        "partially_removed": 0,
        "affected_continuations": 0
      },
      "status": "measured; requires protocol review"
    },
    "bigbench_dyck_languages": {
      "trim_calls": 1000,
      "truncated_calls": 0,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 0,
        "partially_removed": 0,
        "affected_continuations": 0
      },
      "status": "measured; requires protocol review"
    },
    "agi_eval_lsat_ar": {
      "trim_calls": 920,
      "truncated_calls": 0,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 0,
        "partially_removed": 0,
        "affected_continuations": 0
      },
      "status": "measured; requires protocol review"
    },
    "bigbench_cs_algorithms": {
      "trim_calls": 1320,
      "truncated_calls": 0,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 0,
        "partially_removed": 0,
        "affected_continuations": 0
      },
      "status": "measured; requires protocol review"
    },
    "bigbench_operators": {
      "trim_calls": 210,
      "truncated_calls": 0,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 0,
        "partially_removed": 0,
        "affected_continuations": 0
      },
      "status": "measured; requires protocol review"
    },
    "bigbench_repeat_copy_logic": {
      "trim_calls": 32,
      "truncated_calls": 0,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 0,
        "partially_removed": 0,
        "affected_continuations": 0
      },
      "status": "measured; requires protocol review"
    },
    "squad": {
      "trim_calls": 10570,
      "truncated_calls": 6902,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 5139,
        "partially_removed": 6864,
        "affected_continuations": 6889
      },
      "status": "measured; requires protocol review"
    },
    "coqa": {
      "trim_calls": 7983,
      "truncated_calls": 0,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 0,
        "partially_removed": 0,
        "affected_continuations": 0
      },
      "status": "measured; requires protocol review"
    },
    "boolq": {
      "trim_calls": 6540,
      "truncated_calls": 338,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 242,
        "partially_removed": 332,
        "affected_continuations": 334
      },
      "status": "measured; requires protocol review"
    },
    "bigbench_language_identification": {
      "trim_calls": 40000,
      "truncated_calls": 7360,
      "lost_demonstrations_per_continuation": {
        "fully_removed": 12308,
        "partially_removed": 7288,
        "affected_continuations": 7332
      },
      "status": "measured; requires protocol review"
    }
  },
  "artifact": {
    "local_path": "/root/data/rfc005/full-core-e01-v1/full-core.json",
    "sha256": "090942c43a7cf5f3af376ac54f6bcd78bc99a1155fbe4c01d2935a236014b2d1",
    "bytes": 6825705
  },
  "verification": {
    "all_source_continuations_accounted_for": true,
    "prompt_records": 204534,
    "truncated_continuations": 14600,
    "CORE_recomputed": true,
    "scope": "artifact integrity, original source identities, complete native item coverage, category-weighted scores, prompt audit and CORE arithmetic; experimental reference, not candidate quality or contamination proof",
    "original_item_identities_verified": true,
    "item_scores_recomputed": true,
    "item_records": 91037
  }
}