| { |
| "complete": true, |
| "evaluation_complete": true, |
| "requested_tasks": [ |
| "agi_eval_lsat_ar", |
| "arc_challenge", |
| "arc_easy", |
| "bigbench_cs_algorithms", |
| "bigbench_dyck_languages", |
| "bigbench_language_identification", |
| "bigbench_operators", |
| "bigbench_qa_wikidata", |
| "bigbench_repeat_copy_logic", |
| "boolq", |
| "commonsense_qa", |
| "copa", |
| "coqa", |
| "hellaswag", |
| "hellaswag_zeroshot", |
| "jeopardy", |
| "lambada_openai", |
| "openbook_qa", |
| "piqa", |
| "squad", |
| "winograd", |
| "winogrande" |
| ], |
| "claim_status": "provisional_pending_prompt_review", |
| "scores": { |
| "hellaswag_zeroshot": 0.2410874366760254, |
| "jeopardy": 0.0, |
| "bigbench_qa_wikidata": 0.0, |
| "arc_easy": 0.27146464586257935, |
| "arc_challenge": 0.20989760756492615, |
| "copa": 0.4699999988079071, |
| "commonsense_qa": 0.1957411915063858, |
| "piqa": 0.5174102187156677, |
| "openbook_qa": 0.23399999737739563, |
| "lambada_openai": 0.0, |
| "hellaswag": 0.24158534407615662, |
| "winograd": 0.5274725556373596, |
| "winogrande": 0.519336998462677, |
| "bigbench_dyck_languages": 0.0, |
| "agi_eval_lsat_ar": 0.2347826063632965, |
| "bigbench_cs_algorithms": 0.0060606058686971664, |
| "bigbench_operators": 0.02857142873108387, |
| "bigbench_repeat_copy_logic": 0.0, |
| "squad": 0.0, |
| "coqa": 0.014155079610645771, |
| "boolq": 0.3782874643802643, |
| "bigbench_language_identification": 0.2500999867916107 |
| }, |
| "aggregate": { |
| "Core_v2": -0.04343042332051612, |
| "centered_tasks": { |
| "hellaswag_zeroshot": -0.011883417765299479, |
| "jeopardy": 0.0, |
| "bigbench_qa_wikidata": 0.0, |
| "arc_easy": 0.02861952781677246, |
| "arc_challenge": -0.05346985658009847, |
| "copa": -0.06000000238418579, |
| "commonsense_qa": -0.3471671834063889, |
| "piqa": 0.03482043743133545, |
| "openbook_qa": -0.02133333683013916, |
| "lambada_openai": 0.0, |
| "hellaswag": -0.011219541231791178, |
| "winograd": 0.05494511127471924, |
| "winogrande": 0.038673996925354004, |
| "bigbench_dyck_languages": 0.0, |
| "agi_eval_lsat_ar": -0.02028985818227132, |
| "bigbench_cs_algorithms": 0.0060606058686971664, |
| "bigbench_operators": 0.02857142873108387, |
| "bigbench_repeat_copy_logic": 0.0, |
| "squad": 0.0, |
| "coqa": 0.014155079610645771, |
| "boolq": -0.636085620051936, |
| "bigbench_language_identification": 0.0001333157221476237 |
| }, |
| "protocol_id": "dclm-core-v2-361714bd-context2048-proposed", |
| "required_tasks": 22 |
| }, |
| "lineage": { |
| "counters": { |
| "updates": 191, |
| "tokens": 50000000 |
| }, |
| "manifests": { |
| "recipe": "3fea67860037c274f5f1e5a5cb48349b4a5ef9345466084e8400bc398547826f", |
| "data": "0c6c0d7bba82974895e1439a84e3f7515d0fadb54d2a18c9d05beb49efe0d641", |
| "tokenizer": "d316298eef9ad7963ac948139d2284b35fc4e8c0a33776062f101a22b296a4fe", |
| "code": "39ef30aa46adb8f8284e49097e0b7ae253aee5d7c4a2211e03b8abb1f4431ab6", |
| "torch": "2.11.0+cu130" |
| }, |
| "schedule": { |
| "warmup_tokens": 5000000, |
| "peak_lr": 0.001, |
| "fork_tokens": null, |
| "end_tokens": null, |
| "floor_ratio": 0.05 |
| }, |
| "checkpoint_sha256": "5f8daa094efc741e6f3c73ae6a91a8a76c8630fe6cd610e86c737fe8cf37d24e", |
| "branch": null |
| }, |
| "prompt_audit_sha256": "6a895f22f85246bc077ae1ba0ebb13d35b8ae0c69002be136ffd01e538e5fef9", |
| "item_ledger_sha256": "cebc0106e9725cfa95d42184543ab2dae6419406bf8b2340d7012584d7e2fcf6", |
| "item_identity": "task, source file SHA256, zero-based source row, raw source row SHA256", |
| "protocol_sha256": "f420fd3c2ab47c852633f60824e1f33a77f24a49baa841c0ff20610238bba345", |
| "data_manifest_sha256": "0a750ef0ccb2e118cd467251faf07b9dc3d774662b9c1fdea807d9251c34c7a5", |
| "tokenizer_policy": "BOS postprocessor; native Foundry context/answer tokenization", |
| "precision": "fp32", |
| "tf32": false, |
| "elapsed_seconds": 22724.797448242083, |
| "peak_allocated_bytes": 3860806656, |
| "launch_manifest": { |
| "scope": "experimental full E01 evaluation with per-example outcomes; 50M training targets and incomplete training decontamination", |
| "source_snapshot": "/data/rfc005/full-core-e01-v1/src; byte-identical E01 source snapshot", |
| "source_sha256": { |
| "fabryka_track.english_base.icl": "c496f571bd78118f306479c62528c67d025c4ff1eca5594198388555230ee1b0", |
| "fabryka_track.english_base.reference": "a8f75bd0134858c4c4cb16d789bfba69316afe480d29960c55fa9fd3ecfac2e6", |
| "fabryka_track.english_base.prompt_audit": "7a891094a73bb9268c4d4bd2ce349d921917196447b57deae11ce2e351fc2af5", |
| "fabryka_track.english_base.scores": "fa4226f90415195f008b7188b3df535aded2dc4039d0b44335ca7a35aa47fd4d", |
| "fabryka_track.english_base.item_ledger": "352120820d0fd6333ad4897f966a57dda17e2b40742bdcf13aab9c7519e6e675" |
| }, |
| "protocol_sha256": "f420fd3c2ab47c852633f60824e1f33a77f24a49baa841c0ff20610238bba345", |
| "data_manifest_sha256": "0a750ef0ccb2e118cd467251faf07b9dc3d774662b9c1fdea807d9251c34c7a5", |
| "model_bundle_sha256": "ee046f48c84be5cad1af96958fc26792f6ebc2aad99605da13e4aedd87eebd90", |
| "harness_sha256": "74aebbcd97945ca64688c56e1e51cd5a6d595f3f791441df2afa00227b3be41a", |
| "precision": "fp32; TF32 disabled", |
| "gpu": "NVIDIA GeForce RTX 3090", |
| "torch": "2.3.1+cu121" |
| }, |
| "prompt_audit": { |
| "hellaswag_zeroshot": { |
| "trim_calls": 40168, |
| "truncated_calls": 0, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 0, |
| "partially_removed": 0, |
| "affected_continuations": 0 |
| }, |
| "status": "measured; requires protocol review" |
| }, |
| "jeopardy": { |
| "trim_calls": 2117, |
| "truncated_calls": 0, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 0, |
| "partially_removed": 0, |
| "affected_continuations": 0 |
| }, |
| "status": "measured; requires protocol review" |
| }, |
| "bigbench_qa_wikidata": { |
| "trim_calls": 20321, |
| "truncated_calls": 0, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 0, |
| "partially_removed": 0, |
| "affected_continuations": 0 |
| }, |
| "status": "measured; requires protocol review" |
| }, |
| "arc_easy": { |
| "trim_calls": 9504, |
| "truncated_calls": 0, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 0, |
| "partially_removed": 0, |
| "affected_continuations": 0 |
| }, |
| "status": "measured; requires protocol review" |
| }, |
| "arc_challenge": { |
| "trim_calls": 4688, |
| "truncated_calls": 0, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 0, |
| "partially_removed": 0, |
| "affected_continuations": 0 |
| }, |
| "status": "measured; requires protocol review" |
| }, |
| "copa": { |
| "trim_calls": 200, |
| "truncated_calls": 0, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 0, |
| "partially_removed": 0, |
| "affected_continuations": 0 |
| }, |
| "status": "measured; requires protocol review" |
| }, |
| "commonsense_qa": { |
| "trim_calls": 4884, |
| "truncated_calls": 0, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 0, |
| "partially_removed": 0, |
| "affected_continuations": 0 |
| }, |
| "status": "measured; requires protocol review" |
| }, |
| "piqa": { |
| "trim_calls": 3676, |
| "truncated_calls": 0, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 0, |
| "partially_removed": 0, |
| "affected_continuations": 0 |
| }, |
| "status": "measured; requires protocol review" |
| }, |
| "openbook_qa": { |
| "trim_calls": 2000, |
| "truncated_calls": 0, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 0, |
| "partially_removed": 0, |
| "affected_continuations": 0 |
| }, |
| "status": "measured; requires protocol review" |
| }, |
| "lambada_openai": { |
| "trim_calls": 5153, |
| "truncated_calls": 0, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 0, |
| "partially_removed": 0, |
| "affected_continuations": 0 |
| }, |
| "status": "measured; requires protocol review" |
| }, |
| "hellaswag": { |
| "trim_calls": 40168, |
| "truncated_calls": 0, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 0, |
| "partially_removed": 0, |
| "affected_continuations": 0 |
| }, |
| "status": "measured; requires protocol review" |
| }, |
| "winograd": { |
| "trim_calls": 546, |
| "truncated_calls": 0, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 0, |
| "partially_removed": 0, |
| "affected_continuations": 0 |
| }, |
| "status": "measured; requires protocol review" |
| }, |
| "winogrande": { |
| "trim_calls": 2534, |
| "truncated_calls": 0, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 0, |
| "partially_removed": 0, |
| "affected_continuations": 0 |
| }, |
| "status": "measured; requires protocol review" |
| }, |
| "bigbench_dyck_languages": { |
| "trim_calls": 1000, |
| "truncated_calls": 0, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 0, |
| "partially_removed": 0, |
| "affected_continuations": 0 |
| }, |
| "status": "measured; requires protocol review" |
| }, |
| "agi_eval_lsat_ar": { |
| "trim_calls": 920, |
| "truncated_calls": 0, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 0, |
| "partially_removed": 0, |
| "affected_continuations": 0 |
| }, |
| "status": "measured; requires protocol review" |
| }, |
| "bigbench_cs_algorithms": { |
| "trim_calls": 1320, |
| "truncated_calls": 0, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 0, |
| "partially_removed": 0, |
| "affected_continuations": 0 |
| }, |
| "status": "measured; requires protocol review" |
| }, |
| "bigbench_operators": { |
| "trim_calls": 210, |
| "truncated_calls": 0, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 0, |
| "partially_removed": 0, |
| "affected_continuations": 0 |
| }, |
| "status": "measured; requires protocol review" |
| }, |
| "bigbench_repeat_copy_logic": { |
| "trim_calls": 32, |
| "truncated_calls": 0, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 0, |
| "partially_removed": 0, |
| "affected_continuations": 0 |
| }, |
| "status": "measured; requires protocol review" |
| }, |
| "squad": { |
| "trim_calls": 10570, |
| "truncated_calls": 6902, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 5139, |
| "partially_removed": 6864, |
| "affected_continuations": 6889 |
| }, |
| "status": "measured; requires protocol review" |
| }, |
| "coqa": { |
| "trim_calls": 7983, |
| "truncated_calls": 0, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 0, |
| "partially_removed": 0, |
| "affected_continuations": 0 |
| }, |
| "status": "measured; requires protocol review" |
| }, |
| "boolq": { |
| "trim_calls": 6540, |
| "truncated_calls": 338, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 242, |
| "partially_removed": 332, |
| "affected_continuations": 334 |
| }, |
| "status": "measured; requires protocol review" |
| }, |
| "bigbench_language_identification": { |
| "trim_calls": 40000, |
| "truncated_calls": 7360, |
| "lost_demonstrations_per_continuation": { |
| "fully_removed": 12308, |
| "partially_removed": 7288, |
| "affected_continuations": 7332 |
| }, |
| "status": "measured; requires protocol review" |
| } |
| }, |
| "artifact": { |
| "local_path": "/root/data/rfc005/full-core-e01-v1/full-core.json", |
| "sha256": "090942c43a7cf5f3af376ac54f6bcd78bc99a1155fbe4c01d2935a236014b2d1", |
| "bytes": 6825705 |
| }, |
| "verification": { |
| "all_source_continuations_accounted_for": true, |
| "prompt_records": 204534, |
| "truncated_continuations": 14600, |
| "CORE_recomputed": true, |
| "scope": "artifact integrity, original source identities, complete native item coverage, category-weighted scores, prompt audit and CORE arithmetic; experimental reference, not candidate quality or contamination proof", |
| "original_item_identities_verified": true, |
| "item_scores_recomputed": true, |
| "item_records": 91037 |
| } |
| } |
|
|
