{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:HZPSVOBKEKBWNXZEWTZVAHYLCG","short_pith_number":"pith:HZPSVOBK","canonical_record":{"source":{"id":"2506.22419","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-06-27T17:44:32Z","cross_cats_sorted":["cs.CL","cs.LG"],"title_canon_sha256":"d233d36f14b129f0d2e95ddf77bd7fb244b9c0218b1a0cc066fdccb9a5b04675","abstract_canon_sha256":"b374aed84f49a1f733713d7084ce8de848227b132759896bc3170f6bef3b4e45"},"schema_version":"1.0"},"canonical_sha256":"3e5f2ab82a228366df24b4f3501f0b11b38b7c1bdd195d2bb744e8bef15d2b26","source":{"kind":"arxiv","id":"2506.22419","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.22419","created_at":"2026-07-05T11:30:02Z"},{"alias_kind":"arxiv_version","alias_value":"2506.22419v2","created_at":"2026-07-05T11:30:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.22419","created_at":"2026-07-05T11:30:02Z"},{"alias_kind":"pith_short_12","alias_value":"HZPSVOBKEKBW","created_at":"2026-07-05T11:30:02Z"},{"alias_kind":"pith_short_16","alias_value":"HZPSVOBKEKBWNXZE","created_at":"2026-07-05T11:30:02Z"},{"alias_kind":"pith_short_8","alias_value":"HZPSVOBK","created_at":"2026-07-05T11:30:02Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:HZPSVOBKEKBWNXZEWTZVAHYLCG","target":"record","payload":{"canonical_record":{"source":{"id":"2506.22419","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-06-27T17:44:32Z","cross_cats_sorted":["cs.CL","cs.LG"],"title_canon_sha256":"d233d36f14b129f0d2e95ddf77bd7fb244b9c0218b1a0cc066fdccb9a5b04675","abstract_canon_sha256":"b374aed84f49a1f733713d7084ce8de848227b132759896bc3170f6bef3b4e45"},"schema_version":"1.0"},"canonical_sha256":"3e5f2ab82a228366df24b4f3501f0b11b38b7c1bdd195d2bb744e8bef15d2b26","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:30:02.990981Z","signature_b64":"VVt0+UfXXuhTSLR5apobFBs9t/LkCZdiE8BdUYTpGquRwr4jzv6t55b53Sl56azqZcui0oAgQsWOT+E/NkUzBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"3e5f2ab82a228366df24b4f3501f0b11b38b7c1bdd195d2bb744e8bef15d2b26","last_reissued_at":"2026-07-05T11:30:02.990501Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:30:02.990501Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2506.22419","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:30:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QV+keqXQlYvXl1EUfYVRWnTMQUD0mgpvqHJElr93CocBauH4V8uCFSD7LM4/GolNTiy9wRXOoxQfBsFrZ0iBBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T18:03:37.448687Z"},"content_sha256":"8d2fc67125c7e758b6a1a22edcaeb298d14ff21b5c6e7b5abdb76a59a8ca88a4","schema_version":"1.0","event_id":"sha256:8d2fc67125c7e758b6a1a22edcaeb298d14ff21b5c6e7b5abdb76a59a8ca88a4"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:HZPSVOBKEKBWNXZEWTZVAHYLCG","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"The Automated LLM Speedrunning Benchmark: Reproducing NanoGPT Improvements","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL","cs.LG"],"primary_cat":"cs.AI","authors_text":"Abhishek Charnalia, Alexander H. Miller, Alisia Lupidi, Andrei Lupu, Bingchen Zhao, Derek Dunfield, Despoina Magka, Edan Toledo, Jakob Foerster, Jean-Christophe Gagnon-Audet, Karen Hambardzumyan, Kelvin Niu, Lucia Cipolina-Kun, Martin Josifoski, Michael Shvartsman, Minqi Jiang, Oisin Mac Aodha, Roberta Raileanu, Shagun Sodhani, Tatiana Shavrina, Thomas Foster, Xian Li, Yoram Bachrach","submitted_at":"2025-06-27T17:44:32Z","abstract_excerpt":"Rapid advancements in large language models (LLMs) have the potential to assist in scientific progress. A critical capability toward this endeavor is the ability to reproduce existing work. To evaluate the ability of AI agents to reproduce results in an active research area, we introduce the Automated LLM Speedrunning Benchmark, leveraging the research community contributions on the NanoGPT speedrun, a competition to train a GPT-2 model in the shortest time. Each of the 19 speedrun tasks provides the agent with the previous records training script, optionally paired with one of three hint form"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.22419","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2506.22419/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:30:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"5fXje9RI9ZAuuVBNm1+Gjep1NfU/3t8QF+SatBLINPeCTDiT5zR0164UYmcaYShI3fRVAZQ5+cFGRmzsJtdQCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T18:03:37.449619Z"},"content_sha256":"0f30707cd478975cfa51d38988ecb070b3d2c6e7465021959aca1faf863c04c3","schema_version":"1.0","event_id":"sha256:0f30707cd478975cfa51d38988ecb070b3d2c6e7465021959aca1faf863c04c3"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/HZPSVOBKEKBWNXZEWTZVAHYLCG/bundle.json","state_url":"https://pith.science/pith/HZPSVOBKEKBWNXZEWTZVAHYLCG/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/HZPSVOBKEKBWNXZEWTZVAHYLCG/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T18:03:37Z","links":{"resolver":"https://pith.science/pith/HZPSVOBKEKBWNXZEWTZVAHYLCG","bundle":"https://pith.science/pith/HZPSVOBKEKBWNXZEWTZVAHYLCG/bundle.json","state":"https://pith.science/pith/HZPSVOBKEKBWNXZEWTZVAHYLCG/state.json","well_known_bundle":"https://pith.science/.well-known/pith/HZPSVOBKEKBWNXZEWTZVAHYLCG/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:HZPSVOBKEKBWNXZEWTZVAHYLCG","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"b374aed84f49a1f733713d7084ce8de848227b132759896bc3170f6bef3b4e45","cross_cats_sorted":["cs.CL","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-06-27T17:44:32Z","title_canon_sha256":"d233d36f14b129f0d2e95ddf77bd7fb244b9c0218b1a0cc066fdccb9a5b04675"},"schema_version":"1.0","source":{"id":"2506.22419","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.22419","created_at":"2026-07-05T11:30:02Z"},{"alias_kind":"arxiv_version","alias_value":"2506.22419v2","created_at":"2026-07-05T11:30:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.22419","created_at":"2026-07-05T11:30:02Z"},{"alias_kind":"pith_short_12","alias_value":"HZPSVOBKEKBW","created_at":"2026-07-05T11:30:02Z"},{"alias_kind":"pith_short_16","alias_value":"HZPSVOBKEKBWNXZE","created_at":"2026-07-05T11:30:02Z"},{"alias_kind":"pith_short_8","alias_value":"HZPSVOBK","created_at":"2026-07-05T11:30:02Z"}],"graph_snapshots":[{"event_id":"sha256:0f30707cd478975cfa51d38988ecb070b3d2c6e7465021959aca1faf863c04c3","target":"graph","created_at":"2026-07-05T11:30:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2506.22419/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Rapid advancements in large language models (LLMs) have the potential to assist in scientific progress. A critical capability toward this endeavor is the ability to reproduce existing work. To evaluate the ability of AI agents to reproduce results in an active research area, we introduce the Automated LLM Speedrunning Benchmark, leveraging the research community contributions on the NanoGPT speedrun, a competition to train a GPT-2 model in the shortest time. Each of the 19 speedrun tasks provides the agent with the previous records training script, optionally paired with one of three hint form","authors_text":"Abhishek Charnalia, Alexander H. Miller, Alisia Lupidi, Andrei Lupu, Bingchen Zhao, Derek Dunfield, Despoina Magka, Edan Toledo, Jakob Foerster, Jean-Christophe Gagnon-Audet, Karen Hambardzumyan, Kelvin Niu, Lucia Cipolina-Kun, Martin Josifoski, Michael Shvartsman, Minqi Jiang, Oisin Mac Aodha, Roberta Raileanu, Shagun Sodhani, Tatiana Shavrina, Thomas Foster, Xian Li, Yoram Bachrach","cross_cats":["cs.CL","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-06-27T17:44:32Z","title":"The Automated LLM Speedrunning Benchmark: Reproducing NanoGPT Improvements"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.22419","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:8d2fc67125c7e758b6a1a22edcaeb298d14ff21b5c6e7b5abdb76a59a8ca88a4","target":"record","created_at":"2026-07-05T11:30:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"b374aed84f49a1f733713d7084ce8de848227b132759896bc3170f6bef3b4e45","cross_cats_sorted":["cs.CL","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-06-27T17:44:32Z","title_canon_sha256":"d233d36f14b129f0d2e95ddf77bd7fb244b9c0218b1a0cc066fdccb9a5b04675"},"schema_version":"1.0","source":{"id":"2506.22419","kind":"arxiv","version":2}},"canonical_sha256":"3e5f2ab82a228366df24b4f3501f0b11b38b7c1bdd195d2bb744e8bef15d2b26","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"3e5f2ab82a228366df24b4f3501f0b11b38b7c1bdd195d2bb744e8bef15d2b26","first_computed_at":"2026-07-05T11:30:02.990501Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:30:02.990501Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"VVt0+UfXXuhTSLR5apobFBs9t/LkCZdiE8BdUYTpGquRwr4jzv6t55b53Sl56azqZcui0oAgQsWOT+E/NkUzBA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:30:02.990981Z","signed_message":"canonical_sha256_bytes"},"source_id":"2506.22419","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:8d2fc67125c7e758b6a1a22edcaeb298d14ff21b5c6e7b5abdb76a59a8ca88a4","sha256:0f30707cd478975cfa51d38988ecb070b3d2c6e7465021959aca1faf863c04c3"],"state_sha256":"9a9190bd9e4e580b8fb607549946749f7002703ce19f50eba1b35106e630794e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"miVjRfoxiH4Cx9pSeWXRhQitN5h7RDUfZLjNjZ9iIpgqa6AXqCS7eM+X1mhbc4XyoXcKIwp4cU0H7A56Wd8CAQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T18:03:37.457551Z","bundle_sha256":"e83ba2bd0c5386d9b267fe67f13a2137dda5bd79a9d750af9d24aed7bcb09aa1"}}