{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:BIUOSSU6LXXHK6N4ZWA4ZGOCGY","short_pith_number":"pith:BIUOSSU6","canonical_record":{"source":{"id":"2506.19724","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-06-24T15:39:20Z","cross_cats_sorted":[],"title_canon_sha256":"6b1f211ad47032c6f00ff532e6cbdc9e347106c4949d49379a3d4abc4db3bac7","abstract_canon_sha256":"9f207c3b96bf3f60aa733bc4f304b0aafaadd069477dc98a9e8821df9e8bfa19"},"schema_version":"1.0"},"canonical_sha256":"0a28e94a9e5dee7579bccd81cc99c2363d650253b6109488e3a333d9cfc4896f","source":{"kind":"arxiv","id":"2506.19724","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.19724","created_at":"2026-07-05T11:26:37Z"},{"alias_kind":"arxiv_version","alias_value":"2506.19724v1","created_at":"2026-07-05T11:26:37Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.19724","created_at":"2026-07-05T11:26:37Z"},{"alias_kind":"pith_short_12","alias_value":"BIUOSSU6LXXH","created_at":"2026-07-05T11:26:37Z"},{"alias_kind":"pith_short_16","alias_value":"BIUOSSU6LXXHK6N4","created_at":"2026-07-05T11:26:37Z"},{"alias_kind":"pith_short_8","alias_value":"BIUOSSU6","created_at":"2026-07-05T11:26:37Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:BIUOSSU6LXXHK6N4ZWA4ZGOCGY","target":"record","payload":{"canonical_record":{"source":{"id":"2506.19724","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-06-24T15:39:20Z","cross_cats_sorted":[],"title_canon_sha256":"6b1f211ad47032c6f00ff532e6cbdc9e347106c4949d49379a3d4abc4db3bac7","abstract_canon_sha256":"9f207c3b96bf3f60aa733bc4f304b0aafaadd069477dc98a9e8821df9e8bfa19"},"schema_version":"1.0"},"canonical_sha256":"0a28e94a9e5dee7579bccd81cc99c2363d650253b6109488e3a333d9cfc4896f","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:26:37.693137Z","signature_b64":"6hM1cmgaZrTbeLeMWrjW+LDuSSKTAG8gFW/xXn80B52jGOoJy/aPKU+9Ou9HM+3LhAMS8mGikQLNPCKl+T9KDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0a28e94a9e5dee7579bccd81cc99c2363d650253b6109488e3a333d9cfc4896f","last_reissued_at":"2026-07-05T11:26:37.692536Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:26:37.692536Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2506.19724","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:26:37Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"PhNR03Ahu4RxBI+EwS6hhAULfWY9bYFDazOl3wGujtx802iksdUA8zYTLTk2bKfhPkhwXqdHP5bB2Q3tJCgDCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-20T20:48:59.665280Z"},"content_sha256":"b7855a91aab4497314542a184eb2b8d633a64b9ce93910aa6ac44c61ebf37a5e","schema_version":"1.0","event_id":"sha256:b7855a91aab4497314542a184eb2b8d633a64b9ce93910aa6ac44c61ebf37a5e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:BIUOSSU6LXXHK6N4ZWA4ZGOCGY","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"From Reproduction to Replication: Evaluating Research Agents with Progressive Code Masking","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Alex Wilf, Daniel Fried, Gyeongwon James Kim, Louis-Philippe Morency","submitted_at":"2025-06-24T15:39:20Z","abstract_excerpt":"Recent progress in autonomous code generation has fueled excitement around AI agents capable of accelerating scientific discovery by running experiments. However, there is currently no benchmark that evaluates whether such agents can implement scientific ideas when given varied amounts of code as a starting point, interpolating between reproduction (running code) and from-scratch replication (fully re-implementing and running code). We introduce AutoExperiment, a benchmark that evaluates AI agents' ability to implement and run machine learning experiments based on natural language descriptions"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.19724","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2506.19724/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:26:37Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"hqntx8oQeFpQA5p5HAkKom4uLFJQqSRW9MAm9wc4EpKCKgGCzfgWC48w0+YnWFPdlXnypfHLYZb1//dvoPpfDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-20T20:48:59.665795Z"},"content_sha256":"e73c4f3b51d6922bfdda8809dd44b60be3d4d5f66ef89a65c77502cf416ec312","schema_version":"1.0","event_id":"sha256:e73c4f3b51d6922bfdda8809dd44b60be3d4d5f66ef89a65c77502cf416ec312"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/BIUOSSU6LXXHK6N4ZWA4ZGOCGY/bundle.json","state_url":"https://pith.science/pith/BIUOSSU6LXXHK6N4ZWA4ZGOCGY/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/BIUOSSU6LXXHK6N4ZWA4ZGOCGY/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-20T20:48:59Z","links":{"resolver":"https://pith.science/pith/BIUOSSU6LXXHK6N4ZWA4ZGOCGY","bundle":"https://pith.science/pith/BIUOSSU6LXXHK6N4ZWA4ZGOCGY/bundle.json","state":"https://pith.science/pith/BIUOSSU6LXXHK6N4ZWA4ZGOCGY/state.json","well_known_bundle":"https://pith.science/.well-known/pith/BIUOSSU6LXXHK6N4ZWA4ZGOCGY/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:BIUOSSU6LXXHK6N4ZWA4ZGOCGY","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"9f207c3b96bf3f60aa733bc4f304b0aafaadd069477dc98a9e8821df9e8bfa19","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-06-24T15:39:20Z","title_canon_sha256":"6b1f211ad47032c6f00ff532e6cbdc9e347106c4949d49379a3d4abc4db3bac7"},"schema_version":"1.0","source":{"id":"2506.19724","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.19724","created_at":"2026-07-05T11:26:37Z"},{"alias_kind":"arxiv_version","alias_value":"2506.19724v1","created_at":"2026-07-05T11:26:37Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.19724","created_at":"2026-07-05T11:26:37Z"},{"alias_kind":"pith_short_12","alias_value":"BIUOSSU6LXXH","created_at":"2026-07-05T11:26:37Z"},{"alias_kind":"pith_short_16","alias_value":"BIUOSSU6LXXHK6N4","created_at":"2026-07-05T11:26:37Z"},{"alias_kind":"pith_short_8","alias_value":"BIUOSSU6","created_at":"2026-07-05T11:26:37Z"}],"graph_snapshots":[{"event_id":"sha256:e73c4f3b51d6922bfdda8809dd44b60be3d4d5f66ef89a65c77502cf416ec312","target":"graph","created_at":"2026-07-05T11:26:37Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2506.19724/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recent progress in autonomous code generation has fueled excitement around AI agents capable of accelerating scientific discovery by running experiments. However, there is currently no benchmark that evaluates whether such agents can implement scientific ideas when given varied amounts of code as a starting point, interpolating between reproduction (running code) and from-scratch replication (fully re-implementing and running code). We introduce AutoExperiment, a benchmark that evaluates AI agents' ability to implement and run machine learning experiments based on natural language descriptions","authors_text":"Alex Wilf, Daniel Fried, Gyeongwon James Kim, Louis-Philippe Morency","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-06-24T15:39:20Z","title":"From Reproduction to Replication: Evaluating Research Agents with Progressive Code Masking"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.19724","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b7855a91aab4497314542a184eb2b8d633a64b9ce93910aa6ac44c61ebf37a5e","target":"record","created_at":"2026-07-05T11:26:37Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"9f207c3b96bf3f60aa733bc4f304b0aafaadd069477dc98a9e8821df9e8bfa19","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-06-24T15:39:20Z","title_canon_sha256":"6b1f211ad47032c6f00ff532e6cbdc9e347106c4949d49379a3d4abc4db3bac7"},"schema_version":"1.0","source":{"id":"2506.19724","kind":"arxiv","version":1}},"canonical_sha256":"0a28e94a9e5dee7579bccd81cc99c2363d650253b6109488e3a333d9cfc4896f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0a28e94a9e5dee7579bccd81cc99c2363d650253b6109488e3a333d9cfc4896f","first_computed_at":"2026-07-05T11:26:37.692536Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:26:37.692536Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"6hM1cmgaZrTbeLeMWrjW+LDuSSKTAG8gFW/xXn80B52jGOoJy/aPKU+9Ou9HM+3LhAMS8mGikQLNPCKl+T9KDg==","signature_status":"signed_v1","signed_at":"2026-07-05T11:26:37.693137Z","signed_message":"canonical_sha256_bytes"},"source_id":"2506.19724","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b7855a91aab4497314542a184eb2b8d633a64b9ce93910aa6ac44c61ebf37a5e","sha256:e73c4f3b51d6922bfdda8809dd44b60be3d4d5f66ef89a65c77502cf416ec312"],"state_sha256":"7e18e37a4d2ce38dc5cfbfd36dba161b15a830925ed5122070e9bcfc19d563b2"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"lC/YgFvylG6sCKMCb1uAjbRU6u0RrZ4rfM0jf2Y1Nxi/U7M9sH9EXDhNJYflJbh9BPItJgRBNDYbFrUkQ4e4Ag==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-20T20:48:59.669295Z","bundle_sha256":"3b99f60454325ccf5e255febe95cd2e3e2ee0a2ed6d36f10774caac08bd2eb43"}}