{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:2AJHJH5QYQEALHXNUW5XCE7UZQ","short_pith_number":"pith:2AJHJH5Q","canonical_record":{"source":{"id":"2506.17335","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2025-06-19T07:04:16Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"e7e40e1de9312e32129cc547ce984542a50ee874f77fdccb03e47f242c700a18","abstract_canon_sha256":"34a5ec6f53f5212c678197c04dd23c78ac2a59608cd8486c88d437aa24658d1a"},"schema_version":"1.0"},"canonical_sha256":"d012749fb0c408059eeda5bb7113f4cc069a031f2635a88aff33a9c6770bbd50","source":{"kind":"arxiv","id":"2506.17335","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.17335","created_at":"2026-07-05T11:25:16Z"},{"alias_kind":"arxiv_version","alias_value":"2506.17335v1","created_at":"2026-07-05T11:25:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.17335","created_at":"2026-07-05T11:25:16Z"},{"alias_kind":"pith_short_12","alias_value":"2AJHJH5QYQEA","created_at":"2026-07-05T11:25:16Z"},{"alias_kind":"pith_short_16","alias_value":"2AJHJH5QYQEALHXN","created_at":"2026-07-05T11:25:16Z"},{"alias_kind":"pith_short_8","alias_value":"2AJHJH5Q","created_at":"2026-07-05T11:25:16Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:2AJHJH5QYQEALHXNUW5XCE7UZQ","target":"record","payload":{"canonical_record":{"source":{"id":"2506.17335","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2025-06-19T07:04:16Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"e7e40e1de9312e32129cc547ce984542a50ee874f77fdccb03e47f242c700a18","abstract_canon_sha256":"34a5ec6f53f5212c678197c04dd23c78ac2a59608cd8486c88d437aa24658d1a"},"schema_version":"1.0"},"canonical_sha256":"d012749fb0c408059eeda5bb7113f4cc069a031f2635a88aff33a9c6770bbd50","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:25:16.179889Z","signature_b64":"BsneDUzeTi7H3bk5efIlJrGocEycVyjJ8botQc0oxorukrYSL1aGx/liWtLNuB1C0WBu/hIDiwC3tY//bT5GCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d012749fb0c408059eeda5bb7113f4cc069a031f2635a88aff33a9c6770bbd50","last_reissued_at":"2026-07-05T11:25:16.179377Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:25:16.179377Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2506.17335","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:25:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"5CmWB+rvO1sLUQmMyJMRDO2Dxrb2PbtUHpKHm2sjKsRLMZk4ReKHyHm9QFCbLqot1wSHLnBMN4xiPACVmXu7BQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-22T08:16:36.546723Z"},"content_sha256":"b268dce851fc37b7e3a30e52387daeff040663bf4407db09c15a518f34151a05","schema_version":"1.0","event_id":"sha256:b268dce851fc37b7e3a30e52387daeff040663bf4407db09c15a518f34151a05"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:2AJHJH5QYQEALHXNUW5XCE7UZQ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"LMR-BENCH: Evaluating LLM Agent's Ability on Reproducing Language Modeling Research","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.SE","authors_text":"Daoyang Li, George Michalopoulos, Kaiyu He, Liqiang Jing, Mian Zhang, Peilin Wu, Ruochen Li, Shuo Yan, Xinya Du, Yue Zhang, Zhiyu Chen, Ziming Luo, Zimu Wang, Ziyang Zhang","submitted_at":"2025-06-19T07:04:16Z","abstract_excerpt":"Large language model (LLM) agents have demonstrated remarkable potential in advancing scientific discovery. However, their capability in the fundamental yet crucial task of reproducing code from research papers, especially in the NLP domain, remains underexplored. This task includes unique complex reasoning challenges in the intellectual synthesis of abstract concepts and the comprehension of code repositories with interdependent files. Motivated by this gap, we present LMR-BENCH, a benchmark designed to systematically evaluate the capability of LLM agents on code reproduction from Language Mo"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.17335","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2506.17335/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:25:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"0uI4KIUsmcYwC8TlnWbyZkXtTpOwYLl4EszubLaZ7OKHCjiKrWPHsEOFEqihrOcv9dEUhNmffNHY/u1RdQGaCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-22T08:16:36.547145Z"},"content_sha256":"b440c31d4af323aaea2f2a89dcf428faf3ca40823f877d4e531930586bec5e50","schema_version":"1.0","event_id":"sha256:b440c31d4af323aaea2f2a89dcf428faf3ca40823f877d4e531930586bec5e50"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/2AJHJH5QYQEALHXNUW5XCE7UZQ/bundle.json","state_url":"https://pith.science/pith/2AJHJH5QYQEALHXNUW5XCE7UZQ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/2AJHJH5QYQEALHXNUW5XCE7UZQ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-22T08:16:36Z","links":{"resolver":"https://pith.science/pith/2AJHJH5QYQEALHXNUW5XCE7UZQ","bundle":"https://pith.science/pith/2AJHJH5QYQEALHXNUW5XCE7UZQ/bundle.json","state":"https://pith.science/pith/2AJHJH5QYQEALHXNUW5XCE7UZQ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/2AJHJH5QYQEALHXNUW5XCE7UZQ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:2AJHJH5QYQEALHXNUW5XCE7UZQ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"34a5ec6f53f5212c678197c04dd23c78ac2a59608cd8486c88d437aa24658d1a","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2025-06-19T07:04:16Z","title_canon_sha256":"e7e40e1de9312e32129cc547ce984542a50ee874f77fdccb03e47f242c700a18"},"schema_version":"1.0","source":{"id":"2506.17335","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.17335","created_at":"2026-07-05T11:25:16Z"},{"alias_kind":"arxiv_version","alias_value":"2506.17335v1","created_at":"2026-07-05T11:25:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.17335","created_at":"2026-07-05T11:25:16Z"},{"alias_kind":"pith_short_12","alias_value":"2AJHJH5QYQEA","created_at":"2026-07-05T11:25:16Z"},{"alias_kind":"pith_short_16","alias_value":"2AJHJH5QYQEALHXN","created_at":"2026-07-05T11:25:16Z"},{"alias_kind":"pith_short_8","alias_value":"2AJHJH5Q","created_at":"2026-07-05T11:25:16Z"}],"graph_snapshots":[{"event_id":"sha256:b440c31d4af323aaea2f2a89dcf428faf3ca40823f877d4e531930586bec5e50","target":"graph","created_at":"2026-07-05T11:25:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2506.17335/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large language model (LLM) agents have demonstrated remarkable potential in advancing scientific discovery. However, their capability in the fundamental yet crucial task of reproducing code from research papers, especially in the NLP domain, remains underexplored. This task includes unique complex reasoning challenges in the intellectual synthesis of abstract concepts and the comprehension of code repositories with interdependent files. Motivated by this gap, we present LMR-BENCH, a benchmark designed to systematically evaluate the capability of LLM agents on code reproduction from Language Mo","authors_text":"Daoyang Li, George Michalopoulos, Kaiyu He, Liqiang Jing, Mian Zhang, Peilin Wu, Ruochen Li, Shuo Yan, Xinya Du, Yue Zhang, Zhiyu Chen, Ziming Luo, Zimu Wang, Ziyang Zhang","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2025-06-19T07:04:16Z","title":"LMR-BENCH: Evaluating LLM Agent's Ability on Reproducing Language Modeling Research"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.17335","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b268dce851fc37b7e3a30e52387daeff040663bf4407db09c15a518f34151a05","target":"record","created_at":"2026-07-05T11:25:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"34a5ec6f53f5212c678197c04dd23c78ac2a59608cd8486c88d437aa24658d1a","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2025-06-19T07:04:16Z","title_canon_sha256":"e7e40e1de9312e32129cc547ce984542a50ee874f77fdccb03e47f242c700a18"},"schema_version":"1.0","source":{"id":"2506.17335","kind":"arxiv","version":1}},"canonical_sha256":"d012749fb0c408059eeda5bb7113f4cc069a031f2635a88aff33a9c6770bbd50","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"d012749fb0c408059eeda5bb7113f4cc069a031f2635a88aff33a9c6770bbd50","first_computed_at":"2026-07-05T11:25:16.179377Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:25:16.179377Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"BsneDUzeTi7H3bk5efIlJrGocEycVyjJ8botQc0oxorukrYSL1aGx/liWtLNuB1C0WBu/hIDiwC3tY//bT5GCA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:25:16.179889Z","signed_message":"canonical_sha256_bytes"},"source_id":"2506.17335","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b268dce851fc37b7e3a30e52387daeff040663bf4407db09c15a518f34151a05","sha256:b440c31d4af323aaea2f2a89dcf428faf3ca40823f877d4e531930586bec5e50"],"state_sha256":"daf5bc8f229d5e1ccc490b8e6a2af38d2d2cae560976c16fb2c135cf1f52c74d"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Wdok2uS0VX5bCv2v1kbfAuLwicodGKRDMnVePfSlTR/Gk7iZBTU//OSvqDcXInM02Kb7I7sTYW3E5uRroMYdCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-22T08:16:36.549791Z","bundle_sha256":"51ac2fa7ee14d85845f26766443d4939b8738510eb93b73cfd064712a74fd499"}}