{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:EUN5ZF2J3JEZEIIF3L2HVUH7EZ","short_pith_number":"pith:EUN5ZF2J","canonical_record":{"source":{"id":"2406.13975","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-20T03:50:23Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"bdf2ed0bd8f383e5ea8fbe0986ea41644e468db4b5a9e4c991f0f93db317091f","abstract_canon_sha256":"0224d930301a8e1e9461b5923327f5dfe044c8c068d82c98e07057a29eaf9a75"},"schema_version":"1.0"},"canonical_sha256":"251bdc9749da49922105daf47ad0ff2645a0976823a461a22aa871ed6d1ed4b9","source":{"kind":"arxiv","id":"2406.13975","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.13975","created_at":"2026-07-05T09:52:12Z"},{"alias_kind":"arxiv_version","alias_value":"2406.13975v3","created_at":"2026-07-05T09:52:12Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.13975","created_at":"2026-07-05T09:52:12Z"},{"alias_kind":"pith_short_12","alias_value":"EUN5ZF2J3JEZ","created_at":"2026-07-05T09:52:12Z"},{"alias_kind":"pith_short_16","alias_value":"EUN5ZF2J3JEZEIIF","created_at":"2026-07-05T09:52:12Z"},{"alias_kind":"pith_short_8","alias_value":"EUN5ZF2J","created_at":"2026-07-05T09:52:12Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:EUN5ZF2J3JEZEIIF3L2HVUH7EZ","target":"record","payload":{"canonical_record":{"source":{"id":"2406.13975","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-20T03:50:23Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"bdf2ed0bd8f383e5ea8fbe0986ea41644e468db4b5a9e4c991f0f93db317091f","abstract_canon_sha256":"0224d930301a8e1e9461b5923327f5dfe044c8c068d82c98e07057a29eaf9a75"},"schema_version":"1.0"},"canonical_sha256":"251bdc9749da49922105daf47ad0ff2645a0976823a461a22aa871ed6d1ed4b9","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:52:12.811456Z","signature_b64":"LCrOnoezmDqK+tfoD9uPZcJUhB7lsOIQVK37VE07mWqCvrmi37NQ/6axNvONIvUpm0WUw3/D8pwSfzN/ZlR1Bg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"251bdc9749da49922105daf47ad0ff2645a0976823a461a22aa871ed6d1ed4b9","last_reissued_at":"2026-07-05T09:52:12.810942Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:52:12.810942Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.13975","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:52:12Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"v9B/gL1klwW3yLiZkND4hIp0dakaxxVyFHv5hqsghqzkqSs4PYxwqG16zwatzmUiU+fpvpUrsfn8u1cI/iWVBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T04:32:29.649810Z"},"content_sha256":"94c6b1bc9e48dd44cf6446b32f0668239348e32f790f28b8aabee0a624030aa6","schema_version":"1.0","event_id":"sha256:94c6b1bc9e48dd44cf6446b32f0668239348e32f790f28b8aabee0a624030aa6"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:EUN5ZF2J3JEZEIIF3L2HVUH7EZ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"MR-Ben: A Meta-Reasoning Benchmark for Evaluating System-2 Thinking in LLMs","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Bailin Wang, Haochen Tan, Hao Zhang, Jianbo Dai, Jianqiao Lu, Jiaya Jia, Jingyao Li, Linling Shen, Pengguang Chen, Rongwu Xu, Wanru Zhao, Yingjia Wan, Yinhong Liu, Yukang Chen, Yuxuan Yao, Zehan Qi, Zhan Shi, Zhijiang Guo, Zhongshen Zeng","submitted_at":"2024-06-20T03:50:23Z","abstract_excerpt":"Large language models (LLMs) have shown increasing capability in problem-solving and decision-making, largely based on the step-by-step chain-of-thought reasoning processes. However, evaluating these reasoning abilities has become increasingly challenging. Existing outcome-based benchmarks are beginning to saturate, becoming less effective in tracking meaningful progress. To address this, we present a process-based benchmark MR-Ben that demands a meta-reasoning skill, where LMs are asked to locate and analyse potential errors in automatically generated reasoning steps. Our meta-reasoning parad"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.13975","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.13975/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:52:12Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"oC566XSR+aCyBPWIk7svzeBr6wwoGYabDU0qvB0vzNp3uC6Be4oDeXYtBnlM/4Ghxlunfo6Gz1TsAylA87qyCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T04:32:29.650764Z"},"content_sha256":"49e6123edd58954156eca97b902df43209aa6b810075f8d4a0ff3ecea209c314","schema_version":"1.0","event_id":"sha256:49e6123edd58954156eca97b902df43209aa6b810075f8d4a0ff3ecea209c314"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/EUN5ZF2J3JEZEIIF3L2HVUH7EZ/bundle.json","state_url":"https://pith.science/pith/EUN5ZF2J3JEZEIIF3L2HVUH7EZ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/EUN5ZF2J3JEZEIIF3L2HVUH7EZ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-06T04:32:29Z","links":{"resolver":"https://pith.science/pith/EUN5ZF2J3JEZEIIF3L2HVUH7EZ","bundle":"https://pith.science/pith/EUN5ZF2J3JEZEIIF3L2HVUH7EZ/bundle.json","state":"https://pith.science/pith/EUN5ZF2J3JEZEIIF3L2HVUH7EZ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/EUN5ZF2J3JEZEIIF3L2HVUH7EZ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:EUN5ZF2J3JEZEIIF3L2HVUH7EZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0224d930301a8e1e9461b5923327f5dfe044c8c068d82c98e07057a29eaf9a75","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-20T03:50:23Z","title_canon_sha256":"bdf2ed0bd8f383e5ea8fbe0986ea41644e468db4b5a9e4c991f0f93db317091f"},"schema_version":"1.0","source":{"id":"2406.13975","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.13975","created_at":"2026-07-05T09:52:12Z"},{"alias_kind":"arxiv_version","alias_value":"2406.13975v3","created_at":"2026-07-05T09:52:12Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.13975","created_at":"2026-07-05T09:52:12Z"},{"alias_kind":"pith_short_12","alias_value":"EUN5ZF2J3JEZ","created_at":"2026-07-05T09:52:12Z"},{"alias_kind":"pith_short_16","alias_value":"EUN5ZF2J3JEZEIIF","created_at":"2026-07-05T09:52:12Z"},{"alias_kind":"pith_short_8","alias_value":"EUN5ZF2J","created_at":"2026-07-05T09:52:12Z"}],"graph_snapshots":[{"event_id":"sha256:49e6123edd58954156eca97b902df43209aa6b810075f8d4a0ff3ecea209c314","target":"graph","created_at":"2026-07-05T09:52:12Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.13975/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large language models (LLMs) have shown increasing capability in problem-solving and decision-making, largely based on the step-by-step chain-of-thought reasoning processes. However, evaluating these reasoning abilities has become increasingly challenging. Existing outcome-based benchmarks are beginning to saturate, becoming less effective in tracking meaningful progress. To address this, we present a process-based benchmark MR-Ben that demands a meta-reasoning skill, where LMs are asked to locate and analyse potential errors in automatically generated reasoning steps. Our meta-reasoning parad","authors_text":"Bailin Wang, Haochen Tan, Hao Zhang, Jianbo Dai, Jianqiao Lu, Jiaya Jia, Jingyao Li, Linling Shen, Pengguang Chen, Rongwu Xu, Wanru Zhao, Yingjia Wan, Yinhong Liu, Yukang Chen, Yuxuan Yao, Zehan Qi, Zhan Shi, Zhijiang Guo, Zhongshen Zeng","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-20T03:50:23Z","title":"MR-Ben: A Meta-Reasoning Benchmark for Evaluating System-2 Thinking in LLMs"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.13975","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:94c6b1bc9e48dd44cf6446b32f0668239348e32f790f28b8aabee0a624030aa6","target":"record","created_at":"2026-07-05T09:52:12Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0224d930301a8e1e9461b5923327f5dfe044c8c068d82c98e07057a29eaf9a75","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-20T03:50:23Z","title_canon_sha256":"bdf2ed0bd8f383e5ea8fbe0986ea41644e468db4b5a9e4c991f0f93db317091f"},"schema_version":"1.0","source":{"id":"2406.13975","kind":"arxiv","version":3}},"canonical_sha256":"251bdc9749da49922105daf47ad0ff2645a0976823a461a22aa871ed6d1ed4b9","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"251bdc9749da49922105daf47ad0ff2645a0976823a461a22aa871ed6d1ed4b9","first_computed_at":"2026-07-05T09:52:12.810942Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:52:12.810942Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"LCrOnoezmDqK+tfoD9uPZcJUhB7lsOIQVK37VE07mWqCvrmi37NQ/6axNvONIvUpm0WUw3/D8pwSfzN/ZlR1Bg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:52:12.811456Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.13975","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:94c6b1bc9e48dd44cf6446b32f0668239348e32f790f28b8aabee0a624030aa6","sha256:49e6123edd58954156eca97b902df43209aa6b810075f8d4a0ff3ecea209c314"],"state_sha256":"b7efb650f3cfb9801cd1157775689c4c3109718365519861c93dd052a486be6a"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Kio6iBxo6muYuvzi1kHSER5QMOxFej+6InDScMWNY7wVP0oVWLIH8qy28XHwbPw90x+M07d97uvn3jELrq9CAg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-06T04:32:29.657858Z","bundle_sha256":"1465909aa772b78116c544ec4bcc03c4ec82ad9e46b9e3d4579874b0dd29aa00"}}