{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:EMLSFKQ2HUFVRWP4R6TVGMWKWD","short_pith_number":"pith:EMLSFKQ2","canonical_record":{"source":{"id":"2410.19056","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-10-24T18:02:37Z","cross_cats_sorted":[],"title_canon_sha256":"6fa8924f0501f6cdcbabb0e2ecc7b12eff0ff53bf2ce1d228ec4feeff89689d4","abstract_canon_sha256":"a1fea6f2ca9d67bcc03a25413ebc5757d0b1531724929b02979c6a4bad89420f"},"schema_version":"1.0"},"canonical_sha256":"231722aa1a3d0b58d9fc8fa75332cab0f8cf00dd326abd5fb0f54fcccb39fe6e","source":{"kind":"arxiv","id":"2410.19056","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.19056","created_at":"2026-07-05T09:25:50Z"},{"alias_kind":"arxiv_version","alias_value":"2410.19056v1","created_at":"2026-07-05T09:25:50Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.19056","created_at":"2026-07-05T09:25:50Z"},{"alias_kind":"pith_short_12","alias_value":"EMLSFKQ2HUFV","created_at":"2026-07-05T09:25:50Z"},{"alias_kind":"pith_short_16","alias_value":"EMLSFKQ2HUFVRWP4","created_at":"2026-07-05T09:25:50Z"},{"alias_kind":"pith_short_8","alias_value":"EMLSFKQ2","created_at":"2026-07-05T09:25:50Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:EMLSFKQ2HUFVRWP4R6TVGMWKWD","target":"record","payload":{"canonical_record":{"source":{"id":"2410.19056","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-10-24T18:02:37Z","cross_cats_sorted":[],"title_canon_sha256":"6fa8924f0501f6cdcbabb0e2ecc7b12eff0ff53bf2ce1d228ec4feeff89689d4","abstract_canon_sha256":"a1fea6f2ca9d67bcc03a25413ebc5757d0b1531724929b02979c6a4bad89420f"},"schema_version":"1.0"},"canonical_sha256":"231722aa1a3d0b58d9fc8fa75332cab0f8cf00dd326abd5fb0f54fcccb39fe6e","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:25:50.900613Z","signature_b64":"/DupAmnw+zDIsQrpol12pJVQ5+SyoEZF5Kv/AOTyOHIrApaDA3TFEXxPPj1Fy+Q2EVvJLhxBFzbN6uWj8QeiBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"231722aa1a3d0b58d9fc8fa75332cab0f8cf00dd326abd5fb0f54fcccb39fe6e","last_reissued_at":"2026-07-05T09:25:50.900116Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:25:50.900116Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.19056","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:25:50Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"GO5E0oV2CSrwVHO4a8wsFxza+yQH4mojf9EF4RhUgFlOnvPFvX+aJg/WsGDTYDcPmn/pygBHlBzweyEUKImaCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T13:54:35.784561Z"},"content_sha256":"c79172434c35601988decdeef26f09c8f227d031bc654def2338936ed2c07c2c","schema_version":"1.0","event_id":"sha256:c79172434c35601988decdeef26f09c8f227d031bc654def2338936ed2c07c2c"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:EMLSFKQ2HUFVRWP4R6TVGMWKWD","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"ReasonAgain: Using Extractable Symbolic Programs to Evaluate Mathematical Reasoning","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Ben Zhou, Dan Roth, Hao Cheng, Xiaodong Yu","submitted_at":"2024-10-24T18:02:37Z","abstract_excerpt":"Existing math datasets evaluate the reasoning abilities of large language models (LLMs) by either using the final answer or the intermediate reasoning steps derived from static examples. However, the former approach fails to surface model's uses of shortcuts and wrong reasoning while the later poses challenges in accommodating alternative solutions. In this work, we seek to use symbolic programs as a means for automated evaluation if a model can consistently produce correct final answers across various inputs to the program. We begin by extracting programs for popular math datasets (GSM8K and "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.19056","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.19056/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:25:50Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"AEGc8g0+tL7hA6q7o15UYSzf7a23FO5S54oKEgvftASNEogaA/2ukWcZtSxBzgKB+vIX1E5wDycGOI1jhBSUCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T13:54:35.785058Z"},"content_sha256":"4835361b27ac4ec2d8c32e177b08e13a9eb34ea3103fd8fb4cf49f1e409fc029","schema_version":"1.0","event_id":"sha256:4835361b27ac4ec2d8c32e177b08e13a9eb34ea3103fd8fb4cf49f1e409fc029"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/EMLSFKQ2HUFVRWP4R6TVGMWKWD/bundle.json","state_url":"https://pith.science/pith/EMLSFKQ2HUFVRWP4R6TVGMWKWD/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/EMLSFKQ2HUFVRWP4R6TVGMWKWD/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T13:54:35Z","links":{"resolver":"https://pith.science/pith/EMLSFKQ2HUFVRWP4R6TVGMWKWD","bundle":"https://pith.science/pith/EMLSFKQ2HUFVRWP4R6TVGMWKWD/bundle.json","state":"https://pith.science/pith/EMLSFKQ2HUFVRWP4R6TVGMWKWD/state.json","well_known_bundle":"https://pith.science/.well-known/pith/EMLSFKQ2HUFVRWP4R6TVGMWKWD/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:EMLSFKQ2HUFVRWP4R6TVGMWKWD","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"a1fea6f2ca9d67bcc03a25413ebc5757d0b1531724929b02979c6a4bad89420f","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-10-24T18:02:37Z","title_canon_sha256":"6fa8924f0501f6cdcbabb0e2ecc7b12eff0ff53bf2ce1d228ec4feeff89689d4"},"schema_version":"1.0","source":{"id":"2410.19056","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.19056","created_at":"2026-07-05T09:25:50Z"},{"alias_kind":"arxiv_version","alias_value":"2410.19056v1","created_at":"2026-07-05T09:25:50Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.19056","created_at":"2026-07-05T09:25:50Z"},{"alias_kind":"pith_short_12","alias_value":"EMLSFKQ2HUFV","created_at":"2026-07-05T09:25:50Z"},{"alias_kind":"pith_short_16","alias_value":"EMLSFKQ2HUFVRWP4","created_at":"2026-07-05T09:25:50Z"},{"alias_kind":"pith_short_8","alias_value":"EMLSFKQ2","created_at":"2026-07-05T09:25:50Z"}],"graph_snapshots":[{"event_id":"sha256:4835361b27ac4ec2d8c32e177b08e13a9eb34ea3103fd8fb4cf49f1e409fc029","target":"graph","created_at":"2026-07-05T09:25:50Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.19056/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Existing math datasets evaluate the reasoning abilities of large language models (LLMs) by either using the final answer or the intermediate reasoning steps derived from static examples. However, the former approach fails to surface model's uses of shortcuts and wrong reasoning while the later poses challenges in accommodating alternative solutions. In this work, we seek to use symbolic programs as a means for automated evaluation if a model can consistently produce correct final answers across various inputs to the program. We begin by extracting programs for popular math datasets (GSM8K and ","authors_text":"Ben Zhou, Dan Roth, Hao Cheng, Xiaodong Yu","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-10-24T18:02:37Z","title":"ReasonAgain: Using Extractable Symbolic Programs to Evaluate Mathematical Reasoning"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.19056","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:c79172434c35601988decdeef26f09c8f227d031bc654def2338936ed2c07c2c","target":"record","created_at":"2026-07-05T09:25:50Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"a1fea6f2ca9d67bcc03a25413ebc5757d0b1531724929b02979c6a4bad89420f","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-10-24T18:02:37Z","title_canon_sha256":"6fa8924f0501f6cdcbabb0e2ecc7b12eff0ff53bf2ce1d228ec4feeff89689d4"},"schema_version":"1.0","source":{"id":"2410.19056","kind":"arxiv","version":1}},"canonical_sha256":"231722aa1a3d0b58d9fc8fa75332cab0f8cf00dd326abd5fb0f54fcccb39fe6e","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"231722aa1a3d0b58d9fc8fa75332cab0f8cf00dd326abd5fb0f54fcccb39fe6e","first_computed_at":"2026-07-05T09:25:50.900116Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:25:50.900116Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"/DupAmnw+zDIsQrpol12pJVQ5+SyoEZF5Kv/AOTyOHIrApaDA3TFEXxPPj1Fy+Q2EVvJLhxBFzbN6uWj8QeiBw==","signature_status":"signed_v1","signed_at":"2026-07-05T09:25:50.900613Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.19056","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:c79172434c35601988decdeef26f09c8f227d031bc654def2338936ed2c07c2c","sha256:4835361b27ac4ec2d8c32e177b08e13a9eb34ea3103fd8fb4cf49f1e409fc029"],"state_sha256":"63c6271a7c4d055e61db95970912641115009e0eef52213718c9a6e45a7276b1"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"zPSynPIo8KrsQzPLJgdnDoNRS/Zn2gSavvgcyRL7zq7JyQdk9PVk1ZwflDdERvJXy6w6QKHSbYGCRDo9g1p/AA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T13:54:35.790106Z","bundle_sha256":"4e16aa58e7dddb68ab321cfc958ac3e2dbfbef63492f194ecbddee2f1e0d6404"}}