{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:O7B3SOF5VXP57QWQN4VJ5ZSNR3","short_pith_number":"pith:O7B3SOF5","canonical_record":{"source":{"id":"2402.19450","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-02-29T18:48:18Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"8c833bb19c8462c964915568c4eac1c78093ef0f9c63083be94e6bb3c6088e8f","abstract_canon_sha256":"ee812ccc7a8ceac6d0d2292ddb583a6acc6a6bf674e559a1f012c3802ce269c8"},"schema_version":"1.0"},"canonical_sha256":"77c3b938bdaddfdfc2d06f2a9ee64d8ee2aff4f79eaf678bd115deba23b066af","source":{"kind":"arxiv","id":"2402.19450","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.19450","created_at":"2026-07-05T07:50:45Z"},{"alias_kind":"arxiv_version","alias_value":"2402.19450v1","created_at":"2026-07-05T07:50:45Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.19450","created_at":"2026-07-05T07:50:45Z"},{"alias_kind":"pith_short_12","alias_value":"O7B3SOF5VXP5","created_at":"2026-07-05T07:50:45Z"},{"alias_kind":"pith_short_16","alias_value":"O7B3SOF5VXP57QWQ","created_at":"2026-07-05T07:50:45Z"},{"alias_kind":"pith_short_8","alias_value":"O7B3SOF5","created_at":"2026-07-05T07:50:45Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:O7B3SOF5VXP57QWQN4VJ5ZSNR3","target":"record","payload":{"canonical_record":{"source":{"id":"2402.19450","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-02-29T18:48:18Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"8c833bb19c8462c964915568c4eac1c78093ef0f9c63083be94e6bb3c6088e8f","abstract_canon_sha256":"ee812ccc7a8ceac6d0d2292ddb583a6acc6a6bf674e559a1f012c3802ce269c8"},"schema_version":"1.0"},"canonical_sha256":"77c3b938bdaddfdfc2d06f2a9ee64d8ee2aff4f79eaf678bd115deba23b066af","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:50:45.472464Z","signature_b64":"Rh26opyywfqyQ4xS3XDj8CClvnVpq6xOl3ayD3EmogEhtHNQ8lOrIVFHy8WDDbcHoC858XYepYQoUiYHr7rIAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"77c3b938bdaddfdfc2d06f2a9ee64d8ee2aff4f79eaf678bd115deba23b066af","last_reissued_at":"2026-07-05T07:50:45.471949Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:50:45.471949Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2402.19450","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:50:45Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ydd49mwaZg3thoPkmnMM6Ill2YLBq0oYHPs/StDOmSFKnY0dQkXsV65vL/148nCKVOQhiNHNFdqhds5nLTEiDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T17:07:23.417000Z"},"content_sha256":"39200f418bc013660124bb6e6987d1e00e226126a59c180233cc8ed28f90cdbc","schema_version":"1.0","event_id":"sha256:39200f418bc013660124bb6e6987d1e00e226126a59c180233cc8ed28f90cdbc"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:O7B3SOF5VXP57QWQN4VJ5ZSNR3","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Functional Benchmarks for Robust Evaluation of Reasoning Performance, and the Reasoning Gap","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.AI","authors_text":"Adwaith Samod T, Ajay Sukumar, Alan Philipose, Annarose M B, Anto P V, Saurabh Srivastava, Shashank Menon, Sooraj Thomas, Stevin Prince","submitted_at":"2024-02-29T18:48:18Z","abstract_excerpt":"We propose a framework for robust evaluation of reasoning capabilities of language models, using functional variants of benchmarks. Models that solve a reasoning test should exhibit no difference in performance over the static version of a problem compared to a snapshot of the functional variant. We have rewritten the relevant fragment of the MATH benchmark into its functional variant MATH(), with functionalization of other benchmarks to follow. When evaluating current state-of-the-art models over snapshots of MATH(), we find a reasoning gap -- the percentage difference between the static and "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.19450","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.19450/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:50:45Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"NvX1+NppaInqVmO4GYrhuhXIFbZAV+LJjLWSL/weHVPluthjo/P5pWRCA96JVrZoefgKEG4DmfQDFdGMT45HAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T17:07:23.417549Z"},"content_sha256":"caaf62a27da95018ed50f21572bea593d6618b88096f25bd8aeef00224f02658","schema_version":"1.0","event_id":"sha256:caaf62a27da95018ed50f21572bea593d6618b88096f25bd8aeef00224f02658"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/O7B3SOF5VXP57QWQN4VJ5ZSNR3/bundle.json","state_url":"https://pith.science/pith/O7B3SOF5VXP57QWQN4VJ5ZSNR3/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/O7B3SOF5VXP57QWQN4VJ5ZSNR3/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T17:07:23Z","links":{"resolver":"https://pith.science/pith/O7B3SOF5VXP57QWQN4VJ5ZSNR3","bundle":"https://pith.science/pith/O7B3SOF5VXP57QWQN4VJ5ZSNR3/bundle.json","state":"https://pith.science/pith/O7B3SOF5VXP57QWQN4VJ5ZSNR3/state.json","well_known_bundle":"https://pith.science/.well-known/pith/O7B3SOF5VXP57QWQN4VJ5ZSNR3/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:O7B3SOF5VXP57QWQN4VJ5ZSNR3","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"ee812ccc7a8ceac6d0d2292ddb583a6acc6a6bf674e559a1f012c3802ce269c8","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-02-29T18:48:18Z","title_canon_sha256":"8c833bb19c8462c964915568c4eac1c78093ef0f9c63083be94e6bb3c6088e8f"},"schema_version":"1.0","source":{"id":"2402.19450","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.19450","created_at":"2026-07-05T07:50:45Z"},{"alias_kind":"arxiv_version","alias_value":"2402.19450v1","created_at":"2026-07-05T07:50:45Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.19450","created_at":"2026-07-05T07:50:45Z"},{"alias_kind":"pith_short_12","alias_value":"O7B3SOF5VXP5","created_at":"2026-07-05T07:50:45Z"},{"alias_kind":"pith_short_16","alias_value":"O7B3SOF5VXP57QWQ","created_at":"2026-07-05T07:50:45Z"},{"alias_kind":"pith_short_8","alias_value":"O7B3SOF5","created_at":"2026-07-05T07:50:45Z"}],"graph_snapshots":[{"event_id":"sha256:caaf62a27da95018ed50f21572bea593d6618b88096f25bd8aeef00224f02658","target":"graph","created_at":"2026-07-05T07:50:45Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.19450/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We propose a framework for robust evaluation of reasoning capabilities of language models, using functional variants of benchmarks. Models that solve a reasoning test should exhibit no difference in performance over the static version of a problem compared to a snapshot of the functional variant. We have rewritten the relevant fragment of the MATH benchmark into its functional variant MATH(), with functionalization of other benchmarks to follow. When evaluating current state-of-the-art models over snapshots of MATH(), we find a reasoning gap -- the percentage difference between the static and ","authors_text":"Adwaith Samod T, Ajay Sukumar, Alan Philipose, Annarose M B, Anto P V, Saurabh Srivastava, Shashank Menon, Sooraj Thomas, Stevin Prince","cross_cats":["cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-02-29T18:48:18Z","title":"Functional Benchmarks for Robust Evaluation of Reasoning Performance, and the Reasoning Gap"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.19450","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:39200f418bc013660124bb6e6987d1e00e226126a59c180233cc8ed28f90cdbc","target":"record","created_at":"2026-07-05T07:50:45Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"ee812ccc7a8ceac6d0d2292ddb583a6acc6a6bf674e559a1f012c3802ce269c8","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-02-29T18:48:18Z","title_canon_sha256":"8c833bb19c8462c964915568c4eac1c78093ef0f9c63083be94e6bb3c6088e8f"},"schema_version":"1.0","source":{"id":"2402.19450","kind":"arxiv","version":1}},"canonical_sha256":"77c3b938bdaddfdfc2d06f2a9ee64d8ee2aff4f79eaf678bd115deba23b066af","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"77c3b938bdaddfdfc2d06f2a9ee64d8ee2aff4f79eaf678bd115deba23b066af","first_computed_at":"2026-07-05T07:50:45.471949Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:50:45.471949Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Rh26opyywfqyQ4xS3XDj8CClvnVpq6xOl3ayD3EmogEhtHNQ8lOrIVFHy8WDDbcHoC858XYepYQoUiYHr7rIAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T07:50:45.472464Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.19450","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:39200f418bc013660124bb6e6987d1e00e226126a59c180233cc8ed28f90cdbc","sha256:caaf62a27da95018ed50f21572bea593d6618b88096f25bd8aeef00224f02658"],"state_sha256":"dfd2c7ace52e96958b41471662c92d286e32a064d0611bc2ee4a72b791c22ccf"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QtaGHQI9j15Gt1df3wPeWBbLbZG6vbMjH3got3cAjkcaeGiYi/CwtvBNTzod2mVHeVDXhvISjmg8HBmMLzXGDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T17:07:23.424184Z","bundle_sha256":"c620111e3f3f443db848a224423a9f041b2598717ef2372ab72a94302e0e4b8f"}}