{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:FXYXVRDRTE4X36QSD6SK3IYSLR","short_pith_number":"pith:FXYXVRDR","canonical_record":{"source":{"id":"2306.01694","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-06-02T17:12:25Z","cross_cats_sorted":["cs.HC"],"title_canon_sha256":"10e0a99e4dc242b416a12b3e3cae4d8ccc7d28dc650922a77ecb66b7de575220","abstract_canon_sha256":"e63062c13c103ca6bfaaf606079c22122010d12b09851d4622a30c809a72c5fc"},"schema_version":"1.0"},"canonical_sha256":"2df17ac47199397dfa121fa4ada3125c4036260fc467cb0d8a73a0e713407eb9","source":{"kind":"arxiv","id":"2306.01694","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2306.01694","created_at":"2026-07-05T07:08:58Z"},{"alias_kind":"arxiv_version","alias_value":"2306.01694v2","created_at":"2026-07-05T07:08:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2306.01694","created_at":"2026-07-05T07:08:58Z"},{"alias_kind":"pith_short_12","alias_value":"FXYXVRDRTE4X","created_at":"2026-07-05T07:08:58Z"},{"alias_kind":"pith_short_16","alias_value":"FXYXVRDRTE4X36QS","created_at":"2026-07-05T07:08:58Z"},{"alias_kind":"pith_short_8","alias_value":"FXYXVRDR","created_at":"2026-07-05T07:08:58Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:FXYXVRDRTE4X36QSD6SK3IYSLR","target":"record","payload":{"canonical_record":{"source":{"id":"2306.01694","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-06-02T17:12:25Z","cross_cats_sorted":["cs.HC"],"title_canon_sha256":"10e0a99e4dc242b416a12b3e3cae4d8ccc7d28dc650922a77ecb66b7de575220","abstract_canon_sha256":"e63062c13c103ca6bfaaf606079c22122010d12b09851d4622a30c809a72c5fc"},"schema_version":"1.0"},"canonical_sha256":"2df17ac47199397dfa121fa4ada3125c4036260fc467cb0d8a73a0e713407eb9","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:08:58.483005Z","signature_b64":"puAOfvqEaNH6w8afCKvmxp/ERc/jkO5JekQ5qS1jt9u4f07uVgHG/KJGzDE53rA8ZeyERlg/rRlAjMKebem5AA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2df17ac47199397dfa121fa4ada3125c4036260fc467cb0d8a73a0e713407eb9","last_reissued_at":"2026-07-05T07:08:58.482511Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:08:58.482511Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2306.01694","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:08:58Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"/ouPCNBQXBLiQtWwZXJsJwj7tbQdqUvvWqXcoMF/HzSWRY4WOyaiXxDXMsVaqQetIrbUNi0Gt9uR/wp86SmSDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T16:21:05.377606Z"},"content_sha256":"679214300d9e3d21dc6f91c5627854b254805e3c78d2a254b9a6bf932770f649","schema_version":"1.0","event_id":"sha256:679214300d9e3d21dc6f91c5627854b254805e3c78d2a254b9a6bf932770f649"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:FXYXVRDRTE4X36QSD6SK3IYSLR","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Evaluating Language Models for Mathematics through Interactions","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.HC"],"primary_cat":"cs.LG","authors_text":"Adrian Weller, Albert Q. Jiang, Joshua B. Tenenbaum, Katherine M. Collins, Lionel Wong, Mateja Jamnik, Miri Zilka, Simon Frieder, Thomas Lukasiewicz, Timothy Gowers, Umang Bhatt, Wenda Li, William Hart, Yuhuai Wu","submitted_at":"2023-06-02T17:12:25Z","abstract_excerpt":"There is much excitement about the opportunity to harness the power of large language models (LLMs) when building problem-solving assistants. However, the standard methodology of evaluating LLMs relies on static pairs of inputs and outputs, and is insufficient for making an informed decision about which LLMs and under which assistive settings can they be sensibly used. Static assessment fails to account for the essential interactive element in LLM deployment, and therefore limits how we understand language model capabilities. We introduce CheckMate, an adaptable prototype platform for humans t"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2306.01694","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2306.01694/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:08:58Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"qV9Bp5phz0DS85Im+fHEWJrzgHWztBAtn+iuko8G+NCEXcBwAL5EF2oU6Epkh7lVM6ZDov4/3ZhW6y+2tDI+Dw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T16:21:05.378487Z"},"content_sha256":"dd03eb5338b000dc3b346004959e5d2339dac28652e904f23db781058b01a38d","schema_version":"1.0","event_id":"sha256:dd03eb5338b000dc3b346004959e5d2339dac28652e904f23db781058b01a38d"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/FXYXVRDRTE4X36QSD6SK3IYSLR/bundle.json","state_url":"https://pith.science/pith/FXYXVRDRTE4X36QSD6SK3IYSLR/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/FXYXVRDRTE4X36QSD6SK3IYSLR/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-03T16:21:05Z","links":{"resolver":"https://pith.science/pith/FXYXVRDRTE4X36QSD6SK3IYSLR","bundle":"https://pith.science/pith/FXYXVRDRTE4X36QSD6SK3IYSLR/bundle.json","state":"https://pith.science/pith/FXYXVRDRTE4X36QSD6SK3IYSLR/state.json","well_known_bundle":"https://pith.science/.well-known/pith/FXYXVRDRTE4X36QSD6SK3IYSLR/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:FXYXVRDRTE4X36QSD6SK3IYSLR","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"e63062c13c103ca6bfaaf606079c22122010d12b09851d4622a30c809a72c5fc","cross_cats_sorted":["cs.HC"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-06-02T17:12:25Z","title_canon_sha256":"10e0a99e4dc242b416a12b3e3cae4d8ccc7d28dc650922a77ecb66b7de575220"},"schema_version":"1.0","source":{"id":"2306.01694","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2306.01694","created_at":"2026-07-05T07:08:58Z"},{"alias_kind":"arxiv_version","alias_value":"2306.01694v2","created_at":"2026-07-05T07:08:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2306.01694","created_at":"2026-07-05T07:08:58Z"},{"alias_kind":"pith_short_12","alias_value":"FXYXVRDRTE4X","created_at":"2026-07-05T07:08:58Z"},{"alias_kind":"pith_short_16","alias_value":"FXYXVRDRTE4X36QS","created_at":"2026-07-05T07:08:58Z"},{"alias_kind":"pith_short_8","alias_value":"FXYXVRDR","created_at":"2026-07-05T07:08:58Z"}],"graph_snapshots":[{"event_id":"sha256:dd03eb5338b000dc3b346004959e5d2339dac28652e904f23db781058b01a38d","target":"graph","created_at":"2026-07-05T07:08:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2306.01694/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"There is much excitement about the opportunity to harness the power of large language models (LLMs) when building problem-solving assistants. However, the standard methodology of evaluating LLMs relies on static pairs of inputs and outputs, and is insufficient for making an informed decision about which LLMs and under which assistive settings can they be sensibly used. Static assessment fails to account for the essential interactive element in LLM deployment, and therefore limits how we understand language model capabilities. We introduce CheckMate, an adaptable prototype platform for humans t","authors_text":"Adrian Weller, Albert Q. Jiang, Joshua B. Tenenbaum, Katherine M. Collins, Lionel Wong, Mateja Jamnik, Miri Zilka, Simon Frieder, Thomas Lukasiewicz, Timothy Gowers, Umang Bhatt, Wenda Li, William Hart, Yuhuai Wu","cross_cats":["cs.HC"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-06-02T17:12:25Z","title":"Evaluating Language Models for Mathematics through Interactions"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2306.01694","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:679214300d9e3d21dc6f91c5627854b254805e3c78d2a254b9a6bf932770f649","target":"record","created_at":"2026-07-05T07:08:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"e63062c13c103ca6bfaaf606079c22122010d12b09851d4622a30c809a72c5fc","cross_cats_sorted":["cs.HC"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2023-06-02T17:12:25Z","title_canon_sha256":"10e0a99e4dc242b416a12b3e3cae4d8ccc7d28dc650922a77ecb66b7de575220"},"schema_version":"1.0","source":{"id":"2306.01694","kind":"arxiv","version":2}},"canonical_sha256":"2df17ac47199397dfa121fa4ada3125c4036260fc467cb0d8a73a0e713407eb9","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"2df17ac47199397dfa121fa4ada3125c4036260fc467cb0d8a73a0e713407eb9","first_computed_at":"2026-07-05T07:08:58.482511Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:08:58.482511Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"puAOfvqEaNH6w8afCKvmxp/ERc/jkO5JekQ5qS1jt9u4f07uVgHG/KJGzDE53rA8ZeyERlg/rRlAjMKebem5AA==","signature_status":"signed_v1","signed_at":"2026-07-05T07:08:58.483005Z","signed_message":"canonical_sha256_bytes"},"source_id":"2306.01694","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:679214300d9e3d21dc6f91c5627854b254805e3c78d2a254b9a6bf932770f649","sha256:dd03eb5338b000dc3b346004959e5d2339dac28652e904f23db781058b01a38d"],"state_sha256":"6f4bcb9898660128399438ef4a5a2fec8f14bfeb6eed44e82eba874201e4858a"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"mPL3pvg+/iBx/JjMs0WbpnknkGDwoR4UHvZTyFa2+b6SyeXD0GIWel4SFPQdMGsEMfc1TwCbDz5v+9exfOwXCg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-03T16:21:05.436261Z","bundle_sha256":"f9f56b8f9f2788f782390b1e1a0981b13d7fc1317e36d4a9e979772a984cc236"}}