{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:WOO3HFOWUB7YQPZ4HKOE55FPAD","short_pith_number":"pith:WOO3HFOW","canonical_record":{"source":{"id":"2410.16007","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-10-21T13:41:15Z","cross_cats_sorted":[],"title_canon_sha256":"361e4fe7f48d2214a641ca5fb31248929e334726c0d797a7ee2193bbad720f46","abstract_canon_sha256":"461e9444eb41c1ff22dd6e689774983e1f222a1e07681f8438a752ccd80cf1af"},"schema_version":"1.0"},"canonical_sha256":"b39db395d6a07f883f3c3a9c4ef4af00f3bfeabbc50f548ab5fe4a07f6ad7045","source":{"kind":"arxiv","id":"2410.16007","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.16007","created_at":"2026-07-05T10:24:26Z"},{"alias_kind":"arxiv_version","alias_value":"2410.16007v2","created_at":"2026-07-05T10:24:26Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.16007","created_at":"2026-07-05T10:24:26Z"},{"alias_kind":"pith_short_12","alias_value":"WOO3HFOWUB7Y","created_at":"2026-07-05T10:24:26Z"},{"alias_kind":"pith_short_16","alias_value":"WOO3HFOWUB7YQPZ4","created_at":"2026-07-05T10:24:26Z"},{"alias_kind":"pith_short_8","alias_value":"WOO3HFOW","created_at":"2026-07-05T10:24:26Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:WOO3HFOWUB7YQPZ4HKOE55FPAD","target":"record","payload":{"canonical_record":{"source":{"id":"2410.16007","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-10-21T13:41:15Z","cross_cats_sorted":[],"title_canon_sha256":"361e4fe7f48d2214a641ca5fb31248929e334726c0d797a7ee2193bbad720f46","abstract_canon_sha256":"461e9444eb41c1ff22dd6e689774983e1f222a1e07681f8438a752ccd80cf1af"},"schema_version":"1.0"},"canonical_sha256":"b39db395d6a07f883f3c3a9c4ef4af00f3bfeabbc50f548ab5fe4a07f6ad7045","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:24:26.658844Z","signature_b64":"4vq+wH0ZhQVaxf8/KbDajcLmA3myzvnm21sz0myWBU+NVJXkoU5OoQ7OO1PB5re4E14PmiZBbmyHV43OokYeBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b39db395d6a07f883f3c3a9c4ef4af00f3bfeabbc50f548ab5fe4a07f6ad7045","last_reissued_at":"2026-07-05T10:24:26.658234Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:24:26.658234Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.16007","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:24:26Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"bSI2uCSZ0ymwxNeZkP+firNLxoFyUzHOQgqXp/pgaFoIdwhGkDq2Aaw1aDongjlRbS9buWcYZWL7qCMhk1ehCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T05:10:37.090656Z"},"content_sha256":"e7063888514223be599802bed917a013af3c4023e68cdf35415dd1d465d53857","schema_version":"1.0","event_id":"sha256:e7063888514223be599802bed917a013af3c4023e68cdf35415dd1d465d53857"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:WOO3HFOWUB7YQPZ4HKOE55FPAD","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Language Model Probabilities are Not Calibrated in Numeric Contexts","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Charles Lovering, Chris Tanner, Michael Krumdick, Nilesh Kumar, Rik Koncel-Kedziorski, Seth Ebner, Varshini Reddy, Viet Dac Lai","submitted_at":"2024-10-21T13:41:15Z","abstract_excerpt":"Some statements have one well-defined continuation (e.g., \"the Eiffel Tower is in [Paris]\"), whereas others have a natural distribution over multiple options (e.g., \"the weighted coin flip was [Heads/Tails].\") We argue that language model (LM) outputs should capture these natural distributions. Our work specifically tests whether LM output probabilities are calibrated to numeric information within their textual contexts. For example, if the context (the prompt) concerns two equally likely options (e.g., heads or tails for a fair coin), the LM output probabilities should also be equal. Likewise"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.16007","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.16007/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:24:26Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"sdniMTlDJyK87em2yU2COUghPgqLEOpUMIk//SLgI6XxTZCys3/Hb5XcgupVFV0GKWnfQjCXlL85bDqMsZq2Bw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T05:10:37.091037Z"},"content_sha256":"c809ec6480bf5c6494dff2d9c2c72fb59005a20ad9e7b116cec6691f24040271","schema_version":"1.0","event_id":"sha256:c809ec6480bf5c6494dff2d9c2c72fb59005a20ad9e7b116cec6691f24040271"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/WOO3HFOWUB7YQPZ4HKOE55FPAD/bundle.json","state_url":"https://pith.science/pith/WOO3HFOWUB7YQPZ4HKOE55FPAD/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/WOO3HFOWUB7YQPZ4HKOE55FPAD/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T05:10:37Z","links":{"resolver":"https://pith.science/pith/WOO3HFOWUB7YQPZ4HKOE55FPAD","bundle":"https://pith.science/pith/WOO3HFOWUB7YQPZ4HKOE55FPAD/bundle.json","state":"https://pith.science/pith/WOO3HFOWUB7YQPZ4HKOE55FPAD/state.json","well_known_bundle":"https://pith.science/.well-known/pith/WOO3HFOWUB7YQPZ4HKOE55FPAD/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:WOO3HFOWUB7YQPZ4HKOE55FPAD","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"461e9444eb41c1ff22dd6e689774983e1f222a1e07681f8438a752ccd80cf1af","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-10-21T13:41:15Z","title_canon_sha256":"361e4fe7f48d2214a641ca5fb31248929e334726c0d797a7ee2193bbad720f46"},"schema_version":"1.0","source":{"id":"2410.16007","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.16007","created_at":"2026-07-05T10:24:26Z"},{"alias_kind":"arxiv_version","alias_value":"2410.16007v2","created_at":"2026-07-05T10:24:26Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.16007","created_at":"2026-07-05T10:24:26Z"},{"alias_kind":"pith_short_12","alias_value":"WOO3HFOWUB7Y","created_at":"2026-07-05T10:24:26Z"},{"alias_kind":"pith_short_16","alias_value":"WOO3HFOWUB7YQPZ4","created_at":"2026-07-05T10:24:26Z"},{"alias_kind":"pith_short_8","alias_value":"WOO3HFOW","created_at":"2026-07-05T10:24:26Z"}],"graph_snapshots":[{"event_id":"sha256:c809ec6480bf5c6494dff2d9c2c72fb59005a20ad9e7b116cec6691f24040271","target":"graph","created_at":"2026-07-05T10:24:26Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.16007/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Some statements have one well-defined continuation (e.g., \"the Eiffel Tower is in [Paris]\"), whereas others have a natural distribution over multiple options (e.g., \"the weighted coin flip was [Heads/Tails].\") We argue that language model (LM) outputs should capture these natural distributions. Our work specifically tests whether LM output probabilities are calibrated to numeric information within their textual contexts. For example, if the context (the prompt) concerns two equally likely options (e.g., heads or tails for a fair coin), the LM output probabilities should also be equal. Likewise","authors_text":"Charles Lovering, Chris Tanner, Michael Krumdick, Nilesh Kumar, Rik Koncel-Kedziorski, Seth Ebner, Varshini Reddy, Viet Dac Lai","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-10-21T13:41:15Z","title":"Language Model Probabilities are Not Calibrated in Numeric Contexts"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.16007","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:e7063888514223be599802bed917a013af3c4023e68cdf35415dd1d465d53857","target":"record","created_at":"2026-07-05T10:24:26Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"461e9444eb41c1ff22dd6e689774983e1f222a1e07681f8438a752ccd80cf1af","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-10-21T13:41:15Z","title_canon_sha256":"361e4fe7f48d2214a641ca5fb31248929e334726c0d797a7ee2193bbad720f46"},"schema_version":"1.0","source":{"id":"2410.16007","kind":"arxiv","version":2}},"canonical_sha256":"b39db395d6a07f883f3c3a9c4ef4af00f3bfeabbc50f548ab5fe4a07f6ad7045","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"b39db395d6a07f883f3c3a9c4ef4af00f3bfeabbc50f548ab5fe4a07f6ad7045","first_computed_at":"2026-07-05T10:24:26.658234Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:24:26.658234Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"4vq+wH0ZhQVaxf8/KbDajcLmA3myzvnm21sz0myWBU+NVJXkoU5OoQ7OO1PB5re4E14PmiZBbmyHV43OokYeBg==","signature_status":"signed_v1","signed_at":"2026-07-05T10:24:26.658844Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.16007","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:e7063888514223be599802bed917a013af3c4023e68cdf35415dd1d465d53857","sha256:c809ec6480bf5c6494dff2d9c2c72fb59005a20ad9e7b116cec6691f24040271"],"state_sha256":"7e80024df0601c9f52fa55fc2c3dff7efef2c5fb662ab5e7f00e1145ad4cee15"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"prq65fBPDIdteFh2btLNjApfZitRz9QPwuLQ+TMrpfIkN2KSgHoo6S02zz/DF70bYfJgIHz7aLdHDXHwFLmYDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T05:10:37.093487Z","bundle_sha256":"0f565082299b01a45200ecd2470a0ef31b8ae065fd2347646e16dd6e7b3a3d10"}}