{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:JZH7WRVNENCYUECFHC3N3BNUZV","short_pith_number":"pith:JZH7WRVN","canonical_record":{"source":{"id":"2104.08315","kind":"arxiv","version":9},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-04-16T18:57:19Z","cross_cats_sorted":[],"title_canon_sha256":"9abbbb5566b1dbee5b097381c40ea3d20d79c6ec44cc137255fc9e92ba8430fd","abstract_canon_sha256":"0dfa9387d6a4c0b36e20f955c2e0fc5fc449859393bcad256a1e3823de339920"},"schema_version":"1.0"},"canonical_sha256":"4e4ffb46ad23458a104538b6dd85b4cd7759775b7049db33c95600328b3120d5","source":{"kind":"arxiv","id":"2104.08315","version":9},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2104.08315","created_at":"2026-07-05T05:17:42Z"},{"alias_kind":"arxiv_version","alias_value":"2104.08315v9","created_at":"2026-07-05T05:17:42Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2104.08315","created_at":"2026-07-05T05:17:42Z"},{"alias_kind":"pith_short_12","alias_value":"JZH7WRVNENCY","created_at":"2026-07-05T05:17:42Z"},{"alias_kind":"pith_short_16","alias_value":"JZH7WRVNENCYUECF","created_at":"2026-07-05T05:17:42Z"},{"alias_kind":"pith_short_8","alias_value":"JZH7WRVN","created_at":"2026-07-05T05:17:42Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:JZH7WRVNENCYUECFHC3N3BNUZV","target":"record","payload":{"canonical_record":{"source":{"id":"2104.08315","kind":"arxiv","version":9},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-04-16T18:57:19Z","cross_cats_sorted":[],"title_canon_sha256":"9abbbb5566b1dbee5b097381c40ea3d20d79c6ec44cc137255fc9e92ba8430fd","abstract_canon_sha256":"0dfa9387d6a4c0b36e20f955c2e0fc5fc449859393bcad256a1e3823de339920"},"schema_version":"1.0"},"canonical_sha256":"4e4ffb46ad23458a104538b6dd85b4cd7759775b7049db33c95600328b3120d5","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:17:42.979420Z","signature_b64":"JmSk7PDpSwltr2lhxe1jbULwjG0EoWMS6X1bF0BiXDNYeqD2j0IUYhwt5l4SZ/GTM9mESrY96JQUGbeEKZZXCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"4e4ffb46ad23458a104538b6dd85b4cd7759775b7049db33c95600328b3120d5","last_reissued_at":"2026-07-05T05:17:42.978894Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:17:42.978894Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2104.08315","source_version":9,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:17:42Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"pLSxgCQMmYvu2wNSR8grg9MELm9qGN1LuvHOVGq0Gzp8w03jghb+19CGAX/yJu65Cc4o5J6KQ6hiUaTvDgGpAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T15:10:07.057447Z"},"content_sha256":"6e5b7537d4497b280f16fbc93d3b4f8052e996816dd4a471538248289bbb391e","schema_version":"1.0","event_id":"sha256:6e5b7537d4497b280f16fbc93d3b4f8052e996816dd4a471538248289bbb391e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:JZH7WRVNENCYUECFHC3N3BNUZV","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Surface Form Competition: Why the Highest Probability Answer Isn't Always Right","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Ari Holtzman, Luke Zettlemoyer, Peter West, Vered Shwartz, Yejin Choi","submitted_at":"2021-04-16T18:57:19Z","abstract_excerpt":"Large language models have shown promising results in zero-shot settings (Brown et al.,2020; Radford et al., 2019). For example, they can perform multiple choice tasks simply by conditioning on a question and selecting the answer with the highest probability.\n  However, ranking by string probability can be problematic due to surface form competition-wherein different surface forms compete for probability mass, even if they represent the same underlying concept, e.g. \"computer\" and \"PC.\" Since probability mass is finite, this lowers the probability of the correct answer, due to competition from"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2104.08315","kind":"arxiv","version":9},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2104.08315/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:17:42Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"8RoqWpNiUsD22iE9g8QxK2OboRktidveN0Fz27cppQ0bywvWpCX0pEjbi0mxWYlrZmQiXBBH77dGKIzb7oBMCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T15:10:07.057814Z"},"content_sha256":"f3fc77e42b86d31b4614dd81d4f55c7ae3c51390e482a0567df297feb84f6ba2","schema_version":"1.0","event_id":"sha256:f3fc77e42b86d31b4614dd81d4f55c7ae3c51390e482a0567df297feb84f6ba2"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/JZH7WRVNENCYUECFHC3N3BNUZV/bundle.json","state_url":"https://pith.science/pith/JZH7WRVNENCYUECFHC3N3BNUZV/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/JZH7WRVNENCYUECFHC3N3BNUZV/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T15:10:07Z","links":{"resolver":"https://pith.science/pith/JZH7WRVNENCYUECFHC3N3BNUZV","bundle":"https://pith.science/pith/JZH7WRVNENCYUECFHC3N3BNUZV/bundle.json","state":"https://pith.science/pith/JZH7WRVNENCYUECFHC3N3BNUZV/state.json","well_known_bundle":"https://pith.science/.well-known/pith/JZH7WRVNENCYUECFHC3N3BNUZV/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:JZH7WRVNENCYUECFHC3N3BNUZV","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0dfa9387d6a4c0b36e20f955c2e0fc5fc449859393bcad256a1e3823de339920","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-04-16T18:57:19Z","title_canon_sha256":"9abbbb5566b1dbee5b097381c40ea3d20d79c6ec44cc137255fc9e92ba8430fd"},"schema_version":"1.0","source":{"id":"2104.08315","kind":"arxiv","version":9}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2104.08315","created_at":"2026-07-05T05:17:42Z"},{"alias_kind":"arxiv_version","alias_value":"2104.08315v9","created_at":"2026-07-05T05:17:42Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2104.08315","created_at":"2026-07-05T05:17:42Z"},{"alias_kind":"pith_short_12","alias_value":"JZH7WRVNENCY","created_at":"2026-07-05T05:17:42Z"},{"alias_kind":"pith_short_16","alias_value":"JZH7WRVNENCYUECF","created_at":"2026-07-05T05:17:42Z"},{"alias_kind":"pith_short_8","alias_value":"JZH7WRVN","created_at":"2026-07-05T05:17:42Z"}],"graph_snapshots":[{"event_id":"sha256:f3fc77e42b86d31b4614dd81d4f55c7ae3c51390e482a0567df297feb84f6ba2","target":"graph","created_at":"2026-07-05T05:17:42Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2104.08315/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large language models have shown promising results in zero-shot settings (Brown et al.,2020; Radford et al., 2019). For example, they can perform multiple choice tasks simply by conditioning on a question and selecting the answer with the highest probability.\n  However, ranking by string probability can be problematic due to surface form competition-wherein different surface forms compete for probability mass, even if they represent the same underlying concept, e.g. \"computer\" and \"PC.\" Since probability mass is finite, this lowers the probability of the correct answer, due to competition from","authors_text":"Ari Holtzman, Luke Zettlemoyer, Peter West, Vered Shwartz, Yejin Choi","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-04-16T18:57:19Z","title":"Surface Form Competition: Why the Highest Probability Answer Isn't Always Right"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2104.08315","kind":"arxiv","version":9},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:6e5b7537d4497b280f16fbc93d3b4f8052e996816dd4a471538248289bbb391e","target":"record","created_at":"2026-07-05T05:17:42Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0dfa9387d6a4c0b36e20f955c2e0fc5fc449859393bcad256a1e3823de339920","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-04-16T18:57:19Z","title_canon_sha256":"9abbbb5566b1dbee5b097381c40ea3d20d79c6ec44cc137255fc9e92ba8430fd"},"schema_version":"1.0","source":{"id":"2104.08315","kind":"arxiv","version":9}},"canonical_sha256":"4e4ffb46ad23458a104538b6dd85b4cd7759775b7049db33c95600328b3120d5","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"4e4ffb46ad23458a104538b6dd85b4cd7759775b7049db33c95600328b3120d5","first_computed_at":"2026-07-05T05:17:42.978894Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:17:42.978894Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"JmSk7PDpSwltr2lhxe1jbULwjG0EoWMS6X1bF0BiXDNYeqD2j0IUYhwt5l4SZ/GTM9mESrY96JQUGbeEKZZXCg==","signature_status":"signed_v1","signed_at":"2026-07-05T05:17:42.979420Z","signed_message":"canonical_sha256_bytes"},"source_id":"2104.08315","source_kind":"arxiv","source_version":9}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:6e5b7537d4497b280f16fbc93d3b4f8052e996816dd4a471538248289bbb391e","sha256:f3fc77e42b86d31b4614dd81d4f55c7ae3c51390e482a0567df297feb84f6ba2"],"state_sha256":"51771702f328ea159d013e0187abc524edc61a8ba6e3f5026c28032a826a7960"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"IrXS0lt8PVWy+FXx83pZUT559KcSuYCOTJwvbTNs6C0+zE3tM/T6RvT9/7nd4F/ghnCIpDwwum1vDS5guEr0BA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T15:10:07.060387Z","bundle_sha256":"8ed76bd13e43d59522adbd4d90d824f11926ae1c9b9ee36192b7e0c6e076f87f"}}