{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:FYPMCGBIWJVQZ4H5OZLVBHV3SB","short_pith_number":"pith:FYPMCGBI","canonical_record":{"source":{"id":"2311.01152","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-02T11:27:21Z","cross_cats_sorted":[],"title_canon_sha256":"9f2b1d4b4389378ae3894fc22ddabbb883224f16d34e621f89ebfa92ae958b8e","abstract_canon_sha256":"67469165d5d1ed75f22a35237314e4d2bcefc794ecfa02f68e9ba9aa40aa497d"},"schema_version":"1.0"},"canonical_sha256":"2e1ec11828b26b0cf0fd7657509ebb90604a52084bb1b4018b540dc3a6dccfd8","source":{"kind":"arxiv","id":"2311.01152","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2311.01152","created_at":"2026-07-05T07:08:21Z"},{"alias_kind":"arxiv_version","alias_value":"2311.01152v1","created_at":"2026-07-05T07:08:21Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2311.01152","created_at":"2026-07-05T07:08:21Z"},{"alias_kind":"pith_short_12","alias_value":"FYPMCGBIWJVQ","created_at":"2026-07-05T07:08:21Z"},{"alias_kind":"pith_short_16","alias_value":"FYPMCGBIWJVQZ4H5","created_at":"2026-07-05T07:08:21Z"},{"alias_kind":"pith_short_8","alias_value":"FYPMCGBI","created_at":"2026-07-05T07:08:21Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:FYPMCGBIWJVQZ4H5OZLVBHV3SB","target":"record","payload":{"canonical_record":{"source":{"id":"2311.01152","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-02T11:27:21Z","cross_cats_sorted":[],"title_canon_sha256":"9f2b1d4b4389378ae3894fc22ddabbb883224f16d34e621f89ebfa92ae958b8e","abstract_canon_sha256":"67469165d5d1ed75f22a35237314e4d2bcefc794ecfa02f68e9ba9aa40aa497d"},"schema_version":"1.0"},"canonical_sha256":"2e1ec11828b26b0cf0fd7657509ebb90604a52084bb1b4018b540dc3a6dccfd8","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:08:21.573026Z","signature_b64":"BNLpmG9C8708iPb0VWdsIM1W8GtZpupYtVF06EyxlgEryV19EjF/Fbb8jMmRKRtlZqhdLV5OCGawvXNbvvy1CA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2e1ec11828b26b0cf0fd7657509ebb90604a52084bb1b4018b540dc3a6dccfd8","last_reissued_at":"2026-07-05T07:08:21.572573Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:08:21.572573Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2311.01152","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:08:21Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"gvurSmz+3d78PR9E9oNDxCNqkWefRnGv2DzC/C5U+09uAbgeulRjtnljYrkn8LSPv2HvrrxsDCtD8xRqvcxqDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-23T04:25:20.458424Z"},"content_sha256":"cc01c57b61d68170abc4f1b1c93fd14a6cfe41683f52645dec3601096780b5a4","schema_version":"1.0","event_id":"sha256:cc01c57b61d68170abc4f1b1c93fd14a6cfe41683f52645dec3601096780b5a4"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:FYPMCGBIWJVQZ4H5OZLVBHV3SB","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Predicting Question-Answering Performance of Large Language Models through Semantic Consistency","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Ateret Anaby-Tavor, Eitan Farchi, Ella Rabinovich, Orna Raz, Samuel Ackerman","submitted_at":"2023-11-02T11:27:21Z","abstract_excerpt":"Semantic consistency of a language model is broadly defined as the model's ability to produce semantically-equivalent outputs, given semantically-equivalent inputs. We address the task of assessing question-answering (QA) semantic consistency of contemporary large language models (LLMs) by manually creating a benchmark dataset with high-quality paraphrases for factual questions, and release the dataset to the community.\n  We further combine the semantic consistency metric with additional measurements suggested in prior work as correlating with LLM QA accuracy, for building and evaluating a fra"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2311.01152","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2311.01152/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:08:21Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"77bF1OQZ6M7PFOuk1/+DjZw0cSmQR/lqMZPfZJNGwLeg43/sOkcYhQM1wLCVGJmQSbrSYWk1CiwgCE1oPVijCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-23T04:25:20.459046Z"},"content_sha256":"c8d40c2819bb8267d191746e2ce5722343be6f853eeb40f7156afe7f94ca7ece","schema_version":"1.0","event_id":"sha256:c8d40c2819bb8267d191746e2ce5722343be6f853eeb40f7156afe7f94ca7ece"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/FYPMCGBIWJVQZ4H5OZLVBHV3SB/bundle.json","state_url":"https://pith.science/pith/FYPMCGBIWJVQZ4H5OZLVBHV3SB/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/FYPMCGBIWJVQZ4H5OZLVBHV3SB/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-23T04:25:20Z","links":{"resolver":"https://pith.science/pith/FYPMCGBIWJVQZ4H5OZLVBHV3SB","bundle":"https://pith.science/pith/FYPMCGBIWJVQZ4H5OZLVBHV3SB/bundle.json","state":"https://pith.science/pith/FYPMCGBIWJVQZ4H5OZLVBHV3SB/state.json","well_known_bundle":"https://pith.science/.well-known/pith/FYPMCGBIWJVQZ4H5OZLVBHV3SB/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:FYPMCGBIWJVQZ4H5OZLVBHV3SB","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"67469165d5d1ed75f22a35237314e4d2bcefc794ecfa02f68e9ba9aa40aa497d","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-02T11:27:21Z","title_canon_sha256":"9f2b1d4b4389378ae3894fc22ddabbb883224f16d34e621f89ebfa92ae958b8e"},"schema_version":"1.0","source":{"id":"2311.01152","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2311.01152","created_at":"2026-07-05T07:08:21Z"},{"alias_kind":"arxiv_version","alias_value":"2311.01152v1","created_at":"2026-07-05T07:08:21Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2311.01152","created_at":"2026-07-05T07:08:21Z"},{"alias_kind":"pith_short_12","alias_value":"FYPMCGBIWJVQ","created_at":"2026-07-05T07:08:21Z"},{"alias_kind":"pith_short_16","alias_value":"FYPMCGBIWJVQZ4H5","created_at":"2026-07-05T07:08:21Z"},{"alias_kind":"pith_short_8","alias_value":"FYPMCGBI","created_at":"2026-07-05T07:08:21Z"}],"graph_snapshots":[{"event_id":"sha256:c8d40c2819bb8267d191746e2ce5722343be6f853eeb40f7156afe7f94ca7ece","target":"graph","created_at":"2026-07-05T07:08:21Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2311.01152/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Semantic consistency of a language model is broadly defined as the model's ability to produce semantically-equivalent outputs, given semantically-equivalent inputs. We address the task of assessing question-answering (QA) semantic consistency of contemporary large language models (LLMs) by manually creating a benchmark dataset with high-quality paraphrases for factual questions, and release the dataset to the community.\n  We further combine the semantic consistency metric with additional measurements suggested in prior work as correlating with LLM QA accuracy, for building and evaluating a fra","authors_text":"Ateret Anaby-Tavor, Eitan Farchi, Ella Rabinovich, Orna Raz, Samuel Ackerman","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-02T11:27:21Z","title":"Predicting Question-Answering Performance of Large Language Models through Semantic Consistency"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2311.01152","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:cc01c57b61d68170abc4f1b1c93fd14a6cfe41683f52645dec3601096780b5a4","target":"record","created_at":"2026-07-05T07:08:21Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"67469165d5d1ed75f22a35237314e4d2bcefc794ecfa02f68e9ba9aa40aa497d","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-02T11:27:21Z","title_canon_sha256":"9f2b1d4b4389378ae3894fc22ddabbb883224f16d34e621f89ebfa92ae958b8e"},"schema_version":"1.0","source":{"id":"2311.01152","kind":"arxiv","version":1}},"canonical_sha256":"2e1ec11828b26b0cf0fd7657509ebb90604a52084bb1b4018b540dc3a6dccfd8","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"2e1ec11828b26b0cf0fd7657509ebb90604a52084bb1b4018b540dc3a6dccfd8","first_computed_at":"2026-07-05T07:08:21.572573Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:08:21.572573Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"BNLpmG9C8708iPb0VWdsIM1W8GtZpupYtVF06EyxlgEryV19EjF/Fbb8jMmRKRtlZqhdLV5OCGawvXNbvvy1CA==","signature_status":"signed_v1","signed_at":"2026-07-05T07:08:21.573026Z","signed_message":"canonical_sha256_bytes"},"source_id":"2311.01152","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:cc01c57b61d68170abc4f1b1c93fd14a6cfe41683f52645dec3601096780b5a4","sha256:c8d40c2819bb8267d191746e2ce5722343be6f853eeb40f7156afe7f94ca7ece"],"state_sha256":"5f5d9d7df17ff146ffcfad724b2114e7173d1225571548d9e67a3e560fb0454d"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"mNDDwG7Zq7YrP5OO5lY4OhbjfHPAG69lUMIRKGbKgB5bUpIgsC+yxZB/LbKvZdufboeu8cFoEzyl9rDWd2T4Ag==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-23T04:25:20.464073Z","bundle_sha256":"37a2802e24c8f5388da1e4de513b22667baa20a3669db3d4edfd9c7f9da31be4"}}