{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2021:SCYXWWAXDWIWOLTOON6HA763NA","short_pith_number":"pith:SCYXWWAX","schema_version":"1.0","canonical_sha256":"90b17b58171d91672e6e737c707fdb6811625b8d5c23d8d6a55d4fa5d4d0eb59","source":{"kind":"arxiv","id":"2108.06665","version":1},"attestation_state":"computed","paper":{"title":"Accurate, yet inconsistent? Consistency Analysis on Language Understanding Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Deuk Sin Kwon, Myeongjun Jang, Thomas Lukasiewicz","submitted_at":"2021-08-15T06:25:07Z","abstract_excerpt":"Consistency, which refers to the capability of generating the same predictions for semantically similar contexts, is a highly desirable property for a sound language understanding model. Although recent pretrained language models (PLMs) deliver outstanding performance in various downstream tasks, they should exhibit consistent behaviour provided the models truly understand language. In this paper, we propose a simple framework named consistency analysis on language understanding models (CALUM)} to evaluate the model's lower-bound consistency ability. Through experiments, we confirmed that curr"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2108.06665","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-08-15T06:25:07Z","cross_cats_sorted":[],"title_canon_sha256":"9bab56090b85b1500b2e11c88de4e31f649f6a5a8321e35e0d252318013d44b1","abstract_canon_sha256":"45d88c5010de0a78a06fd9023b933a77b34aef5faaa235ccb959f681f0064cae"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T03:05:56.803635Z","signature_b64":"1oKIHeYZLYCGykqXUCvuePiHzWJrnnCuMjZ30qQqySc3wqOB8ArJNtgSEdz0fNunS2pWYTLFsFHOrxTtH/arCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"90b17b58171d91672e6e737c707fdb6811625b8d5c23d8d6a55d4fa5d4d0eb59","last_reissued_at":"2026-07-05T03:05:56.803287Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T03:05:56.803287Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Accurate, yet inconsistent? Consistency Analysis on Language Understanding Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Deuk Sin Kwon, Myeongjun Jang, Thomas Lukasiewicz","submitted_at":"2021-08-15T06:25:07Z","abstract_excerpt":"Consistency, which refers to the capability of generating the same predictions for semantically similar contexts, is a highly desirable property for a sound language understanding model. Although recent pretrained language models (PLMs) deliver outstanding performance in various downstream tasks, they should exhibit consistent behaviour provided the models truly understand language. In this paper, we propose a simple framework named consistency analysis on language understanding models (CALUM)} to evaluate the model's lower-bound consistency ability. Through experiments, we confirmed that curr"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2108.06665","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2108.06665/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2108.06665","created_at":"2026-07-05T03:05:56.803354+00:00"},{"alias_kind":"arxiv_version","alias_value":"2108.06665v1","created_at":"2026-07-05T03:05:56.803354+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2108.06665","created_at":"2026-07-05T03:05:56.803354+00:00"},{"alias_kind":"pith_short_12","alias_value":"SCYXWWAXDWIW","created_at":"2026-07-05T03:05:56.803354+00:00"},{"alias_kind":"pith_short_16","alias_value":"SCYXWWAXDWIWOLTO","created_at":"2026-07-05T03:05:56.803354+00:00"},{"alias_kind":"pith_short_8","alias_value":"SCYXWWAX","created_at":"2026-07-05T03:05:56.803354+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2506.21521","citing_title":"Potemkin Understanding in Large Language Models","ref_index":28,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/SCYXWWAXDWIWOLTOON6HA763NA","json":"https://pith.science/pith/SCYXWWAXDWIWOLTOON6HA763NA.json","graph_json":"https://pith.science/api/pith-number/SCYXWWAXDWIWOLTOON6HA763NA/graph.json","events_json":"https://pith.science/api/pith-number/SCYXWWAXDWIWOLTOON6HA763NA/events.json","paper":"https://pith.science/paper/SCYXWWAX"},"agent_actions":{"view_html":"https://pith.science/pith/SCYXWWAXDWIWOLTOON6HA763NA","download_json":"https://pith.science/pith/SCYXWWAXDWIWOLTOON6HA763NA.json","view_paper":"https://pith.science/paper/SCYXWWAX","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2108.06665&json=true","fetch_graph":"https://pith.science/api/pith-number/SCYXWWAXDWIWOLTOON6HA763NA/graph.json","fetch_events":"https://pith.science/api/pith-number/SCYXWWAXDWIWOLTOON6HA763NA/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/SCYXWWAXDWIWOLTOON6HA763NA/action/timestamp_anchor","attest_storage":"https://pith.science/pith/SCYXWWAXDWIWOLTOON6HA763NA/action/storage_attestation","attest_author":"https://pith.science/pith/SCYXWWAXDWIWOLTOON6HA763NA/action/author_attestation","sign_citation":"https://pith.science/pith/SCYXWWAXDWIWOLTOON6HA763NA/action/citation_signature","submit_replication":"https://pith.science/pith/SCYXWWAXDWIWOLTOON6HA763NA/action/replication_record"}},"created_at":"2026-07-05T03:05:56.803354+00:00","updated_at":"2026-07-05T03:05:56.803354+00:00"}