{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:VHWI6HUUAKTWJHXHHTMRLXPJVP","short_pith_number":"pith:VHWI6HUU","canonical_record":{"source":{"id":"2305.13264","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-22T17:33:17Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"150bf013349a280e37683433fdca72c7416cabfa2b5b42d933d1b97d64b3b712","abstract_canon_sha256":"e9724a1137a8f474075b15145a092cbb295c3ddae2389785792a2237c47f5361"},"schema_version":"1.0"},"canonical_sha256":"a9ec8f1e9402a7649ee73cd915dde9abcf1e221966321fc35589bae21c0eb7e7","source":{"kind":"arxiv","id":"2305.13264","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.13264","created_at":"2026-07-05T07:03:58Z"},{"alias_kind":"arxiv_version","alias_value":"2305.13264v2","created_at":"2026-07-05T07:03:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.13264","created_at":"2026-07-05T07:03:58Z"},{"alias_kind":"pith_short_12","alias_value":"VHWI6HUUAKTW","created_at":"2026-07-05T07:03:58Z"},{"alias_kind":"pith_short_16","alias_value":"VHWI6HUUAKTWJHXH","created_at":"2026-07-05T07:03:58Z"},{"alias_kind":"pith_short_8","alias_value":"VHWI6HUU","created_at":"2026-07-05T07:03:58Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:VHWI6HUUAKTWJHXHHTMRLXPJVP","target":"record","payload":{"canonical_record":{"source":{"id":"2305.13264","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-22T17:33:17Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"150bf013349a280e37683433fdca72c7416cabfa2b5b42d933d1b97d64b3b712","abstract_canon_sha256":"e9724a1137a8f474075b15145a092cbb295c3ddae2389785792a2237c47f5361"},"schema_version":"1.0"},"canonical_sha256":"a9ec8f1e9402a7649ee73cd915dde9abcf1e221966321fc35589bae21c0eb7e7","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:03:58.951850Z","signature_b64":"equNJnFSNZ95OYRHV2WUQQ+XmoKAmxVAcn0rNGYz+tyCpKATZ5jlpIfMqPIIcynRi6OyS6PgOU/iK1OLhc0jDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a9ec8f1e9402a7649ee73cd915dde9abcf1e221966321fc35589bae21c0eb7e7","last_reissued_at":"2026-07-05T07:03:58.951310Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:03:58.951310Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2305.13264","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:03:58Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"nP/EbohWUTNKc/ZCt7+psXzvloNEsMsa/xqshOFInY1On4G/vHlBgdsrDdMIx1KTzZWCKjS+3II9g4ZlPXMECw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T12:06:34.635850Z"},"content_sha256":"a9f70d20f7eb2360c1a7c407aa00eaa303bec8321a8ff99fcba9935b67bb2c94","schema_version":"1.0","event_id":"sha256:a9f70d20f7eb2360c1a7c407aa00eaa303bec8321a8ff99fcba9935b67bb2c94"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:VHWI6HUUAKTWJHXHHTMRLXPJVP","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Prompting is not a substitute for probability measurements in large language models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Jennifer Hu, Roger Levy","submitted_at":"2023-05-22T17:33:17Z","abstract_excerpt":"Prompting is now a dominant method for evaluating the linguistic knowledge of large language models (LLMs). While other methods directly read out models' probability distributions over strings, prompting requires models to access this internal information by processing linguistic input, thereby implicitly testing a new type of emergent ability: metalinguistic judgment. In this study, we compare metalinguistic prompting and direct probability measurements as ways of measuring models' linguistic knowledge. Broadly, we find that LLMs' metalinguistic judgments are inferior to quantities directly d"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.13264","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2305.13264/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:03:58Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ArgCEaBM024DJvFyXSVAazMLzdRrGQSqWQxd0T0nVuhNqhjdIVtCjWcCtqLgM0MejORSwqu16M/lq+SlBT5ICw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T12:06:34.636372Z"},"content_sha256":"b481c92dd487448f05d8276611dbffeb444700455029620e9a086f281d212156","schema_version":"1.0","event_id":"sha256:b481c92dd487448f05d8276611dbffeb444700455029620e9a086f281d212156"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/VHWI6HUUAKTWJHXHHTMRLXPJVP/bundle.json","state_url":"https://pith.science/pith/VHWI6HUUAKTWJHXHHTMRLXPJVP/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/VHWI6HUUAKTWJHXHHTMRLXPJVP/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-10T12:06:34Z","links":{"resolver":"https://pith.science/pith/VHWI6HUUAKTWJHXHHTMRLXPJVP","bundle":"https://pith.science/pith/VHWI6HUUAKTWJHXHHTMRLXPJVP/bundle.json","state":"https://pith.science/pith/VHWI6HUUAKTWJHXHHTMRLXPJVP/state.json","well_known_bundle":"https://pith.science/.well-known/pith/VHWI6HUUAKTWJHXHHTMRLXPJVP/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:VHWI6HUUAKTWJHXHHTMRLXPJVP","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"e9724a1137a8f474075b15145a092cbb295c3ddae2389785792a2237c47f5361","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-22T17:33:17Z","title_canon_sha256":"150bf013349a280e37683433fdca72c7416cabfa2b5b42d933d1b97d64b3b712"},"schema_version":"1.0","source":{"id":"2305.13264","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.13264","created_at":"2026-07-05T07:03:58Z"},{"alias_kind":"arxiv_version","alias_value":"2305.13264v2","created_at":"2026-07-05T07:03:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.13264","created_at":"2026-07-05T07:03:58Z"},{"alias_kind":"pith_short_12","alias_value":"VHWI6HUUAKTW","created_at":"2026-07-05T07:03:58Z"},{"alias_kind":"pith_short_16","alias_value":"VHWI6HUUAKTWJHXH","created_at":"2026-07-05T07:03:58Z"},{"alias_kind":"pith_short_8","alias_value":"VHWI6HUU","created_at":"2026-07-05T07:03:58Z"}],"graph_snapshots":[{"event_id":"sha256:b481c92dd487448f05d8276611dbffeb444700455029620e9a086f281d212156","target":"graph","created_at":"2026-07-05T07:03:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2305.13264/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Prompting is now a dominant method for evaluating the linguistic knowledge of large language models (LLMs). While other methods directly read out models' probability distributions over strings, prompting requires models to access this internal information by processing linguistic input, thereby implicitly testing a new type of emergent ability: metalinguistic judgment. In this study, we compare metalinguistic prompting and direct probability measurements as ways of measuring models' linguistic knowledge. Broadly, we find that LLMs' metalinguistic judgments are inferior to quantities directly d","authors_text":"Jennifer Hu, Roger Levy","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-22T17:33:17Z","title":"Prompting is not a substitute for probability measurements in large language models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.13264","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:a9f70d20f7eb2360c1a7c407aa00eaa303bec8321a8ff99fcba9935b67bb2c94","target":"record","created_at":"2026-07-05T07:03:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"e9724a1137a8f474075b15145a092cbb295c3ddae2389785792a2237c47f5361","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-05-22T17:33:17Z","title_canon_sha256":"150bf013349a280e37683433fdca72c7416cabfa2b5b42d933d1b97d64b3b712"},"schema_version":"1.0","source":{"id":"2305.13264","kind":"arxiv","version":2}},"canonical_sha256":"a9ec8f1e9402a7649ee73cd915dde9abcf1e221966321fc35589bae21c0eb7e7","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a9ec8f1e9402a7649ee73cd915dde9abcf1e221966321fc35589bae21c0eb7e7","first_computed_at":"2026-07-05T07:03:58.951310Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:03:58.951310Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"equNJnFSNZ95OYRHV2WUQQ+XmoKAmxVAcn0rNGYz+tyCpKATZ5jlpIfMqPIIcynRi6OyS6PgOU/iK1OLhc0jDA==","signature_status":"signed_v1","signed_at":"2026-07-05T07:03:58.951850Z","signed_message":"canonical_sha256_bytes"},"source_id":"2305.13264","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:a9f70d20f7eb2360c1a7c407aa00eaa303bec8321a8ff99fcba9935b67bb2c94","sha256:b481c92dd487448f05d8276611dbffeb444700455029620e9a086f281d212156"],"state_sha256":"dbbaf6a46e2a74ac188e3a0b689f186cedf26c785c371ddfa13cd820a9fd4f45"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"znZAnaSQlCufWZwckjncg2WFCiFo1vTJSh4XlcO3ZV8pjXVGCqL1eviN/EQg4UIpdkZiBYL57pt9W97l3jerBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-10T12:06:34.641400Z","bundle_sha256":"fbc3cce9e4d0e1414bd9f4be644f4b91e9d0a24df5e0b54e4938e9d51f9f8ac2"}}