{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:BG3LMT3YUZC542PSLMG7MQDAR2","short_pith_number":"pith:BG3LMT3Y","canonical_record":{"source":{"id":"2309.13701","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-09-24T17:15:58Z","cross_cats_sorted":["cs.AI","cs.HC"],"title_canon_sha256":"30b15fae4c90eecbeb6b21529a75e8cf18b4fbac1fc2594e4f0d5ce2abcb6cf2","abstract_canon_sha256":"4c81e67796defed3e4ae25aea2022bbc7d4041c92725f3664f7d3c4575414ccb"},"schema_version":"1.0"},"canonical_sha256":"09b6b64f78a645de69f25b0df640608e8e39245a1978ae6275006f8199ac4f5a","source":{"kind":"arxiv","id":"2309.13701","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.13701","created_at":"2026-07-05T06:54:41Z"},{"alias_kind":"arxiv_version","alias_value":"2309.13701v2","created_at":"2026-07-05T06:54:41Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.13701","created_at":"2026-07-05T06:54:41Z"},{"alias_kind":"pith_short_12","alias_value":"BG3LMT3YUZC5","created_at":"2026-07-05T06:54:41Z"},{"alias_kind":"pith_short_16","alias_value":"BG3LMT3YUZC542PS","created_at":"2026-07-05T06:54:41Z"},{"alias_kind":"pith_short_8","alias_value":"BG3LMT3Y","created_at":"2026-07-05T06:54:41Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:BG3LMT3YUZC542PSLMG7MQDAR2","target":"record","payload":{"canonical_record":{"source":{"id":"2309.13701","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-09-24T17:15:58Z","cross_cats_sorted":["cs.AI","cs.HC"],"title_canon_sha256":"30b15fae4c90eecbeb6b21529a75e8cf18b4fbac1fc2594e4f0d5ce2abcb6cf2","abstract_canon_sha256":"4c81e67796defed3e4ae25aea2022bbc7d4041c92725f3664f7d3c4575414ccb"},"schema_version":"1.0"},"canonical_sha256":"09b6b64f78a645de69f25b0df640608e8e39245a1978ae6275006f8199ac4f5a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:54:41.006923Z","signature_b64":"TB9GVZSNMZuXF63H4xZb/bsDeadb+HXDh0pR54XpzCBkSosOvZnDeqe3/+9SAApLzHCiftz3nuWRvhJIYgMGCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"09b6b64f78a645de69f25b0df640608e8e39245a1978ae6275006f8199ac4f5a","last_reissued_at":"2026-07-05T06:54:41.006427Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:54:41.006427Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2309.13701","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:54:41Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"8X0NwYRGIOm7gMhFgWqoADA21TJKZ8iO8AVHz3WfzB97bErXYRzQfdhGj+IUyFgv+/fSVUgDi7XrMQCqeGuZCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-17T19:38:46.631771Z"},"content_sha256":"af0974f8dfcec516c714e9d6eb5709118da3bcc5c9e9d04637ed65e4e1fa71c7","schema_version":"1.0","event_id":"sha256:af0974f8dfcec516c714e9d6eb5709118da3bcc5c9e9d04637ed65e4e1fa71c7"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:BG3LMT3YUZC542PSLMG7MQDAR2","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"ALLURE: Auditing and Improving LLM-based Evaluation of Text using Iterative In-Context-Learning","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.HC"],"primary_cat":"cs.CL","authors_text":"Felipe Vieira Frujeri, Hiteshi Sharma, Hosein Hasanbeig, Ida Momennejad, Leo Betthauser","submitted_at":"2023-09-24T17:15:58Z","abstract_excerpt":"From grading papers to summarizing medical documents, large language models (LLMs) are evermore used for evaluation of text generated by humans and AI alike. However, despite their extensive utility, LLMs exhibit distinct failure modes, necessitating a thorough audit and improvement of their text evaluation capabilities. Here we introduce ALLURE, a systematic approach to Auditing Large Language Models Understanding and Reasoning Errors. ALLURE involves comparing LLM-generated evaluations with annotated data, and iteratively incorporating instances of significant deviation into the evaluator, w"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.13701","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2309.13701/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:54:41Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"2Dg7optv6JHd8vbe+S+2nFFSvZWisgP5l5u2s7te6b38Qlu0NbzcpH47BR7I9feZWCn/XXiSuFURYXW/ByfGCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-17T19:38:46.632266Z"},"content_sha256":"8f64eda73a025b89b7a5c0aa34cab51144cb86379dd6e464aae38d672521a437","schema_version":"1.0","event_id":"sha256:8f64eda73a025b89b7a5c0aa34cab51144cb86379dd6e464aae38d672521a437"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/BG3LMT3YUZC542PSLMG7MQDAR2/bundle.json","state_url":"https://pith.science/pith/BG3LMT3YUZC542PSLMG7MQDAR2/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/BG3LMT3YUZC542PSLMG7MQDAR2/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-17T19:38:46Z","links":{"resolver":"https://pith.science/pith/BG3LMT3YUZC542PSLMG7MQDAR2","bundle":"https://pith.science/pith/BG3LMT3YUZC542PSLMG7MQDAR2/bundle.json","state":"https://pith.science/pith/BG3LMT3YUZC542PSLMG7MQDAR2/state.json","well_known_bundle":"https://pith.science/.well-known/pith/BG3LMT3YUZC542PSLMG7MQDAR2/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:BG3LMT3YUZC542PSLMG7MQDAR2","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"4c81e67796defed3e4ae25aea2022bbc7d4041c92725f3664f7d3c4575414ccb","cross_cats_sorted":["cs.AI","cs.HC"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-09-24T17:15:58Z","title_canon_sha256":"30b15fae4c90eecbeb6b21529a75e8cf18b4fbac1fc2594e4f0d5ce2abcb6cf2"},"schema_version":"1.0","source":{"id":"2309.13701","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.13701","created_at":"2026-07-05T06:54:41Z"},{"alias_kind":"arxiv_version","alias_value":"2309.13701v2","created_at":"2026-07-05T06:54:41Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.13701","created_at":"2026-07-05T06:54:41Z"},{"alias_kind":"pith_short_12","alias_value":"BG3LMT3YUZC5","created_at":"2026-07-05T06:54:41Z"},{"alias_kind":"pith_short_16","alias_value":"BG3LMT3YUZC542PS","created_at":"2026-07-05T06:54:41Z"},{"alias_kind":"pith_short_8","alias_value":"BG3LMT3Y","created_at":"2026-07-05T06:54:41Z"}],"graph_snapshots":[{"event_id":"sha256:8f64eda73a025b89b7a5c0aa34cab51144cb86379dd6e464aae38d672521a437","target":"graph","created_at":"2026-07-05T06:54:41Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2309.13701/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"From grading papers to summarizing medical documents, large language models (LLMs) are evermore used for evaluation of text generated by humans and AI alike. However, despite their extensive utility, LLMs exhibit distinct failure modes, necessitating a thorough audit and improvement of their text evaluation capabilities. Here we introduce ALLURE, a systematic approach to Auditing Large Language Models Understanding and Reasoning Errors. ALLURE involves comparing LLM-generated evaluations with annotated data, and iteratively incorporating instances of significant deviation into the evaluator, w","authors_text":"Felipe Vieira Frujeri, Hiteshi Sharma, Hosein Hasanbeig, Ida Momennejad, Leo Betthauser","cross_cats":["cs.AI","cs.HC"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-09-24T17:15:58Z","title":"ALLURE: Auditing and Improving LLM-based Evaluation of Text using Iterative In-Context-Learning"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.13701","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:af0974f8dfcec516c714e9d6eb5709118da3bcc5c9e9d04637ed65e4e1fa71c7","target":"record","created_at":"2026-07-05T06:54:41Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"4c81e67796defed3e4ae25aea2022bbc7d4041c92725f3664f7d3c4575414ccb","cross_cats_sorted":["cs.AI","cs.HC"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-09-24T17:15:58Z","title_canon_sha256":"30b15fae4c90eecbeb6b21529a75e8cf18b4fbac1fc2594e4f0d5ce2abcb6cf2"},"schema_version":"1.0","source":{"id":"2309.13701","kind":"arxiv","version":2}},"canonical_sha256":"09b6b64f78a645de69f25b0df640608e8e39245a1978ae6275006f8199ac4f5a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"09b6b64f78a645de69f25b0df640608e8e39245a1978ae6275006f8199ac4f5a","first_computed_at":"2026-07-05T06:54:41.006427Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:54:41.006427Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"TB9GVZSNMZuXF63H4xZb/bsDeadb+HXDh0pR54XpzCBkSosOvZnDeqe3/+9SAApLzHCiftz3nuWRvhJIYgMGCw==","signature_status":"signed_v1","signed_at":"2026-07-05T06:54:41.006923Z","signed_message":"canonical_sha256_bytes"},"source_id":"2309.13701","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:af0974f8dfcec516c714e9d6eb5709118da3bcc5c9e9d04637ed65e4e1fa71c7","sha256:8f64eda73a025b89b7a5c0aa34cab51144cb86379dd6e464aae38d672521a437"],"state_sha256":"3f51b490d79d61f63601900947d1680e4872724088b9b3c72fb75be8a9a6cf0b"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"pgL90XAzaOMOWH7q3gEQgOdHXAbU/Aff/cvbN3mq03oUeCc3rvIvzGi7+dVdYU3aUiZG564EQT5h2lRdPdXGAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-17T19:38:46.637604Z","bundle_sha256":"d7f7a4c0876aba6454602c59966726d220cd3ba6f55d67f1c94b11f0cabb0e54"}}