{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:2ZH2OD77L3FSKRWRYLFFLYOJ5Y","short_pith_number":"pith:2ZH2OD77","schema_version":"1.0","canonical_sha256":"d64fa70fff5ecb2546d1c2ca55e1c9ee133be3f0c7e7bc465d0a078b396f1934","source":{"kind":"arxiv","id":"2305.06311","version":2},"attestation_state":"computed","paper":{"title":"Automatic Evaluation of Attribution by Large Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Boshi Wang, Huan Sun, Kai Zhang, Xiang Yue, Yu Su, Ziru Chen","submitted_at":"2023-05-10T16:58:33Z","abstract_excerpt":"A recent focus of large language model (LLM) development, as exemplified by generative search engines, is to incorporate external references to generate and support its claims. However, evaluating the attribution, i.e., verifying whether the generated statement is fully supported by the cited reference, remains an open problem. Although human evaluation is common practice, it is costly and time-consuming. In this paper, we investigate the automatic evaluation of attribution given by LLMs. We begin by defining different types of attribution errors, and then explore two approaches for automatic "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2305.06311","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-10T16:58:33Z","cross_cats_sorted":[],"title_canon_sha256":"9e323b346edf8c7ba0f8c06972f41a5eca96add43fcc758da55b5ddf78be6e00","abstract_canon_sha256":"690f4f61898babdc4f325ec4e6373f6c8a43f56b02b133033e6905a4df0dd559"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:58:09.762639Z","signature_b64":"0CcvOi2ppBaQWOIMiRuG5e31XJM1MlceJfK8VUOPrZ5wODzhgObeksv3Yin10VF/LGDJTCZ+OdJVg8jC684CDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d64fa70fff5ecb2546d1c2ca55e1c9ee133be3f0c7e7bc465d0a078b396f1934","last_reissued_at":"2026-07-05T06:58:09.762075Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:58:09.762075Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Automatic Evaluation of Attribution by Large Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Boshi Wang, Huan Sun, Kai Zhang, Xiang Yue, Yu Su, Ziru Chen","submitted_at":"2023-05-10T16:58:33Z","abstract_excerpt":"A recent focus of large language model (LLM) development, as exemplified by generative search engines, is to incorporate external references to generate and support its claims. However, evaluating the attribution, i.e., verifying whether the generated statement is fully supported by the cited reference, remains an open problem. Although human evaluation is common practice, it is costly and time-consuming. In this paper, we investigate the automatic evaluation of attribution given by LLMs. We begin by defining different types of attribution errors, and then explore two approaches for automatic "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.06311","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2305.06311/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2305.06311","created_at":"2026-07-05T06:58:09.762139+00:00"},{"alias_kind":"arxiv_version","alias_value":"2305.06311v2","created_at":"2026-07-05T06:58:09.762139+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.06311","created_at":"2026-07-05T06:58:09.762139+00:00"},{"alias_kind":"pith_short_12","alias_value":"2ZH2OD77L3FS","created_at":"2026-07-05T06:58:09.762139+00:00"},{"alias_kind":"pith_short_16","alias_value":"2ZH2OD77L3FSKRWR","created_at":"2026-07-05T06:58:09.762139+00:00"},{"alias_kind":"pith_short_8","alias_value":"2ZH2OD77","created_at":"2026-07-05T06:58:09.762139+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":4,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2605.02443","citing_title":"HalluScan: A Systematic Benchmark for Detecting and Mitigating Hallucinations in Instruction-Following LLMs","ref_index":45,"is_internal_anchor":false},{"citing_arxiv_id":"2304.12244","citing_title":"WizardLM: Empowering large pre-trained language models to follow complex instructions","ref_index":50,"is_internal_anchor":false},{"citing_arxiv_id":"2310.11511","citing_title":"Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection","ref_index":170,"is_internal_anchor":false},{"citing_arxiv_id":"2412.05579","citing_title":"LLMs-as-Judges: A Comprehensive Survey on LLM-based Evaluation Methods","ref_index":287,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/2ZH2OD77L3FSKRWRYLFFLYOJ5Y","json":"https://pith.science/pith/2ZH2OD77L3FSKRWRYLFFLYOJ5Y.json","graph_json":"https://pith.science/api/pith-number/2ZH2OD77L3FSKRWRYLFFLYOJ5Y/graph.json","events_json":"https://pith.science/api/pith-number/2ZH2OD77L3FSKRWRYLFFLYOJ5Y/events.json","paper":"https://pith.science/paper/2ZH2OD77"},"agent_actions":{"view_html":"https://pith.science/pith/2ZH2OD77L3FSKRWRYLFFLYOJ5Y","download_json":"https://pith.science/pith/2ZH2OD77L3FSKRWRYLFFLYOJ5Y.json","view_paper":"https://pith.science/paper/2ZH2OD77","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2305.06311&json=true","fetch_graph":"https://pith.science/api/pith-number/2ZH2OD77L3FSKRWRYLFFLYOJ5Y/graph.json","fetch_events":"https://pith.science/api/pith-number/2ZH2OD77L3FSKRWRYLFFLYOJ5Y/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/2ZH2OD77L3FSKRWRYLFFLYOJ5Y/action/timestamp_anchor","attest_storage":"https://pith.science/pith/2ZH2OD77L3FSKRWRYLFFLYOJ5Y/action/storage_attestation","attest_author":"https://pith.science/pith/2ZH2OD77L3FSKRWRYLFFLYOJ5Y/action/author_attestation","sign_citation":"https://pith.science/pith/2ZH2OD77L3FSKRWRYLFFLYOJ5Y/action/citation_signature","submit_replication":"https://pith.science/pith/2ZH2OD77L3FSKRWRYLFFLYOJ5Y/action/replication_record"}},"created_at":"2026-07-05T06:58:09.762139+00:00","updated_at":"2026-07-05T06:58:09.762139+00:00"}