{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:PPZALUITMBQBZCEQWOM77YUY7D","short_pith_number":"pith:PPZALUIT","schema_version":"1.0","canonical_sha256":"7bf205d11360601c8890b399ffe298f8f7f19c911c7defb8a32e75a0bf49e857","source":{"kind":"arxiv","id":"2305.13091","version":2},"attestation_state":"computed","paper":{"title":"Large Language Models are Not Yet Human-Level Evaluators for Abstractive Summarization","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Chenhui Shen, Lidong Bing, Liying Cheng, Xuan-Phi Nguyen, Yang You","submitted_at":"2023-05-22T14:58:13Z","abstract_excerpt":"With the recent undeniable advancement in reasoning abilities in large language models (LLMs) like ChatGPT and GPT-4, there is a growing trend for using LLMs on various tasks. One area where LLMs can be employed is as an alternative evaluation metric for complex generative tasks, which generally demands expensive human judges to complement the traditional automatic metrics for various evaluation dimensions such as fluency and consistency. In this work, we conduct extensive analysis to investigate the stability and reliability of LLMs as automatic evaluators for abstractive summarization. We fo"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2305.13091","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-22T14:58:13Z","cross_cats_sorted":[],"title_canon_sha256":"78532b64378858ab8216b8cb715283c2af336bcd54d6240bbf253d549931d0e3","abstract_canon_sha256":"17c206d5f303a9c5c7f5eefdda84b5262bc889adc797c6ccd9595d670e317c16"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:02:50.306187Z","signature_b64":"7XCrBZlnPPgDO57j3hVYIeNdt6fNUXihf1gsVhsOhtQvyNehHqUfAUx04994Wo13+AzvniZL6v2R334RJR8RAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"7bf205d11360601c8890b399ffe298f8f7f19c911c7defb8a32e75a0bf49e857","last_reissued_at":"2026-07-05T07:02:50.305645Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:02:50.305645Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Large Language Models are Not Yet Human-Level Evaluators for Abstractive Summarization","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Chenhui Shen, Lidong Bing, Liying Cheng, Xuan-Phi Nguyen, Yang You","submitted_at":"2023-05-22T14:58:13Z","abstract_excerpt":"With the recent undeniable advancement in reasoning abilities in large language models (LLMs) like ChatGPT and GPT-4, there is a growing trend for using LLMs on various tasks. One area where LLMs can be employed is as an alternative evaluation metric for complex generative tasks, which generally demands expensive human judges to complement the traditional automatic metrics for various evaluation dimensions such as fluency and consistency. In this work, we conduct extensive analysis to investigate the stability and reliability of LLMs as automatic evaluators for abstractive summarization. We fo"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.13091","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2305.13091/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2305.13091","created_at":"2026-07-05T07:02:50.305719+00:00"},{"alias_kind":"arxiv_version","alias_value":"2305.13091v2","created_at":"2026-07-05T07:02:50.305719+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.13091","created_at":"2026-07-05T07:02:50.305719+00:00"},{"alias_kind":"pith_short_12","alias_value":"PPZALUITMBQB","created_at":"2026-07-05T07:02:50.305719+00:00"},{"alias_kind":"pith_short_16","alias_value":"PPZALUITMBQBZCEQ","created_at":"2026-07-05T07:02:50.305719+00:00"},{"alias_kind":"pith_short_8","alias_value":"PPZALUIT","created_at":"2026-07-05T07:02:50.305719+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":4,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.28044","citing_title":"A Tree-of-Thoughts Inspired Hybrid Approach for Legal Case Judgement Summarization using LLMs","ref_index":250,"is_internal_anchor":false},{"citing_arxiv_id":"2311.07911","citing_title":"Instruction-Following Evaluation for Large Language Models","ref_index":19,"is_internal_anchor":false},{"citing_arxiv_id":"2504.20605","citing_title":"TF1-EN-3M: Three Million Synthetic Moral Fables for Training Small, Open Language Models","ref_index":18,"is_internal_anchor":false},{"citing_arxiv_id":"2308.07201","citing_title":"ChatEval: Towards Better LLM-based Evaluators through Multi-Agent Debate","ref_index":22,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/PPZALUITMBQBZCEQWOM77YUY7D","json":"https://pith.science/pith/PPZALUITMBQBZCEQWOM77YUY7D.json","graph_json":"https://pith.science/api/pith-number/PPZALUITMBQBZCEQWOM77YUY7D/graph.json","events_json":"https://pith.science/api/pith-number/PPZALUITMBQBZCEQWOM77YUY7D/events.json","paper":"https://pith.science/paper/PPZALUIT"},"agent_actions":{"view_html":"https://pith.science/pith/PPZALUITMBQBZCEQWOM77YUY7D","download_json":"https://pith.science/pith/PPZALUITMBQBZCEQWOM77YUY7D.json","view_paper":"https://pith.science/paper/PPZALUIT","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2305.13091&json=true","fetch_graph":"https://pith.science/api/pith-number/PPZALUITMBQBZCEQWOM77YUY7D/graph.json","fetch_events":"https://pith.science/api/pith-number/PPZALUITMBQBZCEQWOM77YUY7D/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/PPZALUITMBQBZCEQWOM77YUY7D/action/timestamp_anchor","attest_storage":"https://pith.science/pith/PPZALUITMBQBZCEQWOM77YUY7D/action/storage_attestation","attest_author":"https://pith.science/pith/PPZALUITMBQBZCEQWOM77YUY7D/action/author_attestation","sign_citation":"https://pith.science/pith/PPZALUITMBQBZCEQWOM77YUY7D/action/citation_signature","submit_replication":"https://pith.science/pith/PPZALUITMBQBZCEQWOM77YUY7D/action/replication_record"}},"created_at":"2026-07-05T07:02:50.305719+00:00","updated_at":"2026-07-05T07:02:50.305719+00:00"}