{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:BBUSP2EBWKQ3UM7S2KDPNPTXBA","short_pith_number":"pith:BBUSP2EB","schema_version":"1.0","canonical_sha256":"086927e881b2a1ba33f2d286f6be77080c7efd5dd70cfa9647a1367da8b7b627","source":{"kind":"arxiv","id":"2504.18838","version":1},"attestation_state":"computed","paper":{"title":"Toward Generalizable Evaluation in the LLM Era: A Survey Beyond Benchmarks","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Aixin Sun, Dan Huang, David Lo, Haiyuan Liang, Heng Ji, Jiahao Ying, Jing Jiang, Juanzi Li, Lei Hou, Lifu Huang, Min-Yen Kan, Muhao Chen, Qianru Sun, Qi Zhang, Shibo Hong, Tat-Seng Chua, Wenxuan Zhang, Xiaozhi Wang, Xingjun Ma, Xinze Li, Xuanjing Huang, Yantao Liu, Yixin Cao, Yubo Ma, Yu-Gang Jiang, Zijun Yao, Zuxuan Wu","submitted_at":"2025-04-26T07:48:52Z","abstract_excerpt":"Large Language Models (LLMs) are advancing at an amazing speed and have become indispensable across academia, industry, and daily applications. To keep pace with the status quo, this survey probes the core challenges that the rise of LLMs poses for evaluation. We identify and analyze two pivotal transitions: (i) from task-specific to capability-based evaluation, which reorganizes benchmarks around core competencies such as knowledge, reasoning, instruction following, multi-modal understanding, and safety; and (ii) from manual to automated evaluation, encompassing dynamic dataset curation and \""},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2504.18838","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-04-26T07:48:52Z","cross_cats_sorted":[],"title_canon_sha256":"0e01f613119391217fe68d823c70b6c95497a092c3ccd5be1c7e48c092b276af","abstract_canon_sha256":"0b863cb345317a626878c276c06cafa4f1f2a25137fb570e4286e8664e711f00"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:54:22.229446Z","signature_b64":"/fQZcqk0LzsPFEGQCzXH/XbjGYXXK5WJ6BOLX/E4EgAO0XZuSnIRoQufJi7urfX4zvWMluwMhOO5I2ExMAnyCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"086927e881b2a1ba33f2d286f6be77080c7efd5dd70cfa9647a1367da8b7b627","last_reissued_at":"2026-07-05T10:54:22.228978Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:54:22.228978Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Toward Generalizable Evaluation in the LLM Era: A Survey Beyond Benchmarks","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Aixin Sun, Dan Huang, David Lo, Haiyuan Liang, Heng Ji, Jiahao Ying, Jing Jiang, Juanzi Li, Lei Hou, Lifu Huang, Min-Yen Kan, Muhao Chen, Qianru Sun, Qi Zhang, Shibo Hong, Tat-Seng Chua, Wenxuan Zhang, Xiaozhi Wang, Xingjun Ma, Xinze Li, Xuanjing Huang, Yantao Liu, Yixin Cao, Yubo Ma, Yu-Gang Jiang, Zijun Yao, Zuxuan Wu","submitted_at":"2025-04-26T07:48:52Z","abstract_excerpt":"Large Language Models (LLMs) are advancing at an amazing speed and have become indispensable across academia, industry, and daily applications. To keep pace with the status quo, this survey probes the core challenges that the rise of LLMs poses for evaluation. We identify and analyze two pivotal transitions: (i) from task-specific to capability-based evaluation, which reorganizes benchmarks around core competencies such as knowledge, reasoning, instruction following, multi-modal understanding, and safety; and (ii) from manual to automated evaluation, encompassing dynamic dataset curation and \""},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2504.18838","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2504.18838/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2504.18838","created_at":"2026-07-05T10:54:22.229032+00:00"},{"alias_kind":"arxiv_version","alias_value":"2504.18838v1","created_at":"2026-07-05T10:54:22.229032+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2504.18838","created_at":"2026-07-05T10:54:22.229032+00:00"},{"alias_kind":"pith_short_12","alias_value":"BBUSP2EBWKQ3","created_at":"2026-07-05T10:54:22.229032+00:00"},{"alias_kind":"pith_short_16","alias_value":"BBUSP2EBWKQ3UM7S","created_at":"2026-07-05T10:54:22.229032+00:00"},{"alias_kind":"pith_short_8","alias_value":"BBUSP2EB","created_at":"2026-07-05T10:54:22.229032+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":9,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2605.08665","citing_title":"Hint Tuning: Less Data Makes Better Reasoners","ref_index":42,"is_internal_anchor":false},{"citing_arxiv_id":"2605.23657","citing_title":"OpenSkillEval: Automatically Auditing the Open Skill Ecosystem for LLM Agents","ref_index":32,"is_internal_anchor":false},{"citing_arxiv_id":"2606.21296","citing_title":"Discriminatory Compliance: How LLMs Answer Queries from Protected Groups","ref_index":94,"is_internal_anchor":false},{"citing_arxiv_id":"2605.23657","citing_title":"OpenSkillEval: Automatically Auditing the Open Skill Ecosystem for LLM Agents","ref_index":31,"is_internal_anchor":false},{"citing_arxiv_id":"2512.10687","citing_title":"Safe for Whom? Rethinking How We Evaluate the Safety of LLMs for Real Users","ref_index":46,"is_internal_anchor":false},{"citing_arxiv_id":"2603.29403","citing_title":"Security in LLM-as-a-Judge: A Comprehensive SoK","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2605.08665","citing_title":"Hint Tuning: Less Data Makes Better Reasoners","ref_index":42,"is_internal_anchor":false},{"citing_arxiv_id":"2605.08142","citing_title":"Reasoning emerges from constrained inference manifolds in large language models","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2605.01484","citing_title":"Evaluating LLMs on Large-Scale Graph Property Estimation via Random Walks","ref_index":41,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/BBUSP2EBWKQ3UM7S2KDPNPTXBA","json":"https://pith.science/pith/BBUSP2EBWKQ3UM7S2KDPNPTXBA.json","graph_json":"https://pith.science/api/pith-number/BBUSP2EBWKQ3UM7S2KDPNPTXBA/graph.json","events_json":"https://pith.science/api/pith-number/BBUSP2EBWKQ3UM7S2KDPNPTXBA/events.json","paper":"https://pith.science/paper/BBUSP2EB"},"agent_actions":{"view_html":"https://pith.science/pith/BBUSP2EBWKQ3UM7S2KDPNPTXBA","download_json":"https://pith.science/pith/BBUSP2EBWKQ3UM7S2KDPNPTXBA.json","view_paper":"https://pith.science/paper/BBUSP2EB","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2504.18838&json=true","fetch_graph":"https://pith.science/api/pith-number/BBUSP2EBWKQ3UM7S2KDPNPTXBA/graph.json","fetch_events":"https://pith.science/api/pith-number/BBUSP2EBWKQ3UM7S2KDPNPTXBA/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/BBUSP2EBWKQ3UM7S2KDPNPTXBA/action/timestamp_anchor","attest_storage":"https://pith.science/pith/BBUSP2EBWKQ3UM7S2KDPNPTXBA/action/storage_attestation","attest_author":"https://pith.science/pith/BBUSP2EBWKQ3UM7S2KDPNPTXBA/action/author_attestation","sign_citation":"https://pith.science/pith/BBUSP2EBWKQ3UM7S2KDPNPTXBA/action/citation_signature","submit_replication":"https://pith.science/pith/BBUSP2EBWKQ3UM7S2KDPNPTXBA/action/replication_record"}},"created_at":"2026-07-05T10:54:22.229032+00:00","updated_at":"2026-07-05T10:54:22.229032+00:00"}