{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:LXF4TV64P4APXX4WEJFCSVUNIA","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"49a6077984306dba9e9b675a8b306eaf2af1521b41140a4aa2d59ba9bfde54ce","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-19T07:15:59Z","title_canon_sha256":"b65f7029a2cb3bf16daaae4ff0615b85b2b267699e35cb9f7d45a2329e30f625"},"schema_version":"1.0","source":{"id":"2402.11894","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.11894","created_at":"2026-07-05T08:28:06Z"},{"alias_kind":"arxiv_version","alias_value":"2402.11894v3","created_at":"2026-07-05T08:28:06Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.11894","created_at":"2026-07-05T08:28:06Z"},{"alias_kind":"pith_short_12","alias_value":"LXF4TV64P4AP","created_at":"2026-07-05T08:28:06Z"},{"alias_kind":"pith_short_16","alias_value":"LXF4TV64P4APXX4W","created_at":"2026-07-05T08:28:06Z"},{"alias_kind":"pith_short_8","alias_value":"LXF4TV64","created_at":"2026-07-05T08:28:06Z"}],"graph_snapshots":[{"event_id":"sha256:0489c4df0e12bd55f816f01148df1d057c3c4742c4e5c5026cccc5da02ee3685","target":"graph","created_at":"2026-07-05T08:28:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.11894/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large language models (LLMs) have achieved impressive performance across various natural language benchmarks, prompting a continual need to curate more difficult datasets for larger LLMs, which is costly and time-consuming. In this paper, we propose to automate dataset updating and provide systematic analysis regarding its effectiveness in dealing with benchmark leakage issue, difficulty control, and stability. Thus, once the current benchmark has been mastered or leaked, we can update it for timely and reliable evaluation. There are two updating strategies: 1) mimicking strategy to generate s","authors_text":"Bo Wang, Jiahao Ying, Qianru Sun, Shuicheng Yan, Wei Tang, Xuanjing Huang, Yixin Cao, Yizhe Yang, Yushi Bai, Zhaojun Ding","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-19T07:15:59Z","title":"Automating Dataset Updates Towards Reliable and Timely Evaluation of Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.11894","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:df17a59257c2d6dfd6b2bd5ce3bd4be2f777b136b6c0a2ffb5b8c6383b582ffc","target":"record","created_at":"2026-07-05T08:28:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"49a6077984306dba9e9b675a8b306eaf2af1521b41140a4aa2d59ba9bfde54ce","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-19T07:15:59Z","title_canon_sha256":"b65f7029a2cb3bf16daaae4ff0615b85b2b267699e35cb9f7d45a2329e30f625"},"schema_version":"1.0","source":{"id":"2402.11894","kind":"arxiv","version":3}},"canonical_sha256":"5dcbc9d7dc7f00fbdf96224a29568d401b890dc4640e456bdf44d8a1671546fd","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"5dcbc9d7dc7f00fbdf96224a29568d401b890dc4640e456bdf44d8a1671546fd","first_computed_at":"2026-07-05T08:28:06.363807Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:28:06.363807Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"jlU1cb74ZeeTKV25iqSNcnxtZnGJ9th0KbA9i5K5rWI7D4TZqz4lhQDP3AbkE0bAE4HQtkRP+CF5sVBo7d7nAA==","signature_status":"signed_v1","signed_at":"2026-07-05T08:28:06.364243Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.11894","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:df17a59257c2d6dfd6b2bd5ce3bd4be2f777b136b6c0a2ffb5b8c6383b582ffc","sha256:0489c4df0e12bd55f816f01148df1d057c3c4742c4e5c5026cccc5da02ee3685"],"state_sha256":"92ec004a4e9e5bd70fb84d4aeeeef2ab8f6105fc58590e1f0c807fa27942cf66"}