{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:7P3J5G7G7R3MWSK6D4QLYIEYA4","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"659d1ca8208bc66123fb37bfec8368929d35bcadc528308a96fe802cf1457814","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-12-31T17:46:51Z","title_canon_sha256":"b7b7ba6f684d4aa33c2a26b4f0ca0c3bd3a6b2324d2a37ddc30cc7e47fed8fa1"},"schema_version":"1.0","source":{"id":"2501.00560","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2501.00560","created_at":"2026-07-05T10:12:24Z"},{"alias_kind":"arxiv_version","alias_value":"2501.00560v2","created_at":"2026-07-05T10:12:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2501.00560","created_at":"2026-07-05T10:12:24Z"},{"alias_kind":"pith_short_12","alias_value":"7P3J5G7G7R3M","created_at":"2026-07-05T10:12:24Z"},{"alias_kind":"pith_short_16","alias_value":"7P3J5G7G7R3MWSK6","created_at":"2026-07-05T10:12:24Z"},{"alias_kind":"pith_short_8","alias_value":"7P3J5G7G","created_at":"2026-07-05T10:12:24Z"}],"graph_snapshots":[{"event_id":"sha256:5ca2c97377d1af64689a6dc8f935fa972105afbe74131f4f49bd5997761b25a5","target":"graph","created_at":"2026-07-05T10:12:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2501.00560/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Evaluating and ranking the capabilities of different LLMs is crucial for understanding their performance and alignment with human preferences. Due to the high cost and time-consuming nature of human evaluations, an automatic LLM bencher (i.e., an automatic evaluation framework that aims to rank LLMs based on their alignment with human preferences) is indispensable. An automatic LLM bencher consists of four components: the input set (e.g., a user instruction), the evaluation model (e.g., an LLM), the evaluation type (e.g., pairwise comparison), and the aggregation method (e.g., the ELO rating s","authors_text":"Arman Cohan, Jonathan Bragg, Mingqi Gao, Xiaojun Wan, Xinyu Hu, Yixin Liu","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-12-31T17:46:51Z","title":"Re-evaluating Automatic LLM System Ranking for Alignment with Human Preference"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2501.00560","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:d6678058818e5febe897329d36198a61d2d3c1f8af18bfea126ceb0273bc4959","target":"record","created_at":"2026-07-05T10:12:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"659d1ca8208bc66123fb37bfec8368929d35bcadc528308a96fe802cf1457814","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-12-31T17:46:51Z","title_canon_sha256":"b7b7ba6f684d4aa33c2a26b4f0ca0c3bd3a6b2324d2a37ddc30cc7e47fed8fa1"},"schema_version":"1.0","source":{"id":"2501.00560","kind":"arxiv","version":2}},"canonical_sha256":"fbf69e9be6fc76cb495e1f20bc2098071126a9c671313f007a5f31d1211f5180","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"fbf69e9be6fc76cb495e1f20bc2098071126a9c671313f007a5f31d1211f5180","first_computed_at":"2026-07-05T10:12:24.718932Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:12:24.718932Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"EZNYNXZ4edjXD6IFRPl4Rf97NLaX0/LyAeMKOCmPN4o6V9MuP+FfO6o2jT4LLHTRe5f4FLNChbzUfbCHZstRAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T10:12:24.719450Z","signed_message":"canonical_sha256_bytes"},"source_id":"2501.00560","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:d6678058818e5febe897329d36198a61d2d3c1f8af18bfea126ceb0273bc4959","sha256:5ca2c97377d1af64689a6dc8f935fa972105afbe74131f4f49bd5997761b25a5"],"state_sha256":"f565b016c52bdd1e5a9f1dd7df2ae44ddd9c9f68534e7e0ede12bcc51ac9db44"}