{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:2VAZD5WB5R2T5MLYV63X5BMMJ2","short_pith_number":"pith:2VAZD5WB","canonical_record":{"source":{"id":"2411.14483","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-19T20:16:26Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"e98dc6768ecb3eb1bd6a567b58bcfc3d5cc158bf6e27b294de36615618a35ed2","abstract_canon_sha256":"ae79ea117e0a190f81623b101b30218faa2146b04919c1836d7d17c98056da0f"},"schema_version":"1.0"},"canonical_sha256":"d54191f6c1ec753eb178afb77e858c4ebe2a570f633e21b3686465b71d81027e","source":{"kind":"arxiv","id":"2411.14483","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.14483","created_at":"2026-07-05T10:15:23Z"},{"alias_kind":"arxiv_version","alias_value":"2411.14483v2","created_at":"2026-07-05T10:15:23Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.14483","created_at":"2026-07-05T10:15:23Z"},{"alias_kind":"pith_short_12","alias_value":"2VAZD5WB5R2T","created_at":"2026-07-05T10:15:23Z"},{"alias_kind":"pith_short_16","alias_value":"2VAZD5WB5R2T5MLY","created_at":"2026-07-05T10:15:23Z"},{"alias_kind":"pith_short_8","alias_value":"2VAZD5WB","created_at":"2026-07-05T10:15:23Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:2VAZD5WB5R2T5MLYV63X5BMMJ2","target":"record","payload":{"canonical_record":{"source":{"id":"2411.14483","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-19T20:16:26Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"e98dc6768ecb3eb1bd6a567b58bcfc3d5cc158bf6e27b294de36615618a35ed2","abstract_canon_sha256":"ae79ea117e0a190f81623b101b30218faa2146b04919c1836d7d17c98056da0f"},"schema_version":"1.0"},"canonical_sha256":"d54191f6c1ec753eb178afb77e858c4ebe2a570f633e21b3686465b71d81027e","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:15:23.166051Z","signature_b64":"oak92Ygf9xzKFj9Xyzcl1SCf7R6MxY+3uwuXN/j0hUD4VqN9RFW4ZFO80jKR0ZjPGKksNXxriwdNhtUBOs6eBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d54191f6c1ec753eb178afb77e858c4ebe2a570f633e21b3686465b71d81027e","last_reissued_at":"2026-07-05T10:15:23.165578Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:15:23.165578Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2411.14483","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:15:23Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"dh6NjKCGbfssyyYQ6RvK6FIym0mnghqXZTDY/57v3BPoPr12Ht+loiUgTAQop5e5jO368blNGxgZv5Sda5+FDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T16:53:20.748426Z"},"content_sha256":"d57aba5e45bb442afc067941a950fd5517aa664d6d620381a20f7b96aebe0de3","schema_version":"1.0","event_id":"sha256:d57aba5e45bb442afc067941a950fd5517aa664d6d620381a20f7b96aebe0de3"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:2VAZD5WB5R2T5MLYV63X5BMMJ2","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Ranking Unraveled: Recipes for LLM Rankings in Head-to-Head AI Combat","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Christopher Clarke, Jason Mars, Krisztian Flautner, Lingjia Tang, Roland Daynauth","submitted_at":"2024-11-19T20:16:26Z","abstract_excerpt":"Deciding which large language model (LLM) to use is a complex challenge. Pairwise ranking has emerged as a new method for evaluating human preferences for LLMs. This approach entails humans evaluating pairs of model outputs based on a predefined criterion. By collecting these comparisons, a ranking can be constructed using methods such as Elo. However, applying these algorithms as constructed in the context of LLM evaluation introduces several challenges. In this paper, we explore the effectiveness of ranking systems for head-to-head comparisons of LLMs. We formally define a set of fundamental"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.14483","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2411.14483/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:15:23Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"G2GIAdsfTfh7YTumEfNeLgMH9wHczMZxcvBMUS1Rt3uxDFH0kyYdQMilu1S4wsLJYblqCKGGSP361bG4n2VfAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T16:53:20.748991Z"},"content_sha256":"26d193fe4f639c824b550d4e4ab890707c24d11170176d78f954a924797a5517","schema_version":"1.0","event_id":"sha256:26d193fe4f639c824b550d4e4ab890707c24d11170176d78f954a924797a5517"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/2VAZD5WB5R2T5MLYV63X5BMMJ2/bundle.json","state_url":"https://pith.science/pith/2VAZD5WB5R2T5MLYV63X5BMMJ2/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/2VAZD5WB5R2T5MLYV63X5BMMJ2/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T16:53:20Z","links":{"resolver":"https://pith.science/pith/2VAZD5WB5R2T5MLYV63X5BMMJ2","bundle":"https://pith.science/pith/2VAZD5WB5R2T5MLYV63X5BMMJ2/bundle.json","state":"https://pith.science/pith/2VAZD5WB5R2T5MLYV63X5BMMJ2/state.json","well_known_bundle":"https://pith.science/.well-known/pith/2VAZD5WB5R2T5MLYV63X5BMMJ2/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:2VAZD5WB5R2T5MLYV63X5BMMJ2","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"ae79ea117e0a190f81623b101b30218faa2146b04919c1836d7d17c98056da0f","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-19T20:16:26Z","title_canon_sha256":"e98dc6768ecb3eb1bd6a567b58bcfc3d5cc158bf6e27b294de36615618a35ed2"},"schema_version":"1.0","source":{"id":"2411.14483","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.14483","created_at":"2026-07-05T10:15:23Z"},{"alias_kind":"arxiv_version","alias_value":"2411.14483v2","created_at":"2026-07-05T10:15:23Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.14483","created_at":"2026-07-05T10:15:23Z"},{"alias_kind":"pith_short_12","alias_value":"2VAZD5WB5R2T","created_at":"2026-07-05T10:15:23Z"},{"alias_kind":"pith_short_16","alias_value":"2VAZD5WB5R2T5MLY","created_at":"2026-07-05T10:15:23Z"},{"alias_kind":"pith_short_8","alias_value":"2VAZD5WB","created_at":"2026-07-05T10:15:23Z"}],"graph_snapshots":[{"event_id":"sha256:26d193fe4f639c824b550d4e4ab890707c24d11170176d78f954a924797a5517","target":"graph","created_at":"2026-07-05T10:15:23Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2411.14483/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Deciding which large language model (LLM) to use is a complex challenge. Pairwise ranking has emerged as a new method for evaluating human preferences for LLMs. This approach entails humans evaluating pairs of model outputs based on a predefined criterion. By collecting these comparisons, a ranking can be constructed using methods such as Elo. However, applying these algorithms as constructed in the context of LLM evaluation introduces several challenges. In this paper, we explore the effectiveness of ranking systems for head-to-head comparisons of LLMs. We formally define a set of fundamental","authors_text":"Christopher Clarke, Jason Mars, Krisztian Flautner, Lingjia Tang, Roland Daynauth","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-19T20:16:26Z","title":"Ranking Unraveled: Recipes for LLM Rankings in Head-to-Head AI Combat"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.14483","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:d57aba5e45bb442afc067941a950fd5517aa664d6d620381a20f7b96aebe0de3","target":"record","created_at":"2026-07-05T10:15:23Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"ae79ea117e0a190f81623b101b30218faa2146b04919c1836d7d17c98056da0f","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-19T20:16:26Z","title_canon_sha256":"e98dc6768ecb3eb1bd6a567b58bcfc3d5cc158bf6e27b294de36615618a35ed2"},"schema_version":"1.0","source":{"id":"2411.14483","kind":"arxiv","version":2}},"canonical_sha256":"d54191f6c1ec753eb178afb77e858c4ebe2a570f633e21b3686465b71d81027e","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"d54191f6c1ec753eb178afb77e858c4ebe2a570f633e21b3686465b71d81027e","first_computed_at":"2026-07-05T10:15:23.165578Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:15:23.165578Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"oak92Ygf9xzKFj9Xyzcl1SCf7R6MxY+3uwuXN/j0hUD4VqN9RFW4ZFO80jKR0ZjPGKksNXxriwdNhtUBOs6eBA==","signature_status":"signed_v1","signed_at":"2026-07-05T10:15:23.166051Z","signed_message":"canonical_sha256_bytes"},"source_id":"2411.14483","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:d57aba5e45bb442afc067941a950fd5517aa664d6d620381a20f7b96aebe0de3","sha256:26d193fe4f639c824b550d4e4ab890707c24d11170176d78f954a924797a5517"],"state_sha256":"d49f90fcea9598d97ac0645d01e4b5ca4113d3e184403797dce1b1ae36115994"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"BD21z4Mh7U9Y1B0ybnBbiMDwU34xMmZsNFshHOLKztFc9OXvuxUs18o07G47h+aNEYpQ42DpArn8Q1d9NDVOCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T16:53:20.754187Z","bundle_sha256":"64bd7a868935247a9c12b0409a681970a14d14cc61dbe1ee67b0d820416660f9"}}