{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:M7CNLHHNJUL56PCMI64ALYHIVY","short_pith_number":"pith:M7CNLHHN","canonical_record":{"source":{"id":"2311.17295","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-11-29T00:45:23Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"ee13fa4d0e181839638e9f8be83801258a85bd9eb2204ec30445dd83aa182b4e","abstract_canon_sha256":"1c2ebf67584c7dc9a32e92225c19c047a8542d1ab1c449894ab17db82a2929b3"},"schema_version":"1.0"},"canonical_sha256":"67c4d59ced4d17df3c4c47b805e0e8ae30489712f043239ee1a3a8f05f651635","source":{"kind":"arxiv","id":"2311.17295","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2311.17295","created_at":"2026-07-05T07:18:00Z"},{"alias_kind":"arxiv_version","alias_value":"2311.17295v1","created_at":"2026-07-05T07:18:00Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2311.17295","created_at":"2026-07-05T07:18:00Z"},{"alias_kind":"pith_short_12","alias_value":"M7CNLHHNJUL5","created_at":"2026-07-05T07:18:00Z"},{"alias_kind":"pith_short_16","alias_value":"M7CNLHHNJUL56PCM","created_at":"2026-07-05T07:18:00Z"},{"alias_kind":"pith_short_8","alias_value":"M7CNLHHN","created_at":"2026-07-05T07:18:00Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:M7CNLHHNJUL56PCMI64ALYHIVY","target":"record","payload":{"canonical_record":{"source":{"id":"2311.17295","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-11-29T00:45:23Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"ee13fa4d0e181839638e9f8be83801258a85bd9eb2204ec30445dd83aa182b4e","abstract_canon_sha256":"1c2ebf67584c7dc9a32e92225c19c047a8542d1ab1c449894ab17db82a2929b3"},"schema_version":"1.0"},"canonical_sha256":"67c4d59ced4d17df3c4c47b805e0e8ae30489712f043239ee1a3a8f05f651635","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:18:00.870451Z","signature_b64":"3Ebk3F08H7KI6Bxin4JwfrsFyIAH1tw36m+RmanrtmAfh+EoL28NuIufrZ0BHbWv3H7GI3ZNaMWiYWCTMGF4Bw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"67c4d59ced4d17df3c4c47b805e0e8ae30489712f043239ee1a3a8f05f651635","last_reissued_at":"2026-07-05T07:18:00.869991Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:18:00.869991Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2311.17295","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:18:00Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"INImLt+6EFpHm7j3J33sAPX07854s2e6KZM5LXnoCvNSpmi4zo+MWwHJBRq/Xqua/6vHvsu5rm/AjIizl+F+DA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T18:47:08.123919Z"},"content_sha256":"d8eea208c6b321e60924c2ce8e1b49eb56eeabf5efb88ebb1188608392b67b20","schema_version":"1.0","event_id":"sha256:d8eea208c6b321e60924c2ce8e1b49eb56eeabf5efb88ebb1188608392b67b20"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:M7CNLHHNJUL56PCMI64ALYHIVY","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Elo Uncovered: Robustness and Best Practices in Language Model Evaluation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Beyza Ermis, Edward Kim, Marzieh Fadaee, Meriem Boubdir, Sara Hooker","submitted_at":"2023-11-29T00:45:23Z","abstract_excerpt":"In Natural Language Processing (NLP), the Elo rating system, originally designed for ranking players in dynamic games such as chess, is increasingly being used to evaluate Large Language Models (LLMs) through \"A vs B\" paired comparisons. However, while popular, the system's suitability for assessing entities with constant skill levels, such as LLMs, remains relatively unexplored. We study two fundamental axioms that evaluation methods should adhere to: reliability and transitivity. We conduct extensive evaluation of Elo behaviour, illustrating that individual Elo computations exhibit volatilit"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2311.17295","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2311.17295/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:18:00Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ZN2oI/ZatZlJZVWLh20o+ClxFapuIRjjwZNs7FxbeFhQApdH/caD61g4Xd6/V0UZlMyg0n+dyRu5WINqkHSPBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T18:47:08.124448Z"},"content_sha256":"d4c11c0e01d0a7e2bdb088a5c68d810f19425561be2adf014e8a8eb28cb1e7c1","schema_version":"1.0","event_id":"sha256:d4c11c0e01d0a7e2bdb088a5c68d810f19425561be2adf014e8a8eb28cb1e7c1"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/M7CNLHHNJUL56PCMI64ALYHIVY/bundle.json","state_url":"https://pith.science/pith/M7CNLHHNJUL56PCMI64ALYHIVY/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/M7CNLHHNJUL56PCMI64ALYHIVY/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-03T18:47:08Z","links":{"resolver":"https://pith.science/pith/M7CNLHHNJUL56PCMI64ALYHIVY","bundle":"https://pith.science/pith/M7CNLHHNJUL56PCMI64ALYHIVY/bundle.json","state":"https://pith.science/pith/M7CNLHHNJUL56PCMI64ALYHIVY/state.json","well_known_bundle":"https://pith.science/.well-known/pith/M7CNLHHNJUL56PCMI64ALYHIVY/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:M7CNLHHNJUL56PCMI64ALYHIVY","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"1c2ebf67584c7dc9a32e92225c19c047a8542d1ab1c449894ab17db82a2929b3","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-11-29T00:45:23Z","title_canon_sha256":"ee13fa4d0e181839638e9f8be83801258a85bd9eb2204ec30445dd83aa182b4e"},"schema_version":"1.0","source":{"id":"2311.17295","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2311.17295","created_at":"2026-07-05T07:18:00Z"},{"alias_kind":"arxiv_version","alias_value":"2311.17295v1","created_at":"2026-07-05T07:18:00Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2311.17295","created_at":"2026-07-05T07:18:00Z"},{"alias_kind":"pith_short_12","alias_value":"M7CNLHHNJUL5","created_at":"2026-07-05T07:18:00Z"},{"alias_kind":"pith_short_16","alias_value":"M7CNLHHNJUL56PCM","created_at":"2026-07-05T07:18:00Z"},{"alias_kind":"pith_short_8","alias_value":"M7CNLHHN","created_at":"2026-07-05T07:18:00Z"}],"graph_snapshots":[{"event_id":"sha256:d4c11c0e01d0a7e2bdb088a5c68d810f19425561be2adf014e8a8eb28cb1e7c1","target":"graph","created_at":"2026-07-05T07:18:00Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2311.17295/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In Natural Language Processing (NLP), the Elo rating system, originally designed for ranking players in dynamic games such as chess, is increasingly being used to evaluate Large Language Models (LLMs) through \"A vs B\" paired comparisons. However, while popular, the system's suitability for assessing entities with constant skill levels, such as LLMs, remains relatively unexplored. We study two fundamental axioms that evaluation methods should adhere to: reliability and transitivity. We conduct extensive evaluation of Elo behaviour, illustrating that individual Elo computations exhibit volatilit","authors_text":"Beyza Ermis, Edward Kim, Marzieh Fadaee, Meriem Boubdir, Sara Hooker","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-11-29T00:45:23Z","title":"Elo Uncovered: Robustness and Best Practices in Language Model Evaluation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2311.17295","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:d8eea208c6b321e60924c2ce8e1b49eb56eeabf5efb88ebb1188608392b67b20","target":"record","created_at":"2026-07-05T07:18:00Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"1c2ebf67584c7dc9a32e92225c19c047a8542d1ab1c449894ab17db82a2929b3","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-11-29T00:45:23Z","title_canon_sha256":"ee13fa4d0e181839638e9f8be83801258a85bd9eb2204ec30445dd83aa182b4e"},"schema_version":"1.0","source":{"id":"2311.17295","kind":"arxiv","version":1}},"canonical_sha256":"67c4d59ced4d17df3c4c47b805e0e8ae30489712f043239ee1a3a8f05f651635","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"67c4d59ced4d17df3c4c47b805e0e8ae30489712f043239ee1a3a8f05f651635","first_computed_at":"2026-07-05T07:18:00.869991Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:18:00.869991Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"3Ebk3F08H7KI6Bxin4JwfrsFyIAH1tw36m+RmanrtmAfh+EoL28NuIufrZ0BHbWv3H7GI3ZNaMWiYWCTMGF4Bw==","signature_status":"signed_v1","signed_at":"2026-07-05T07:18:00.870451Z","signed_message":"canonical_sha256_bytes"},"source_id":"2311.17295","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:d8eea208c6b321e60924c2ce8e1b49eb56eeabf5efb88ebb1188608392b67b20","sha256:d4c11c0e01d0a7e2bdb088a5c68d810f19425561be2adf014e8a8eb28cb1e7c1"],"state_sha256":"8dc7653ef035d127bfd6b0ead12702ad79dad4d2c423478746755e1bac300207"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"c7n1nBQ7xmuzn78rpmZuCMgxMZKpvsy2zR5r0/jYlO2wH16TK3H9iy9YifC/btvv+MPcjqJoMw6drM6b1I9UCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-03T18:47:08.128254Z","bundle_sha256":"27595340cb34436a9147541a87e087fe5affc070f3e7d307b4bbf4c80e2b4ab4"}}