{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:OQH27T57TURF5DHIJOOKJCOPGT","short_pith_number":"pith:OQH27T57","canonical_record":{"source":{"id":"2406.10229","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-06-14T17:59:54Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"fc5b9d15324b5e2c7dddb29436a9567e8d119c640e86fc416ae93d431bcabc15","abstract_canon_sha256":"f025ff8324adb33f35d6ad213a84f12d4458c56317acf6f520122dc4bf67a961"},"schema_version":"1.0"},"canonical_sha256":"740fafcfbf9d225e8ce84b9ca489cf34eb39c427a6ed715a06fd21d602a2ad3e","source":{"kind":"arxiv","id":"2406.10229","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.10229","created_at":"2026-07-05T08:32:07Z"},{"alias_kind":"arxiv_version","alias_value":"2406.10229v1","created_at":"2026-07-05T08:32:07Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.10229","created_at":"2026-07-05T08:32:07Z"},{"alias_kind":"pith_short_12","alias_value":"OQH27T57TURF","created_at":"2026-07-05T08:32:07Z"},{"alias_kind":"pith_short_16","alias_value":"OQH27T57TURF5DHI","created_at":"2026-07-05T08:32:07Z"},{"alias_kind":"pith_short_8","alias_value":"OQH27T57","created_at":"2026-07-05T08:32:07Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:OQH27T57TURF5DHIJOOKJCOPGT","target":"record","payload":{"canonical_record":{"source":{"id":"2406.10229","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-06-14T17:59:54Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"fc5b9d15324b5e2c7dddb29436a9567e8d119c640e86fc416ae93d431bcabc15","abstract_canon_sha256":"f025ff8324adb33f35d6ad213a84f12d4458c56317acf6f520122dc4bf67a961"},"schema_version":"1.0"},"canonical_sha256":"740fafcfbf9d225e8ce84b9ca489cf34eb39c427a6ed715a06fd21d602a2ad3e","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:32:07.082359Z","signature_b64":"3f8XNAwTgTIO+reBlUpz25V7sSihTOerMn688uD0YESWN95FnmilvJKFoPHhQWisOrfiWKbMJfp0V8GMf+vWAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"740fafcfbf9d225e8ce84b9ca489cf34eb39c427a6ed715a06fd21d602a2ad3e","last_reissued_at":"2026-07-05T08:32:07.081847Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:32:07.081847Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.10229","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:32:07Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"mQaTghKhKRNIugelGmiTEzsmZsrUMXFbZE27+ETTEB665N/HaigSdjXJrsY9VRzk7x8Fxzo27Q4qt0hUuwESBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T16:07:07.077100Z"},"content_sha256":"a3ed8f12599b9c22ea707355c91eb1e44be6bbe3db723100aceb36bf5ffa24df","schema_version":"1.0","event_id":"sha256:a3ed8f12599b9c22ea707355c91eb1e44be6bbe3db723100aceb36bf5ffa24df"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:OQH27T57TURF5DHIJOOKJCOPGT","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Quantifying Variance in Evaluation Benchmarks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Aaditya K. Singh, Andrew Poulton, Dieuwke Hupkes, Lovish Madaan, Pontus Stenetorp, Rylan Schaeffer, Sanmi Koyejo, Sharan Narang","submitted_at":"2024-06-14T17:59:54Z","abstract_excerpt":"Evaluation benchmarks are the cornerstone of measuring capabilities of large language models (LLMs), as well as driving progress in said capabilities. Originally designed to make claims about capabilities (or lack thereof) in fully pretrained models, evaluation benchmarks are now also extensively used to decide between various training choices. Despite this widespread usage, we rarely quantify the variance in our evaluation benchmarks, which dictates whether differences in performance are meaningful. Here, we define and measure a range of metrics geared towards measuring variance in evaluation"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.10229","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.10229/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:32:07Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"hOpGYhuB1qh1PgV2DpbmPFp7Sm2P9ewHl9dpDhvk4EkfVNOBsQKzHuzzDHK14xaADk9xhW56zP1UwXfQAxIBCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T16:07:07.077589Z"},"content_sha256":"d752b7f34f65ec6c00a92c7d68d473acc8f96a25e80d9b5d30f5648a241763b9","schema_version":"1.0","event_id":"sha256:d752b7f34f65ec6c00a92c7d68d473acc8f96a25e80d9b5d30f5648a241763b9"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/OQH27T57TURF5DHIJOOKJCOPGT/bundle.json","state_url":"https://pith.science/pith/OQH27T57TURF5DHIJOOKJCOPGT/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/OQH27T57TURF5DHIJOOKJCOPGT/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-04T16:07:07Z","links":{"resolver":"https://pith.science/pith/OQH27T57TURF5DHIJOOKJCOPGT","bundle":"https://pith.science/pith/OQH27T57TURF5DHIJOOKJCOPGT/bundle.json","state":"https://pith.science/pith/OQH27T57TURF5DHIJOOKJCOPGT/state.json","well_known_bundle":"https://pith.science/.well-known/pith/OQH27T57TURF5DHIJOOKJCOPGT/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:OQH27T57TURF5DHIJOOKJCOPGT","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"f025ff8324adb33f35d6ad213a84f12d4458c56317acf6f520122dc4bf67a961","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-06-14T17:59:54Z","title_canon_sha256":"fc5b9d15324b5e2c7dddb29436a9567e8d119c640e86fc416ae93d431bcabc15"},"schema_version":"1.0","source":{"id":"2406.10229","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.10229","created_at":"2026-07-05T08:32:07Z"},{"alias_kind":"arxiv_version","alias_value":"2406.10229v1","created_at":"2026-07-05T08:32:07Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.10229","created_at":"2026-07-05T08:32:07Z"},{"alias_kind":"pith_short_12","alias_value":"OQH27T57TURF","created_at":"2026-07-05T08:32:07Z"},{"alias_kind":"pith_short_16","alias_value":"OQH27T57TURF5DHI","created_at":"2026-07-05T08:32:07Z"},{"alias_kind":"pith_short_8","alias_value":"OQH27T57","created_at":"2026-07-05T08:32:07Z"}],"graph_snapshots":[{"event_id":"sha256:d752b7f34f65ec6c00a92c7d68d473acc8f96a25e80d9b5d30f5648a241763b9","target":"graph","created_at":"2026-07-05T08:32:07Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.10229/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Evaluation benchmarks are the cornerstone of measuring capabilities of large language models (LLMs), as well as driving progress in said capabilities. Originally designed to make claims about capabilities (or lack thereof) in fully pretrained models, evaluation benchmarks are now also extensively used to decide between various training choices. Despite this widespread usage, we rarely quantify the variance in our evaluation benchmarks, which dictates whether differences in performance are meaningful. Here, we define and measure a range of metrics geared towards measuring variance in evaluation","authors_text":"Aaditya K. Singh, Andrew Poulton, Dieuwke Hupkes, Lovish Madaan, Pontus Stenetorp, Rylan Schaeffer, Sanmi Koyejo, Sharan Narang","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-06-14T17:59:54Z","title":"Quantifying Variance in Evaluation Benchmarks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.10229","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:a3ed8f12599b9c22ea707355c91eb1e44be6bbe3db723100aceb36bf5ffa24df","target":"record","created_at":"2026-07-05T08:32:07Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"f025ff8324adb33f35d6ad213a84f12d4458c56317acf6f520122dc4bf67a961","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-06-14T17:59:54Z","title_canon_sha256":"fc5b9d15324b5e2c7dddb29436a9567e8d119c640e86fc416ae93d431bcabc15"},"schema_version":"1.0","source":{"id":"2406.10229","kind":"arxiv","version":1}},"canonical_sha256":"740fafcfbf9d225e8ce84b9ca489cf34eb39c427a6ed715a06fd21d602a2ad3e","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"740fafcfbf9d225e8ce84b9ca489cf34eb39c427a6ed715a06fd21d602a2ad3e","first_computed_at":"2026-07-05T08:32:07.081847Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:32:07.081847Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"3f8XNAwTgTIO+reBlUpz25V7sSihTOerMn688uD0YESWN95FnmilvJKFoPHhQWisOrfiWKbMJfp0V8GMf+vWAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T08:32:07.082359Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.10229","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:a3ed8f12599b9c22ea707355c91eb1e44be6bbe3db723100aceb36bf5ffa24df","sha256:d752b7f34f65ec6c00a92c7d68d473acc8f96a25e80d9b5d30f5648a241763b9"],"state_sha256":"257bfaad9a784d005956e804ffe03672a53489f6d7ea917501de42b47a5147f0"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"GRu2eN2e4QqXKrJpBKsqwqDrGxdSicpBIr78GujGWboGccW4oLw5gqT379gCIqGY/dcK26JIxPDdMpXGTy4qCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-04T16:07:07.081002Z","bundle_sha256":"18070f8289ed0ba18bc729bccc02e2411df04f6b5fc2d42b150904dbdc7e95c3"}}