{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:YO4I724J3WSZPMO56ZJGQO6OJU","short_pith_number":"pith:YO4I724J","canonical_record":{"source":{"id":"2407.13696","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-18T17:00:23Z","cross_cats_sorted":[],"title_canon_sha256":"ec5644dd3a19e5566a647314823aa2ef1ca9381ab5a128a9f4c64fecaf389a09","abstract_canon_sha256":"125b53e508d791e60255ef32b8d47f1e8b24787cb2b0fbe45b3203e194c1fa82"},"schema_version":"1.0"},"canonical_sha256":"c3b88feb89dda597b1ddf652683bce4d29b58b4ea3a65afee404c758fcca5ba5","source":{"kind":"arxiv","id":"2407.13696","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.13696","created_at":"2026-07-05T09:06:07Z"},{"alias_kind":"arxiv_version","alias_value":"2407.13696v2","created_at":"2026-07-05T09:06:07Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.13696","created_at":"2026-07-05T09:06:07Z"},{"alias_kind":"pith_short_12","alias_value":"YO4I724J3WSZ","created_at":"2026-07-05T09:06:07Z"},{"alias_kind":"pith_short_16","alias_value":"YO4I724J3WSZPMO5","created_at":"2026-07-05T09:06:07Z"},{"alias_kind":"pith_short_8","alias_value":"YO4I724J","created_at":"2026-07-05T09:06:07Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:YO4I724J3WSZPMO56ZJGQO6OJU","target":"record","payload":{"canonical_record":{"source":{"id":"2407.13696","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-18T17:00:23Z","cross_cats_sorted":[],"title_canon_sha256":"ec5644dd3a19e5566a647314823aa2ef1ca9381ab5a128a9f4c64fecaf389a09","abstract_canon_sha256":"125b53e508d791e60255ef32b8d47f1e8b24787cb2b0fbe45b3203e194c1fa82"},"schema_version":"1.0"},"canonical_sha256":"c3b88feb89dda597b1ddf652683bce4d29b58b4ea3a65afee404c758fcca5ba5","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:06:07.271802Z","signature_b64":"7/sGP7t9zkm2BQ6R7EvIStUaIOHg1adB054LGzRVlBOIU+Je/PGEcPdLdfRwaf3RGkXr/K1E2315fIpuCXzxDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c3b88feb89dda597b1ddf652683bce4d29b58b4ea3a65afee404c758fcca5ba5","last_reissued_at":"2026-07-05T09:06:07.271299Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:06:07.271299Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2407.13696","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:06:07Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"cuUwjFJPO/Rz7SUAkP4inPyC6G8OS1eHCjzh6e+Tju3jfxPdYMpl2qJKx4eiIsjKAoZbXfynEnQ/H/47AYaLDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-22T23:36:00.985489Z"},"content_sha256":"15fc9596804e251f03fba6a80a3e5bd8a6ff78865cb9efe10043c18ba4b99ace","schema_version":"1.0","event_id":"sha256:15fc9596804e251f03fba6a80a3e5bd8a6ff78865cb9efe10043c18ba4b99ace"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:YO4I724J3WSZPMO56ZJGQO6OJU","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Do These LLM Benchmarks Agree? Fixing Benchmark Evaluation with BenchBench","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Ariel Gera, Asaf Yehudai, Elron Bandel, Eyal Shnarch, Leshem Choshen, Michal Shmueli-Scheuer, Ofir Arviv, Yotam Perlitz","submitted_at":"2024-07-18T17:00:23Z","abstract_excerpt":"Recent advancements in Language Models (LMs) have catalyzed the creation of multiple benchmarks, designed to assess these models' general capabilities. A crucial task, however, is assessing the validity of the benchmarks themselves. This is most commonly done via Benchmark Agreement Testing (BAT), where new benchmarks are validated against established ones using some agreement metric (e.g., rank correlation). Despite the crucial role of BAT for benchmark builders and consumers, there are no standardized procedures for such agreement testing. This deficiency can lead to invalid conclusions, fos"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.13696","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2407.13696/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:06:07Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4GN2yak6UghyDRh59ZBdm2uW2zF/GSlcGgIJt5Q5Ly/o8EJoHWL3eEWYJZ0yMJMLIdwKI+wk3DBVthMimraOBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-22T23:36:00.986023Z"},"content_sha256":"712bd5c5281eef401e1192dc3971560147ef0e601722069ddfdc9c54fe0fbee5","schema_version":"1.0","event_id":"sha256:712bd5c5281eef401e1192dc3971560147ef0e601722069ddfdc9c54fe0fbee5"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/YO4I724J3WSZPMO56ZJGQO6OJU/bundle.json","state_url":"https://pith.science/pith/YO4I724J3WSZPMO56ZJGQO6OJU/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/YO4I724J3WSZPMO56ZJGQO6OJU/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-22T23:36:00Z","links":{"resolver":"https://pith.science/pith/YO4I724J3WSZPMO56ZJGQO6OJU","bundle":"https://pith.science/pith/YO4I724J3WSZPMO56ZJGQO6OJU/bundle.json","state":"https://pith.science/pith/YO4I724J3WSZPMO56ZJGQO6OJU/state.json","well_known_bundle":"https://pith.science/.well-known/pith/YO4I724J3WSZPMO56ZJGQO6OJU/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:YO4I724J3WSZPMO56ZJGQO6OJU","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"125b53e508d791e60255ef32b8d47f1e8b24787cb2b0fbe45b3203e194c1fa82","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-18T17:00:23Z","title_canon_sha256":"ec5644dd3a19e5566a647314823aa2ef1ca9381ab5a128a9f4c64fecaf389a09"},"schema_version":"1.0","source":{"id":"2407.13696","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.13696","created_at":"2026-07-05T09:06:07Z"},{"alias_kind":"arxiv_version","alias_value":"2407.13696v2","created_at":"2026-07-05T09:06:07Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.13696","created_at":"2026-07-05T09:06:07Z"},{"alias_kind":"pith_short_12","alias_value":"YO4I724J3WSZ","created_at":"2026-07-05T09:06:07Z"},{"alias_kind":"pith_short_16","alias_value":"YO4I724J3WSZPMO5","created_at":"2026-07-05T09:06:07Z"},{"alias_kind":"pith_short_8","alias_value":"YO4I724J","created_at":"2026-07-05T09:06:07Z"}],"graph_snapshots":[{"event_id":"sha256:712bd5c5281eef401e1192dc3971560147ef0e601722069ddfdc9c54fe0fbee5","target":"graph","created_at":"2026-07-05T09:06:07Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2407.13696/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recent advancements in Language Models (LMs) have catalyzed the creation of multiple benchmarks, designed to assess these models' general capabilities. A crucial task, however, is assessing the validity of the benchmarks themselves. This is most commonly done via Benchmark Agreement Testing (BAT), where new benchmarks are validated against established ones using some agreement metric (e.g., rank correlation). Despite the crucial role of BAT for benchmark builders and consumers, there are no standardized procedures for such agreement testing. This deficiency can lead to invalid conclusions, fos","authors_text":"Ariel Gera, Asaf Yehudai, Elron Bandel, Eyal Shnarch, Leshem Choshen, Michal Shmueli-Scheuer, Ofir Arviv, Yotam Perlitz","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-18T17:00:23Z","title":"Do These LLM Benchmarks Agree? Fixing Benchmark Evaluation with BenchBench"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.13696","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:15fc9596804e251f03fba6a80a3e5bd8a6ff78865cb9efe10043c18ba4b99ace","target":"record","created_at":"2026-07-05T09:06:07Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"125b53e508d791e60255ef32b8d47f1e8b24787cb2b0fbe45b3203e194c1fa82","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-18T17:00:23Z","title_canon_sha256":"ec5644dd3a19e5566a647314823aa2ef1ca9381ab5a128a9f4c64fecaf389a09"},"schema_version":"1.0","source":{"id":"2407.13696","kind":"arxiv","version":2}},"canonical_sha256":"c3b88feb89dda597b1ddf652683bce4d29b58b4ea3a65afee404c758fcca5ba5","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"c3b88feb89dda597b1ddf652683bce4d29b58b4ea3a65afee404c758fcca5ba5","first_computed_at":"2026-07-05T09:06:07.271299Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:06:07.271299Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"7/sGP7t9zkm2BQ6R7EvIStUaIOHg1adB054LGzRVlBOIU+Je/PGEcPdLdfRwaf3RGkXr/K1E2315fIpuCXzxDw==","signature_status":"signed_v1","signed_at":"2026-07-05T09:06:07.271802Z","signed_message":"canonical_sha256_bytes"},"source_id":"2407.13696","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:15fc9596804e251f03fba6a80a3e5bd8a6ff78865cb9efe10043c18ba4b99ace","sha256:712bd5c5281eef401e1192dc3971560147ef0e601722069ddfdc9c54fe0fbee5"],"state_sha256":"aa767a90a67f34e54e52941d46bc4a9b83bad29808c43abf0862155ed100b6ee"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Pf8kiOCte58wOUzpqrRNtSWvJUDM+zxRJRx+XF97NDp/LhY72TI34dt+e9/ts7QQNgkYQ+uGtU90zCBNxmCxAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-22T23:36:00.992620Z","bundle_sha256":"e76fd44ff9dc23b0aa6e81f273b5e6a0d0d64f60d9b9ecd3a92024c15840c6de"}}