{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:O7QEOJCFKKWWMRI3PC6PMNU44Q","short_pith_number":"pith:O7QEOJCF","canonical_record":{"source":{"id":"2407.12844","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-04T17:57:38Z","cross_cats_sorted":["cs.LG","stat.ML"],"title_canon_sha256":"e799fbe20cbe362ceb9df04ced5e08dc6ae14ad94e00241d65ea512c2d2a8c29","abstract_canon_sha256":"3d4e8105c85b49a50f9af6dd65753b6a4d229e8a12edcea22843fee1e920b288"},"schema_version":"1.0"},"canonical_sha256":"77e047244552ad66451b78bcf6369ce4131f15b3d1bad805e53fe08745434fc8","source":{"kind":"arxiv","id":"2407.12844","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.12844","created_at":"2026-07-05T10:17:09Z"},{"alias_kind":"arxiv_version","alias_value":"2407.12844v2","created_at":"2026-07-05T10:17:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.12844","created_at":"2026-07-05T10:17:09Z"},{"alias_kind":"pith_short_12","alias_value":"O7QEOJCFKKWW","created_at":"2026-07-05T10:17:09Z"},{"alias_kind":"pith_short_16","alias_value":"O7QEOJCFKKWWMRI3","created_at":"2026-07-05T10:17:09Z"},{"alias_kind":"pith_short_8","alias_value":"O7QEOJCF","created_at":"2026-07-05T10:17:09Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:O7QEOJCFKKWWMRI3PC6PMNU44Q","target":"record","payload":{"canonical_record":{"source":{"id":"2407.12844","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-04T17:57:38Z","cross_cats_sorted":["cs.LG","stat.ML"],"title_canon_sha256":"e799fbe20cbe362ceb9df04ced5e08dc6ae14ad94e00241d65ea512c2d2a8c29","abstract_canon_sha256":"3d4e8105c85b49a50f9af6dd65753b6a4d229e8a12edcea22843fee1e920b288"},"schema_version":"1.0"},"canonical_sha256":"77e047244552ad66451b78bcf6369ce4131f15b3d1bad805e53fe08745434fc8","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:17:09.740576Z","signature_b64":"ePxgNN63Ly14v2F1VFLegNh0FvNXReN0huuQPIyzHozPZhUw4p4sPCHjB4n/ZNrvUcOvHXXK/Vh1pQAh5vzICA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"77e047244552ad66451b78bcf6369ce4131f15b3d1bad805e53fe08745434fc8","last_reissued_at":"2026-07-05T10:17:09.740067Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:17:09.740067Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2407.12844","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:17:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"JXjwDFFabNDwDmJ+hXn5ioFVKHUlSKam9LCj5AdF/Hffs3sQFgFGa0Mwu0ueg7abe7YX9iUPx40xjnhnNIcpCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T17:43:17.623412Z"},"content_sha256":"67883757e78ae649b49376804db8929825d3f15c571546226ed08458d6d4e9d7","schema_version":"1.0","event_id":"sha256:67883757e78ae649b49376804db8929825d3f15c571546226ed08458d6d4e9d7"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:O7QEOJCFKKWWMRI3PC6PMNU44Q","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"metabench -- A Sparse Benchmark of Reasoning and Knowledge in Large Language Models","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.LG","stat.ML"],"primary_cat":"cs.CL","authors_text":"Alex Kipnis, Eric Schulz, Konstantinos Voudouris, Luca M. Schulze Buschoff","submitted_at":"2024-07-04T17:57:38Z","abstract_excerpt":"Large Language Models (LLMs) vary in their abilities on a range of tasks. Initiatives such as the Open LLM Leaderboard aim to quantify these differences with several large benchmarks (sets of test items to which an LLM can respond either correctly or incorrectly). However, high correlations within and between benchmark scores suggest that (1) there exists a small set of common underlying abilities that these benchmarks measure, and (2) items tap into redundant information and the benchmarks may thus be considerably compressed. We use data from n > 5000 LLMs to identify the most informative ite"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.12844","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2407.12844/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:17:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"zIHt/+lkVYOIPmOcjgGdjXyL+/9lwbYI3n/Jl3PF8WUIbg5giVy26x8AVEL+elHqnVxDeacsAyxng+JSOTVwDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T17:43:17.624837Z"},"content_sha256":"0937403d2b5d7a89ee204d114dc83cd75c5ecf9ab8e83c4a81d44fd8746a68fa","schema_version":"1.0","event_id":"sha256:0937403d2b5d7a89ee204d114dc83cd75c5ecf9ab8e83c4a81d44fd8746a68fa"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/O7QEOJCFKKWWMRI3PC6PMNU44Q/bundle.json","state_url":"https://pith.science/pith/O7QEOJCFKKWWMRI3PC6PMNU44Q/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/O7QEOJCFKKWWMRI3PC6PMNU44Q/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-05T17:43:17Z","links":{"resolver":"https://pith.science/pith/O7QEOJCFKKWWMRI3PC6PMNU44Q","bundle":"https://pith.science/pith/O7QEOJCFKKWWMRI3PC6PMNU44Q/bundle.json","state":"https://pith.science/pith/O7QEOJCFKKWWMRI3PC6PMNU44Q/state.json","well_known_bundle":"https://pith.science/.well-known/pith/O7QEOJCFKKWWMRI3PC6PMNU44Q/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:O7QEOJCFKKWWMRI3PC6PMNU44Q","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"3d4e8105c85b49a50f9af6dd65753b6a4d229e8a12edcea22843fee1e920b288","cross_cats_sorted":["cs.LG","stat.ML"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-04T17:57:38Z","title_canon_sha256":"e799fbe20cbe362ceb9df04ced5e08dc6ae14ad94e00241d65ea512c2d2a8c29"},"schema_version":"1.0","source":{"id":"2407.12844","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.12844","created_at":"2026-07-05T10:17:09Z"},{"alias_kind":"arxiv_version","alias_value":"2407.12844v2","created_at":"2026-07-05T10:17:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.12844","created_at":"2026-07-05T10:17:09Z"},{"alias_kind":"pith_short_12","alias_value":"O7QEOJCFKKWW","created_at":"2026-07-05T10:17:09Z"},{"alias_kind":"pith_short_16","alias_value":"O7QEOJCFKKWWMRI3","created_at":"2026-07-05T10:17:09Z"},{"alias_kind":"pith_short_8","alias_value":"O7QEOJCF","created_at":"2026-07-05T10:17:09Z"}],"graph_snapshots":[{"event_id":"sha256:0937403d2b5d7a89ee204d114dc83cd75c5ecf9ab8e83c4a81d44fd8746a68fa","target":"graph","created_at":"2026-07-05T10:17:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2407.12844/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large Language Models (LLMs) vary in their abilities on a range of tasks. Initiatives such as the Open LLM Leaderboard aim to quantify these differences with several large benchmarks (sets of test items to which an LLM can respond either correctly or incorrectly). However, high correlations within and between benchmark scores suggest that (1) there exists a small set of common underlying abilities that these benchmarks measure, and (2) items tap into redundant information and the benchmarks may thus be considerably compressed. We use data from n > 5000 LLMs to identify the most informative ite","authors_text":"Alex Kipnis, Eric Schulz, Konstantinos Voudouris, Luca M. Schulze Buschoff","cross_cats":["cs.LG","stat.ML"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-04T17:57:38Z","title":"metabench -- A Sparse Benchmark of Reasoning and Knowledge in Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.12844","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:67883757e78ae649b49376804db8929825d3f15c571546226ed08458d6d4e9d7","target":"record","created_at":"2026-07-05T10:17:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"3d4e8105c85b49a50f9af6dd65753b6a4d229e8a12edcea22843fee1e920b288","cross_cats_sorted":["cs.LG","stat.ML"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-04T17:57:38Z","title_canon_sha256":"e799fbe20cbe362ceb9df04ced5e08dc6ae14ad94e00241d65ea512c2d2a8c29"},"schema_version":"1.0","source":{"id":"2407.12844","kind":"arxiv","version":2}},"canonical_sha256":"77e047244552ad66451b78bcf6369ce4131f15b3d1bad805e53fe08745434fc8","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"77e047244552ad66451b78bcf6369ce4131f15b3d1bad805e53fe08745434fc8","first_computed_at":"2026-07-05T10:17:09.740067Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:17:09.740067Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"ePxgNN63Ly14v2F1VFLegNh0FvNXReN0huuQPIyzHozPZhUw4p4sPCHjB4n/ZNrvUcOvHXXK/Vh1pQAh5vzICA==","signature_status":"signed_v1","signed_at":"2026-07-05T10:17:09.740576Z","signed_message":"canonical_sha256_bytes"},"source_id":"2407.12844","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:67883757e78ae649b49376804db8929825d3f15c571546226ed08458d6d4e9d7","sha256:0937403d2b5d7a89ee204d114dc83cd75c5ecf9ab8e83c4a81d44fd8746a68fa"],"state_sha256":"c8919fc2c8939ebe4544ab53294292c96412ffe26e762fb7f2864b7fb458eecf"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"XwW+VSoVVWVsZG2wLt+Qd8is5GeYJYQMdNREJ0VE4kKLFvoHBu7KVHSBlIMElCDOo1VsIZNhSM7o7bA/W/F3Bw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-05T17:43:17.633075Z","bundle_sha256":"c06d2e36e92515325e64ca22da7f8770e0626dcf1ecde992eaae01b88eff80c8"}}