{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:DHJJXNWH6MXZHVQIJTAEFCNFXB","short_pith_number":"pith:DHJJXNWH","canonical_record":{"source":{"id":"2509.11106","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-09-14T05:49:42Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"18b55c17488cc5054b9dd4d5d1193ef6db7b9f9a9673c280455f9cadbafeaa5a","abstract_canon_sha256":"5eca93cebaa40444044cdcb33e16c26e1924428ecc71d4984a1b8fc7068f931b"},"schema_version":"1.0"},"canonical_sha256":"19d29bb6c7f32f93d6084cc04289a5b8722c76360d3270faa33fe061cb1573ed","source":{"kind":"arxiv","id":"2509.11106","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2509.11106","created_at":"2026-07-05T12:11:33Z"},{"alias_kind":"arxiv_version","alias_value":"2509.11106v1","created_at":"2026-07-05T12:11:33Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2509.11106","created_at":"2026-07-05T12:11:33Z"},{"alias_kind":"pith_short_12","alias_value":"DHJJXNWH6MXZ","created_at":"2026-07-05T12:11:33Z"},{"alias_kind":"pith_short_16","alias_value":"DHJJXNWH6MXZHVQI","created_at":"2026-07-05T12:11:33Z"},{"alias_kind":"pith_short_8","alias_value":"DHJJXNWH","created_at":"2026-07-05T12:11:33Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:DHJJXNWH6MXZHVQIJTAEFCNFXB","target":"record","payload":{"canonical_record":{"source":{"id":"2509.11106","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-09-14T05:49:42Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"18b55c17488cc5054b9dd4d5d1193ef6db7b9f9a9673c280455f9cadbafeaa5a","abstract_canon_sha256":"5eca93cebaa40444044cdcb33e16c26e1924428ecc71d4984a1b8fc7068f931b"},"schema_version":"1.0"},"canonical_sha256":"19d29bb6c7f32f93d6084cc04289a5b8722c76360d3270faa33fe061cb1573ed","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T12:11:33.655690Z","signature_b64":"kj7egW+zPxtGFEtvOO3ZEJvV/XjsfgdL7r1ywzqQmJwiQK0JUjkGkDEGct1rFUOAAnyDjR7590eoAV1q+51kCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"19d29bb6c7f32f93d6084cc04289a5b8722c76360d3270faa33fe061cb1573ed","last_reissued_at":"2026-07-05T12:11:33.655191Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T12:11:33.655191Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2509.11106","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T12:11:33Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"rg/2oZ3J20kjhBSwc0c9k89ucOqR2Nia6X9b0Tk2Ak+n6QNFVJ5n/giaTyFcdBGV7dmpRza6eCCmPDpIzFh2Cg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-13T02:53:00.517487Z"},"content_sha256":"9411ac09f3b44af2d4f4a6524757c533c7280cc747a31bd4ddcd2ea777fb946e","schema_version":"1.0","event_id":"sha256:9411ac09f3b44af2d4f4a6524757c533c7280cc747a31bd4ddcd2ea777fb946e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:DHJJXNWH6MXZHVQIJTAEFCNFXB","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Fluid Language Model Benchmarking","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Chun Wang, David Heineman, Hannaneh Hajishirzi, Ian Magnusson, Jesse Dodge, Kyle Lo, Maarten Sap, Noah A. Smith, Pang Wei Koh, Valentin Hofmann","submitted_at":"2025-09-14T05:49:42Z","abstract_excerpt":"Language model (LM) benchmarking faces several challenges: comprehensive evaluations are costly, benchmarks often fail to measure the intended capabilities, and evaluation quality can degrade due to labeling errors and benchmark saturation. Although various strategies have been proposed to mitigate these issues, they tend to address individual aspects in isolation, neglecting broader questions about overall evaluation quality. Here, we introduce Fluid Benchmarking, a new evaluation approach that advances LM benchmarking across multiple dimensions. Inspired by psychometrics, Fluid Benchmarking "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2509.11106","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2509.11106/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T12:11:33Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"z0MSMFHMiLdEKAOLQsEP7sc+nB4zAJk9shHsugAbZCI4WbiUYsGVnLfBmeXZ9HX1jG2ZD7jB1NqU/vek2MgcDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-13T02:53:00.518397Z"},"content_sha256":"5d9366390efa926aa13ac86c489aec37b5157466afc267da1427abd7002c9dcd","schema_version":"1.0","event_id":"sha256:5d9366390efa926aa13ac86c489aec37b5157466afc267da1427abd7002c9dcd"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/DHJJXNWH6MXZHVQIJTAEFCNFXB/bundle.json","state_url":"https://pith.science/pith/DHJJXNWH6MXZHVQIJTAEFCNFXB/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/DHJJXNWH6MXZHVQIJTAEFCNFXB/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-13T02:53:00Z","links":{"resolver":"https://pith.science/pith/DHJJXNWH6MXZHVQIJTAEFCNFXB","bundle":"https://pith.science/pith/DHJJXNWH6MXZHVQIJTAEFCNFXB/bundle.json","state":"https://pith.science/pith/DHJJXNWH6MXZHVQIJTAEFCNFXB/state.json","well_known_bundle":"https://pith.science/.well-known/pith/DHJJXNWH6MXZHVQIJTAEFCNFXB/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:DHJJXNWH6MXZHVQIJTAEFCNFXB","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"5eca93cebaa40444044cdcb33e16c26e1924428ecc71d4984a1b8fc7068f931b","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-09-14T05:49:42Z","title_canon_sha256":"18b55c17488cc5054b9dd4d5d1193ef6db7b9f9a9673c280455f9cadbafeaa5a"},"schema_version":"1.0","source":{"id":"2509.11106","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2509.11106","created_at":"2026-07-05T12:11:33Z"},{"alias_kind":"arxiv_version","alias_value":"2509.11106v1","created_at":"2026-07-05T12:11:33Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2509.11106","created_at":"2026-07-05T12:11:33Z"},{"alias_kind":"pith_short_12","alias_value":"DHJJXNWH6MXZ","created_at":"2026-07-05T12:11:33Z"},{"alias_kind":"pith_short_16","alias_value":"DHJJXNWH6MXZHVQI","created_at":"2026-07-05T12:11:33Z"},{"alias_kind":"pith_short_8","alias_value":"DHJJXNWH","created_at":"2026-07-05T12:11:33Z"}],"graph_snapshots":[{"event_id":"sha256:5d9366390efa926aa13ac86c489aec37b5157466afc267da1427abd7002c9dcd","target":"graph","created_at":"2026-07-05T12:11:33Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2509.11106/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Language model (LM) benchmarking faces several challenges: comprehensive evaluations are costly, benchmarks often fail to measure the intended capabilities, and evaluation quality can degrade due to labeling errors and benchmark saturation. Although various strategies have been proposed to mitigate these issues, they tend to address individual aspects in isolation, neglecting broader questions about overall evaluation quality. Here, we introduce Fluid Benchmarking, a new evaluation approach that advances LM benchmarking across multiple dimensions. Inspired by psychometrics, Fluid Benchmarking ","authors_text":"Chun Wang, David Heineman, Hannaneh Hajishirzi, Ian Magnusson, Jesse Dodge, Kyle Lo, Maarten Sap, Noah A. Smith, Pang Wei Koh, Valentin Hofmann","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-09-14T05:49:42Z","title":"Fluid Language Model Benchmarking"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2509.11106","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:9411ac09f3b44af2d4f4a6524757c533c7280cc747a31bd4ddcd2ea777fb946e","target":"record","created_at":"2026-07-05T12:11:33Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"5eca93cebaa40444044cdcb33e16c26e1924428ecc71d4984a1b8fc7068f931b","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-09-14T05:49:42Z","title_canon_sha256":"18b55c17488cc5054b9dd4d5d1193ef6db7b9f9a9673c280455f9cadbafeaa5a"},"schema_version":"1.0","source":{"id":"2509.11106","kind":"arxiv","version":1}},"canonical_sha256":"19d29bb6c7f32f93d6084cc04289a5b8722c76360d3270faa33fe061cb1573ed","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"19d29bb6c7f32f93d6084cc04289a5b8722c76360d3270faa33fe061cb1573ed","first_computed_at":"2026-07-05T12:11:33.655191Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T12:11:33.655191Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"kj7egW+zPxtGFEtvOO3ZEJvV/XjsfgdL7r1ywzqQmJwiQK0JUjkGkDEGct1rFUOAAnyDjR7590eoAV1q+51kCA==","signature_status":"signed_v1","signed_at":"2026-07-05T12:11:33.655690Z","signed_message":"canonical_sha256_bytes"},"source_id":"2509.11106","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:9411ac09f3b44af2d4f4a6524757c533c7280cc747a31bd4ddcd2ea777fb946e","sha256:5d9366390efa926aa13ac86c489aec37b5157466afc267da1427abd7002c9dcd"],"state_sha256":"9ca6dd5ded58b773e6cd948c014faae4f2a33f2e5faf3dcae28171ef23895e31"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"e8xTKeWl32wjXfK0np8D4PSOkuoLajS6LyIwHelyR37dNWvZ3WbeQlTb0couTLfb8hxIhlW9Wn3tl6CL451sCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-13T02:53:00.524811Z","bundle_sha256":"1194c194abbe9977ed19db423a5c46bff96e2d6fb5eb058ea11e99ad884ccd40"}}