{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:S22CP3GECNKYKLBRRRWEVRFH5Y","short_pith_number":"pith:S22CP3GE","canonical_record":{"source":{"id":"2406.17681","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-25T16:13:53Z","cross_cats_sorted":[],"title_canon_sha256":"435ff44ee88e89f6b6a29c16d9cbe883f985762c9878acac60d4f8c73a6fbd7f","abstract_canon_sha256":"879535f07ff1e7c9997607bbbbf9339dcfaff121b6e0348d7a304343edb236bc"},"schema_version":"1.0"},"canonical_sha256":"96b427ecc41355852c318c6c4ac4a7ee2ad811a82fe5f1c7bb374a3880646a67","source":{"kind":"arxiv","id":"2406.17681","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.17681","created_at":"2026-07-05T08:37:07Z"},{"alias_kind":"arxiv_version","alias_value":"2406.17681v2","created_at":"2026-07-05T08:37:07Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.17681","created_at":"2026-07-05T08:37:07Z"},{"alias_kind":"pith_short_12","alias_value":"S22CP3GECNKY","created_at":"2026-07-05T08:37:07Z"},{"alias_kind":"pith_short_16","alias_value":"S22CP3GECNKYKLBR","created_at":"2026-07-05T08:37:07Z"},{"alias_kind":"pith_short_8","alias_value":"S22CP3GE","created_at":"2026-07-05T08:37:07Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:S22CP3GECNKYKLBRRRWEVRFH5Y","target":"record","payload":{"canonical_record":{"source":{"id":"2406.17681","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-25T16:13:53Z","cross_cats_sorted":[],"title_canon_sha256":"435ff44ee88e89f6b6a29c16d9cbe883f985762c9878acac60d4f8c73a6fbd7f","abstract_canon_sha256":"879535f07ff1e7c9997607bbbbf9339dcfaff121b6e0348d7a304343edb236bc"},"schema_version":"1.0"},"canonical_sha256":"96b427ecc41355852c318c6c4ac4a7ee2ad811a82fe5f1c7bb374a3880646a67","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:37:07.534091Z","signature_b64":"lKrpclq0N7DxKO053E87G78iTXllWGBnMIDHJZAx0qcND4msi4phyk1ygAC4X+O2TmMaWJSX4FoE2Ob4lHx9AA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"96b427ecc41355852c318c6c4ac4a7ee2ad811a82fe5f1c7bb374a3880646a67","last_reissued_at":"2026-07-05T08:37:07.533625Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:37:07.533625Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.17681","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:37:07Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"zq0+M7G61968B+eWz3VuzsHBtBBF2DH3sLIIfqB5zYnGvXIvqjwA9j5D5r88DHNZoDSXvfMbOcVVHa2waBXyAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T16:39:49.334074Z"},"content_sha256":"1684932de65d5747a60b23de887d38741d40f198bb6c375491c111600ec78c3f","schema_version":"1.0","event_id":"sha256:1684932de65d5747a60b23de887d38741d40f198bb6c375491c111600ec78c3f"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:S22CP3GECNKYKLBRRRWEVRFH5Y","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"VarBench: Robust Language Model Benchmarking Through Dynamic Variable Perturbation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Claudia Tang, Kun Qian, Maximillian Chen, Shunji Wan, Xuanming Zhang, Youzhi Wang, Zhou Yu","submitted_at":"2024-06-25T16:13:53Z","abstract_excerpt":"As large language models achieve impressive scores on traditional benchmarks, an increasing number of researchers are becoming concerned about benchmark data leakage during pre-training, commonly known as the data contamination problem. To ensure fair evaluation, recent benchmarks release only the training and validation sets, keeping the test set labels closed-source. They require anyone wishing to evaluate his language model to submit the model's predictions for centralized processing and then publish the model's result on their leaderboard. However, this submission process is inefficient an"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.17681","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.17681/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:37:07Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Hg+EdOITfKMCC/QT8V/7hM/Q0lf/ekOcIK35bSIAEx52LvQSGoLiUi5M4c1bOksNfioTosj3mHDQLq6tlufeDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T16:39:49.335357Z"},"content_sha256":"54ea5c68e9d69b63c0526403003d9e25443ca0fc229020c4b46e970891f74858","schema_version":"1.0","event_id":"sha256:54ea5c68e9d69b63c0526403003d9e25443ca0fc229020c4b46e970891f74858"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/S22CP3GECNKYKLBRRRWEVRFH5Y/bundle.json","state_url":"https://pith.science/pith/S22CP3GECNKYKLBRRRWEVRFH5Y/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/S22CP3GECNKYKLBRRRWEVRFH5Y/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T16:39:49Z","links":{"resolver":"https://pith.science/pith/S22CP3GECNKYKLBRRRWEVRFH5Y","bundle":"https://pith.science/pith/S22CP3GECNKYKLBRRRWEVRFH5Y/bundle.json","state":"https://pith.science/pith/S22CP3GECNKYKLBRRRWEVRFH5Y/state.json","well_known_bundle":"https://pith.science/.well-known/pith/S22CP3GECNKYKLBRRRWEVRFH5Y/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:S22CP3GECNKYKLBRRRWEVRFH5Y","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"879535f07ff1e7c9997607bbbbf9339dcfaff121b6e0348d7a304343edb236bc","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-25T16:13:53Z","title_canon_sha256":"435ff44ee88e89f6b6a29c16d9cbe883f985762c9878acac60d4f8c73a6fbd7f"},"schema_version":"1.0","source":{"id":"2406.17681","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.17681","created_at":"2026-07-05T08:37:07Z"},{"alias_kind":"arxiv_version","alias_value":"2406.17681v2","created_at":"2026-07-05T08:37:07Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.17681","created_at":"2026-07-05T08:37:07Z"},{"alias_kind":"pith_short_12","alias_value":"S22CP3GECNKY","created_at":"2026-07-05T08:37:07Z"},{"alias_kind":"pith_short_16","alias_value":"S22CP3GECNKYKLBR","created_at":"2026-07-05T08:37:07Z"},{"alias_kind":"pith_short_8","alias_value":"S22CP3GE","created_at":"2026-07-05T08:37:07Z"}],"graph_snapshots":[{"event_id":"sha256:54ea5c68e9d69b63c0526403003d9e25443ca0fc229020c4b46e970891f74858","target":"graph","created_at":"2026-07-05T08:37:07Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.17681/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"As large language models achieve impressive scores on traditional benchmarks, an increasing number of researchers are becoming concerned about benchmark data leakage during pre-training, commonly known as the data contamination problem. To ensure fair evaluation, recent benchmarks release only the training and validation sets, keeping the test set labels closed-source. They require anyone wishing to evaluate his language model to submit the model's predictions for centralized processing and then publish the model's result on their leaderboard. However, this submission process is inefficient an","authors_text":"Claudia Tang, Kun Qian, Maximillian Chen, Shunji Wan, Xuanming Zhang, Youzhi Wang, Zhou Yu","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-25T16:13:53Z","title":"VarBench: Robust Language Model Benchmarking Through Dynamic Variable Perturbation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.17681","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1684932de65d5747a60b23de887d38741d40f198bb6c375491c111600ec78c3f","target":"record","created_at":"2026-07-05T08:37:07Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"879535f07ff1e7c9997607bbbbf9339dcfaff121b6e0348d7a304343edb236bc","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-25T16:13:53Z","title_canon_sha256":"435ff44ee88e89f6b6a29c16d9cbe883f985762c9878acac60d4f8c73a6fbd7f"},"schema_version":"1.0","source":{"id":"2406.17681","kind":"arxiv","version":2}},"canonical_sha256":"96b427ecc41355852c318c6c4ac4a7ee2ad811a82fe5f1c7bb374a3880646a67","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"96b427ecc41355852c318c6c4ac4a7ee2ad811a82fe5f1c7bb374a3880646a67","first_computed_at":"2026-07-05T08:37:07.533625Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:37:07.533625Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"lKrpclq0N7DxKO053E87G78iTXllWGBnMIDHJZAx0qcND4msi4phyk1ygAC4X+O2TmMaWJSX4FoE2Ob4lHx9AA==","signature_status":"signed_v1","signed_at":"2026-07-05T08:37:07.534091Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.17681","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1684932de65d5747a60b23de887d38741d40f198bb6c375491c111600ec78c3f","sha256:54ea5c68e9d69b63c0526403003d9e25443ca0fc229020c4b46e970891f74858"],"state_sha256":"2648ad5d6ee89569c85f76db6227a0f59cedae19c34a07987d3b7373b1e64068"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"5DKvmJGw6GMDGXmM4yJiS8Efu1ptCgfaOkAbnjL2Pu+dxvAUL7AoL3ojA+CSdqgeWQ1c9CwomCLuCEHIZLTJAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T16:39:49.344756Z","bundle_sha256":"2b9428d135b24f094da846a2afdc564e921bcb609b1913d5f00df272ac4e6636"}}