{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:2MCH4PMMNUF3USOMNKOESKCW6W","short_pith_number":"pith:2MCH4PMM","canonical_record":{"source":{"id":"2210.07566","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-14T06:44:14Z","cross_cats_sorted":["cs.CV"],"title_canon_sha256":"e4ab9bfde1a14bfb1a61d9da74953226350e8a58dfda5bb07f90ef6519819382","abstract_canon_sha256":"664c0f58eaceba33d30ff8b8aab66c4087c60fb461ae73e74f08cbce99ddc529"},"schema_version":"1.0"},"canonical_sha256":"d3047e3d8c6d0bba49cc6a9c492856f5888bf45921429710539ed881e47b82b3","source":{"kind":"arxiv","id":"2210.07566","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2210.07566","created_at":"2026-07-05T05:06:38Z"},{"alias_kind":"arxiv_version","alias_value":"2210.07566v1","created_at":"2026-07-05T05:06:38Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2210.07566","created_at":"2026-07-05T05:06:38Z"},{"alias_kind":"pith_short_12","alias_value":"2MCH4PMMNUF3","created_at":"2026-07-05T05:06:38Z"},{"alias_kind":"pith_short_16","alias_value":"2MCH4PMMNUF3USOM","created_at":"2026-07-05T05:06:38Z"},{"alias_kind":"pith_short_8","alias_value":"2MCH4PMM","created_at":"2026-07-05T05:06:38Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:2MCH4PMMNUF3USOMNKOESKCW6W","target":"record","payload":{"canonical_record":{"source":{"id":"2210.07566","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-14T06:44:14Z","cross_cats_sorted":["cs.CV"],"title_canon_sha256":"e4ab9bfde1a14bfb1a61d9da74953226350e8a58dfda5bb07f90ef6519819382","abstract_canon_sha256":"664c0f58eaceba33d30ff8b8aab66c4087c60fb461ae73e74f08cbce99ddc529"},"schema_version":"1.0"},"canonical_sha256":"d3047e3d8c6d0bba49cc6a9c492856f5888bf45921429710539ed881e47b82b3","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:06:38.843065Z","signature_b64":"8zIIgi3vpypJtPjs8ujysGDQsKBjpygp3YdwstQqWM1P5cc9PwoyV3MEJSoQ2mPj8xpGvNTGkDZsQbTJyc5yCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d3047e3d8c6d0bba49cc6a9c492856f5888bf45921429710539ed881e47b82b3","last_reissued_at":"2026-07-05T05:06:38.842719Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:06:38.842719Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2210.07566","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:06:38Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"VUVFGg/1WZRXPx+/ZcZ1z60E1mXQoYaVLFCiS2wsqu6I0Kit6jfbWcCwrFjP2zB+ZKljPorjusBSbxiWiCKbDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T15:36:40.808040Z"},"content_sha256":"21296b6d6cfeec256e04b5c09d7bee433ecef6108d101476e5d6c28815ee8a67","schema_version":"1.0","event_id":"sha256:21296b6d6cfeec256e04b5c09d7bee433ecef6108d101476e5d6c28815ee8a67"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:2MCH4PMMNUF3USOMNKOESKCW6W","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"A Survey of Parameters Associated with the Quality of Benchmarks in NLP","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CV"],"primary_cat":"cs.CL","authors_text":"Anjana Arunkumar, Chitta Baral, Chris Bryan, Swaroop Mishra","submitted_at":"2022-10-14T06:44:14Z","abstract_excerpt":"Several benchmarks have been built with heavy investment in resources to track our progress in NLP. Thousands of papers published in response to those benchmarks have competed to top leaderboards, with models often surpassing human performance. However, recent studies have shown that models triumph over several popular benchmarks just by overfitting on spurious biases, without truly learning the desired task. Despite this finding, benchmarking, while trying to tackle bias, still relies on workarounds, which do not fully utilize the resources invested in benchmark creation, due to the discardin"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2210.07566","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2210.07566/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:06:38Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"hswacc7SdwU0PlSAuKX9naNiVRPXjrHH2bHaU1CpaKPBVfAOFyXJpyBWvjWex4JovoYdsreUtdOujqNO+NRuDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T15:36:40.808681Z"},"content_sha256":"8dd629e46622f326d83e415495ddf5da9906bfc3cc87c63ac11b36c665c7719a","schema_version":"1.0","event_id":"sha256:8dd629e46622f326d83e415495ddf5da9906bfc3cc87c63ac11b36c665c7719a"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/2MCH4PMMNUF3USOMNKOESKCW6W/bundle.json","state_url":"https://pith.science/pith/2MCH4PMMNUF3USOMNKOESKCW6W/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/2MCH4PMMNUF3USOMNKOESKCW6W/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-03T15:36:40Z","links":{"resolver":"https://pith.science/pith/2MCH4PMMNUF3USOMNKOESKCW6W","bundle":"https://pith.science/pith/2MCH4PMMNUF3USOMNKOESKCW6W/bundle.json","state":"https://pith.science/pith/2MCH4PMMNUF3USOMNKOESKCW6W/state.json","well_known_bundle":"https://pith.science/.well-known/pith/2MCH4PMMNUF3USOMNKOESKCW6W/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:2MCH4PMMNUF3USOMNKOESKCW6W","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"664c0f58eaceba33d30ff8b8aab66c4087c60fb461ae73e74f08cbce99ddc529","cross_cats_sorted":["cs.CV"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-14T06:44:14Z","title_canon_sha256":"e4ab9bfde1a14bfb1a61d9da74953226350e8a58dfda5bb07f90ef6519819382"},"schema_version":"1.0","source":{"id":"2210.07566","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2210.07566","created_at":"2026-07-05T05:06:38Z"},{"alias_kind":"arxiv_version","alias_value":"2210.07566v1","created_at":"2026-07-05T05:06:38Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2210.07566","created_at":"2026-07-05T05:06:38Z"},{"alias_kind":"pith_short_12","alias_value":"2MCH4PMMNUF3","created_at":"2026-07-05T05:06:38Z"},{"alias_kind":"pith_short_16","alias_value":"2MCH4PMMNUF3USOM","created_at":"2026-07-05T05:06:38Z"},{"alias_kind":"pith_short_8","alias_value":"2MCH4PMM","created_at":"2026-07-05T05:06:38Z"}],"graph_snapshots":[{"event_id":"sha256:8dd629e46622f326d83e415495ddf5da9906bfc3cc87c63ac11b36c665c7719a","target":"graph","created_at":"2026-07-05T05:06:38Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2210.07566/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Several benchmarks have been built with heavy investment in resources to track our progress in NLP. Thousands of papers published in response to those benchmarks have competed to top leaderboards, with models often surpassing human performance. However, recent studies have shown that models triumph over several popular benchmarks just by overfitting on spurious biases, without truly learning the desired task. Despite this finding, benchmarking, while trying to tackle bias, still relies on workarounds, which do not fully utilize the resources invested in benchmark creation, due to the discardin","authors_text":"Anjana Arunkumar, Chitta Baral, Chris Bryan, Swaroop Mishra","cross_cats":["cs.CV"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-14T06:44:14Z","title":"A Survey of Parameters Associated with the Quality of Benchmarks in NLP"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2210.07566","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:21296b6d6cfeec256e04b5c09d7bee433ecef6108d101476e5d6c28815ee8a67","target":"record","created_at":"2026-07-05T05:06:38Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"664c0f58eaceba33d30ff8b8aab66c4087c60fb461ae73e74f08cbce99ddc529","cross_cats_sorted":["cs.CV"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-10-14T06:44:14Z","title_canon_sha256":"e4ab9bfde1a14bfb1a61d9da74953226350e8a58dfda5bb07f90ef6519819382"},"schema_version":"1.0","source":{"id":"2210.07566","kind":"arxiv","version":1}},"canonical_sha256":"d3047e3d8c6d0bba49cc6a9c492856f5888bf45921429710539ed881e47b82b3","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"d3047e3d8c6d0bba49cc6a9c492856f5888bf45921429710539ed881e47b82b3","first_computed_at":"2026-07-05T05:06:38.842719Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:06:38.842719Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"8zIIgi3vpypJtPjs8ujysGDQsKBjpygp3YdwstQqWM1P5cc9PwoyV3MEJSoQ2mPj8xpGvNTGkDZsQbTJyc5yCw==","signature_status":"signed_v1","signed_at":"2026-07-05T05:06:38.843065Z","signed_message":"canonical_sha256_bytes"},"source_id":"2210.07566","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:21296b6d6cfeec256e04b5c09d7bee433ecef6108d101476e5d6c28815ee8a67","sha256:8dd629e46622f326d83e415495ddf5da9906bfc3cc87c63ac11b36c665c7719a"],"state_sha256":"527e50b34f401c34fce1b644b11e6cd532daaa451bde38b40ebb2d1b90e00140"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jH2DOUyjGcig6ORd7iCNeMcHf/5G/5tFqM0n1bil+/fkq2E/20tL4yygafdt+Eccr7pB91O7hGe6eYI/KYhSAQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-03T15:36:40.813676Z","bundle_sha256":"032611d617ecf806b391d11aef15de04a26b542b323ba0f79b60e8795bcd071a"}}