{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:4CAA2XJ2A2XT4VAJO2DVZWL532","short_pith_number":"pith:4CAA2XJ2","canonical_record":{"source":{"id":"2111.08181","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-11-16T01:45:26Z","cross_cats_sorted":[],"title_canon_sha256":"b9fd6016325a5b5a5745189613bf84bffa41e5f14c4f511496c1e251fd8631e9","abstract_canon_sha256":"6e321dda920b7960673150804381cfe0ad4c6b82bb490f3a20d894ef8e852636"},"schema_version":"1.0"},"canonical_sha256":"e0800d5d3a06af3e540976875cd97ddea3b23a644f06e96b4428c2e3e50e655a","source":{"kind":"arxiv","id":"2111.08181","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2111.08181","created_at":"2026-07-05T03:32:30Z"},{"alias_kind":"arxiv_version","alias_value":"2111.08181v1","created_at":"2026-07-05T03:32:30Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2111.08181","created_at":"2026-07-05T03:32:30Z"},{"alias_kind":"pith_short_12","alias_value":"4CAA2XJ2A2XT","created_at":"2026-07-05T03:32:30Z"},{"alias_kind":"pith_short_16","alias_value":"4CAA2XJ2A2XT4VAJ","created_at":"2026-07-05T03:32:30Z"},{"alias_kind":"pith_short_8","alias_value":"4CAA2XJ2","created_at":"2026-07-05T03:32:30Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:4CAA2XJ2A2XT4VAJO2DVZWL532","target":"record","payload":{"canonical_record":{"source":{"id":"2111.08181","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-11-16T01:45:26Z","cross_cats_sorted":[],"title_canon_sha256":"b9fd6016325a5b5a5745189613bf84bffa41e5f14c4f511496c1e251fd8631e9","abstract_canon_sha256":"6e321dda920b7960673150804381cfe0ad4c6b82bb490f3a20d894ef8e852636"},"schema_version":"1.0"},"canonical_sha256":"e0800d5d3a06af3e540976875cd97ddea3b23a644f06e96b4428c2e3e50e655a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T03:32:30.636880Z","signature_b64":"XYn+e3VCW2Ek9ZbU+HxaUYSLmFT4WWpLGupLPki5h2aFDI/A0/itnDzUggFE1pxyOThjQi0F35yc51pvqCjHBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"e0800d5d3a06af3e540976875cd97ddea3b23a644f06e96b4428c2e3e50e655a","last_reissued_at":"2026-07-05T03:32:30.636386Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T03:32:30.636386Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2111.08181","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T03:32:30Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"+BbmDYiQ/pWMMVR8V6dp5DmJaEfFKmfSuR5sovJAlHh24paQYCUpIhsSDk/Zxjvu0iVPBolnSkHCak3wt7pCBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T00:30:39.101071Z"},"content_sha256":"5a3152300148e33d74ed171c879267ba1a90d2c30a3f4d7cf613ee8c21b2680d","schema_version":"1.0","event_id":"sha256:5a3152300148e33d74ed171c879267ba1a90d2c30a3f4d7cf613ee8c21b2680d"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:4CAA2XJ2A2XT4VAJO2DVZWL532","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Adversarially Constructed Evaluation Sets Are More Challenging, but May Not Be Fair","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Angelica Chen, Jason Phang, Samuel R. Bowman, William Huang","submitted_at":"2021-11-16T01:45:26Z","abstract_excerpt":"More capable language models increasingly saturate existing task benchmarks, in some cases outperforming humans. This has left little headroom with which to measure further progress. Adversarial dataset creation has been proposed as a strategy to construct more challenging datasets, and two common approaches are: (1) filtering out easy examples and (2) model-in-the-loop data collection. In this work, we study the impact of applying each approach to create more challenging evaluation datasets. We adapt the AFLite algorithm to filter evaluation data, and run experiments against 18 different adve"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2111.08181","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2111.08181/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T03:32:30Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"zqaEPPNByYYzjuh0mOiER4jnYlr71kjFOAKPvg5skLGOaYAH+jSLRjlLKOq13snAMdCHWGqLeZllYeg9dEhwBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T00:30:39.102089Z"},"content_sha256":"c6e51a68a26d25da2fe7d950a43deba98754df8653218e491c89d05efb6c2431","schema_version":"1.0","event_id":"sha256:c6e51a68a26d25da2fe7d950a43deba98754df8653218e491c89d05efb6c2431"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/4CAA2XJ2A2XT4VAJO2DVZWL532/bundle.json","state_url":"https://pith.science/pith/4CAA2XJ2A2XT4VAJO2DVZWL532/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/4CAA2XJ2A2XT4VAJO2DVZWL532/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-04T00:30:39Z","links":{"resolver":"https://pith.science/pith/4CAA2XJ2A2XT4VAJO2DVZWL532","bundle":"https://pith.science/pith/4CAA2XJ2A2XT4VAJO2DVZWL532/bundle.json","state":"https://pith.science/pith/4CAA2XJ2A2XT4VAJO2DVZWL532/state.json","well_known_bundle":"https://pith.science/.well-known/pith/4CAA2XJ2A2XT4VAJO2DVZWL532/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:4CAA2XJ2A2XT4VAJO2DVZWL532","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"6e321dda920b7960673150804381cfe0ad4c6b82bb490f3a20d894ef8e852636","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-11-16T01:45:26Z","title_canon_sha256":"b9fd6016325a5b5a5745189613bf84bffa41e5f14c4f511496c1e251fd8631e9"},"schema_version":"1.0","source":{"id":"2111.08181","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2111.08181","created_at":"2026-07-05T03:32:30Z"},{"alias_kind":"arxiv_version","alias_value":"2111.08181v1","created_at":"2026-07-05T03:32:30Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2111.08181","created_at":"2026-07-05T03:32:30Z"},{"alias_kind":"pith_short_12","alias_value":"4CAA2XJ2A2XT","created_at":"2026-07-05T03:32:30Z"},{"alias_kind":"pith_short_16","alias_value":"4CAA2XJ2A2XT4VAJ","created_at":"2026-07-05T03:32:30Z"},{"alias_kind":"pith_short_8","alias_value":"4CAA2XJ2","created_at":"2026-07-05T03:32:30Z"}],"graph_snapshots":[{"event_id":"sha256:c6e51a68a26d25da2fe7d950a43deba98754df8653218e491c89d05efb6c2431","target":"graph","created_at":"2026-07-05T03:32:30Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2111.08181/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"More capable language models increasingly saturate existing task benchmarks, in some cases outperforming humans. This has left little headroom with which to measure further progress. Adversarial dataset creation has been proposed as a strategy to construct more challenging datasets, and two common approaches are: (1) filtering out easy examples and (2) model-in-the-loop data collection. In this work, we study the impact of applying each approach to create more challenging evaluation datasets. We adapt the AFLite algorithm to filter evaluation data, and run experiments against 18 different adve","authors_text":"Angelica Chen, Jason Phang, Samuel R. Bowman, William Huang","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-11-16T01:45:26Z","title":"Adversarially Constructed Evaluation Sets Are More Challenging, but May Not Be Fair"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2111.08181","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5a3152300148e33d74ed171c879267ba1a90d2c30a3f4d7cf613ee8c21b2680d","target":"record","created_at":"2026-07-05T03:32:30Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"6e321dda920b7960673150804381cfe0ad4c6b82bb490f3a20d894ef8e852636","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-11-16T01:45:26Z","title_canon_sha256":"b9fd6016325a5b5a5745189613bf84bffa41e5f14c4f511496c1e251fd8631e9"},"schema_version":"1.0","source":{"id":"2111.08181","kind":"arxiv","version":1}},"canonical_sha256":"e0800d5d3a06af3e540976875cd97ddea3b23a644f06e96b4428c2e3e50e655a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"e0800d5d3a06af3e540976875cd97ddea3b23a644f06e96b4428c2e3e50e655a","first_computed_at":"2026-07-05T03:32:30.636386Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T03:32:30.636386Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"XYn+e3VCW2Ek9ZbU+HxaUYSLmFT4WWpLGupLPki5h2aFDI/A0/itnDzUggFE1pxyOThjQi0F35yc51pvqCjHBg==","signature_status":"signed_v1","signed_at":"2026-07-05T03:32:30.636880Z","signed_message":"canonical_sha256_bytes"},"source_id":"2111.08181","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5a3152300148e33d74ed171c879267ba1a90d2c30a3f4d7cf613ee8c21b2680d","sha256:c6e51a68a26d25da2fe7d950a43deba98754df8653218e491c89d05efb6c2431"],"state_sha256":"e6d4296abb56e21d051d0cceec0df0c348be428091c6048aac3e5557edd658ee"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Oi+caSvXH26x6s4I+uCLLfmNa4tX3uFLcJoLvuzN8/NfJsD4ljr64/qM1Tm1o7/eh8TC/4vnLI6JgVUbXggcDQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-04T00:30:39.111498Z","bundle_sha256":"acb8d2d6d0b90c45da66aa67b88ae688e4a845c58ff7b6a549637f15428a5b26"}}