{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:2G4G5OIKPBO46FBEAPZFHBI7XI","short_pith_number":"pith:2G4G5OIK","canonical_record":{"source":{"id":"2104.07179","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-04-15T00:31:10Z","cross_cats_sorted":[],"title_canon_sha256":"c6f451f74d223ca0fa22621308716134d73b4f4cb40d592230c8196fc14a9510","abstract_canon_sha256":"2409b4f344a0ef22ce1a05a03e8393348f91fd2aa328dee000f9d0af823939b5"},"schema_version":"1.0"},"canonical_sha256":"d1b86eb90a785dcf142403f253851fba066b4675fa38771f746f6acc7740e987","source":{"kind":"arxiv","id":"2104.07179","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2104.07179","created_at":"2026-07-05T02:32:18Z"},{"alias_kind":"arxiv_version","alias_value":"2104.07179v1","created_at":"2026-07-05T02:32:18Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2104.07179","created_at":"2026-07-05T02:32:18Z"},{"alias_kind":"pith_short_12","alias_value":"2G4G5OIKPBO4","created_at":"2026-07-05T02:32:18Z"},{"alias_kind":"pith_short_16","alias_value":"2G4G5OIKPBO46FBE","created_at":"2026-07-05T02:32:18Z"},{"alias_kind":"pith_short_8","alias_value":"2G4G5OIK","created_at":"2026-07-05T02:32:18Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:2G4G5OIKPBO46FBEAPZFHBI7XI","target":"record","payload":{"canonical_record":{"source":{"id":"2104.07179","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-04-15T00:31:10Z","cross_cats_sorted":[],"title_canon_sha256":"c6f451f74d223ca0fa22621308716134d73b4f4cb40d592230c8196fc14a9510","abstract_canon_sha256":"2409b4f344a0ef22ce1a05a03e8393348f91fd2aa328dee000f9d0af823939b5"},"schema_version":"1.0"},"canonical_sha256":"d1b86eb90a785dcf142403f253851fba066b4675fa38771f746f6acc7740e987","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:32:18.782288Z","signature_b64":"UevHjZ4meEotVx4JbsjsKMl0wRV4StJ+gWpWw5ylDR9SXAUzcge6NFf35W3fnXJad4E5atwn5X4PFUcksW/vCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d1b86eb90a785dcf142403f253851fba066b4675fa38771f746f6acc7740e987","last_reissued_at":"2026-07-05T02:32:18.781879Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:32:18.781879Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2104.07179","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:32:18Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"I4wY82/pdyNM+IbB7tDCgJ3QJFgXl2L2LEP7jI9l1oWYQuvjUgQHgiuP3Z+lcIsE/Q7fRWV95eUvs8xbC5W1DQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T07:41:55.590958Z"},"content_sha256":"0cca439d3b4700d1fceb94da9682a89557db6b4d75749a6d9488eb81ec722f0e","schema_version":"1.0","event_id":"sha256:0cca439d3b4700d1fceb94da9682a89557db6b4d75749a6d9488eb81ec722f0e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:2G4G5OIKPBO46FBEAPZFHBI7XI","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Does Putting a Linguist in the Loop Improve NLU Data Collection?","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Alex Warstadt, Alicia Parrish, Emily Allaway, Karmanya Aggarwal, Nikita Nangia, Omar Agha, Samuel R. Bowman, Soo-Hwan Lee, Tal Linzen, William Huang","submitted_at":"2021-04-15T00:31:10Z","abstract_excerpt":"Many crowdsourced NLP datasets contain systematic gaps and biases that are identified only after data collection is complete. Identifying these issues from early data samples during crowdsourcing should make mitigation more efficient, especially when done iteratively. We take natural language inference as a test case and ask whether it is beneficial to put a linguist `in the loop' during data collection to dynamically identify and address gaps in the data by introducing novel constraints on the task. We directly compare three data collection protocols: (i) a baseline protocol, (ii) a linguist-"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2104.07179","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2104.07179/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:32:18Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"f4mjAlxgCbxQl4bMj9uLLPF6J4pHQwIkfeJvewYkpxQl8R8tSitn7q9UKw6f21CLq0WaDXk/nlqkCodq/9sHCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T07:41:55.591433Z"},"content_sha256":"f75633b6445be300bfa852e868070d4176b72907fc048028783009cd6629b914","schema_version":"1.0","event_id":"sha256:f75633b6445be300bfa852e868070d4176b72907fc048028783009cd6629b914"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/2G4G5OIKPBO46FBEAPZFHBI7XI/bundle.json","state_url":"https://pith.science/pith/2G4G5OIKPBO46FBEAPZFHBI7XI/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/2G4G5OIKPBO46FBEAPZFHBI7XI/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-05T07:41:55Z","links":{"resolver":"https://pith.science/pith/2G4G5OIKPBO46FBEAPZFHBI7XI","bundle":"https://pith.science/pith/2G4G5OIKPBO46FBEAPZFHBI7XI/bundle.json","state":"https://pith.science/pith/2G4G5OIKPBO46FBEAPZFHBI7XI/state.json","well_known_bundle":"https://pith.science/.well-known/pith/2G4G5OIKPBO46FBEAPZFHBI7XI/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:2G4G5OIKPBO46FBEAPZFHBI7XI","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"2409b4f344a0ef22ce1a05a03e8393348f91fd2aa328dee000f9d0af823939b5","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-04-15T00:31:10Z","title_canon_sha256":"c6f451f74d223ca0fa22621308716134d73b4f4cb40d592230c8196fc14a9510"},"schema_version":"1.0","source":{"id":"2104.07179","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2104.07179","created_at":"2026-07-05T02:32:18Z"},{"alias_kind":"arxiv_version","alias_value":"2104.07179v1","created_at":"2026-07-05T02:32:18Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2104.07179","created_at":"2026-07-05T02:32:18Z"},{"alias_kind":"pith_short_12","alias_value":"2G4G5OIKPBO4","created_at":"2026-07-05T02:32:18Z"},{"alias_kind":"pith_short_16","alias_value":"2G4G5OIKPBO46FBE","created_at":"2026-07-05T02:32:18Z"},{"alias_kind":"pith_short_8","alias_value":"2G4G5OIK","created_at":"2026-07-05T02:32:18Z"}],"graph_snapshots":[{"event_id":"sha256:f75633b6445be300bfa852e868070d4176b72907fc048028783009cd6629b914","target":"graph","created_at":"2026-07-05T02:32:18Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2104.07179/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Many crowdsourced NLP datasets contain systematic gaps and biases that are identified only after data collection is complete. Identifying these issues from early data samples during crowdsourcing should make mitigation more efficient, especially when done iteratively. We take natural language inference as a test case and ask whether it is beneficial to put a linguist `in the loop' during data collection to dynamically identify and address gaps in the data by introducing novel constraints on the task. We directly compare three data collection protocols: (i) a baseline protocol, (ii) a linguist-","authors_text":"Alex Warstadt, Alicia Parrish, Emily Allaway, Karmanya Aggarwal, Nikita Nangia, Omar Agha, Samuel R. Bowman, Soo-Hwan Lee, Tal Linzen, William Huang","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-04-15T00:31:10Z","title":"Does Putting a Linguist in the Loop Improve NLU Data Collection?"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2104.07179","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:0cca439d3b4700d1fceb94da9682a89557db6b4d75749a6d9488eb81ec722f0e","target":"record","created_at":"2026-07-05T02:32:18Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"2409b4f344a0ef22ce1a05a03e8393348f91fd2aa328dee000f9d0af823939b5","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-04-15T00:31:10Z","title_canon_sha256":"c6f451f74d223ca0fa22621308716134d73b4f4cb40d592230c8196fc14a9510"},"schema_version":"1.0","source":{"id":"2104.07179","kind":"arxiv","version":1}},"canonical_sha256":"d1b86eb90a785dcf142403f253851fba066b4675fa38771f746f6acc7740e987","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"d1b86eb90a785dcf142403f253851fba066b4675fa38771f746f6acc7740e987","first_computed_at":"2026-07-05T02:32:18.781879Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T02:32:18.781879Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"UevHjZ4meEotVx4JbsjsKMl0wRV4StJ+gWpWw5ylDR9SXAUzcge6NFf35W3fnXJad4E5atwn5X4PFUcksW/vCg==","signature_status":"signed_v1","signed_at":"2026-07-05T02:32:18.782288Z","signed_message":"canonical_sha256_bytes"},"source_id":"2104.07179","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:0cca439d3b4700d1fceb94da9682a89557db6b4d75749a6d9488eb81ec722f0e","sha256:f75633b6445be300bfa852e868070d4176b72907fc048028783009cd6629b914"],"state_sha256":"9e508b0f0447e7c7f6dd6bdccc67738a7e9f624bbe8f345b341b6af6ecd5e6f5"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"amard/ZictOeiJDDrItHCT2rrJFsCKkF5+hT9Rwx3UPYaWW2HnA8r+jkEERKLVuZdb+yDfzt/x9fzSN5IZH+Cw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-05T07:41:55.594720Z","bundle_sha256":"4526668102b98d39587a71c9e977aaa241ebea7cdaf8b7422e8c92a0b4a98c0c"}}