{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:VQ56JDWE2KXMCALE43SWMDO4OE","short_pith_number":"pith:VQ56JDWE","schema_version":"1.0","canonical_sha256":"ac3be48ec4d2aec10164e6e5660ddc7106f3a062a33d9766f89c583af3b1be21","source":{"kind":"arxiv","id":"2408.00880","version":1},"attestation_state":"computed","paper":{"title":"Annotator in the Loop: A Case Study of In-Depth Rater Engagement to Create a Bridging Benchmark Dataset","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CY","authors_text":"Alyssa Chvasta, Emily Saltz, Ruta Wheelock, Scott Friedman, Sonja Schmer-Galunder, Zaria Jalan","submitted_at":"2024-08-01T19:11:08Z","abstract_excerpt":"With the growing prevalence of large language models, it is increasingly common to annotate datasets for machine learning using pools of crowd raters. However, these raters often work in isolation as individual crowdworkers. In this work, we regard annotation not merely as inexpensive, scalable labor, but rather as a nuanced interpretative effort to discern the meaning of what is being said in a text. We describe a novel, collaborative, and iterative annotator-in-the-loop methodology for annotation, resulting in a 'Bridging Benchmark Dataset' of comments relevant to bridging divides, annotated"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2408.00880","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CY","submitted_at":"2024-08-01T19:11:08Z","cross_cats_sorted":[],"title_canon_sha256":"7b1e725c08e740b16658201f7cba53bc16ccc3a765e66f82f263264c57b06569","abstract_canon_sha256":"1798d8084d185a21d330231bff44c655d60c6df0da5bed90c00998b7c33b9ad7"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:51:23.341139Z","signature_b64":"XUlCOAWp2DD8VAekbUf5oba/Rcs6FESwDl0MTyjWmWr09qHTQE7kDVt/3QGSsyVeqnsMRP5nCl+3oWpgXY9HCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"ac3be48ec4d2aec10164e6e5660ddc7106f3a062a33d9766f89c583af3b1be21","last_reissued_at":"2026-07-05T08:51:23.340733Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:51:23.340733Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Annotator in the Loop: A Case Study of In-Depth Rater Engagement to Create a Bridging Benchmark Dataset","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CY","authors_text":"Alyssa Chvasta, Emily Saltz, Ruta Wheelock, Scott Friedman, Sonja Schmer-Galunder, Zaria Jalan","submitted_at":"2024-08-01T19:11:08Z","abstract_excerpt":"With the growing prevalence of large language models, it is increasingly common to annotate datasets for machine learning using pools of crowd raters. However, these raters often work in isolation as individual crowdworkers. In this work, we regard annotation not merely as inexpensive, scalable labor, but rather as a nuanced interpretative effort to discern the meaning of what is being said in a text. We describe a novel, collaborative, and iterative annotator-in-the-loop methodology for annotation, resulting in a 'Bridging Benchmark Dataset' of comments relevant to bridging divides, annotated"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.00880","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2408.00880/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2408.00880","created_at":"2026-07-05T08:51:23.340789+00:00"},{"alias_kind":"arxiv_version","alias_value":"2408.00880v1","created_at":"2026-07-05T08:51:23.340789+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.00880","created_at":"2026-07-05T08:51:23.340789+00:00"},{"alias_kind":"pith_short_12","alias_value":"VQ56JDWE2KXM","created_at":"2026-07-05T08:51:23.340789+00:00"},{"alias_kind":"pith_short_16","alias_value":"VQ56JDWE2KXMCALE","created_at":"2026-07-05T08:51:23.340789+00:00"},{"alias_kind":"pith_short_8","alias_value":"VQ56JDWE","created_at":"2026-07-05T08:51:23.340789+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/VQ56JDWE2KXMCALE43SWMDO4OE","json":"https://pith.science/pith/VQ56JDWE2KXMCALE43SWMDO4OE.json","graph_json":"https://pith.science/api/pith-number/VQ56JDWE2KXMCALE43SWMDO4OE/graph.json","events_json":"https://pith.science/api/pith-number/VQ56JDWE2KXMCALE43SWMDO4OE/events.json","paper":"https://pith.science/paper/VQ56JDWE"},"agent_actions":{"view_html":"https://pith.science/pith/VQ56JDWE2KXMCALE43SWMDO4OE","download_json":"https://pith.science/pith/VQ56JDWE2KXMCALE43SWMDO4OE.json","view_paper":"https://pith.science/paper/VQ56JDWE","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2408.00880&json=true","fetch_graph":"https://pith.science/api/pith-number/VQ56JDWE2KXMCALE43SWMDO4OE/graph.json","fetch_events":"https://pith.science/api/pith-number/VQ56JDWE2KXMCALE43SWMDO4OE/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/VQ56JDWE2KXMCALE43SWMDO4OE/action/timestamp_anchor","attest_storage":"https://pith.science/pith/VQ56JDWE2KXMCALE43SWMDO4OE/action/storage_attestation","attest_author":"https://pith.science/pith/VQ56JDWE2KXMCALE43SWMDO4OE/action/author_attestation","sign_citation":"https://pith.science/pith/VQ56JDWE2KXMCALE43SWMDO4OE/action/citation_signature","submit_replication":"https://pith.science/pith/VQ56JDWE2KXMCALE43SWMDO4OE/action/replication_record"}},"created_at":"2026-07-05T08:51:23.340789+00:00","updated_at":"2026-07-05T08:51:23.340789+00:00"}