{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2017:CZ657XJEO3D3UEQDEOCOZVVTT7","short_pith_number":"pith:CZ657XJE","canonical_record":{"source":{"id":"1712.01765","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2017-12-05T17:13:48Z","cross_cats_sorted":[],"title_canon_sha256":"957c6654ef557ae0e1ba79f4e78ecca003282d3c912139fec67af464ecc48139","abstract_canon_sha256":"34f691ab510170bc42e5bf5340bddb318137a3a262807aa31bfe716aa2a3723f"},"schema_version":"1.0"},"canonical_sha256":"167ddfdd2476c7ba12032384ecd6b39ff2a5fdce22528b2321d05e6553343181","source":{"kind":"arxiv","id":"1712.01765","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1712.01765","created_at":"2026-05-18T00:28:47Z"},{"alias_kind":"arxiv_version","alias_value":"1712.01765v1","created_at":"2026-05-18T00:28:47Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1712.01765","created_at":"2026-05-18T00:28:47Z"},{"alias_kind":"pith_short_12","alias_value":"CZ657XJEO3D3","created_at":"2026-05-18T12:31:10Z"},{"alias_kind":"pith_short_16","alias_value":"CZ657XJEO3D3UEQD","created_at":"2026-05-18T12:31:10Z"},{"alias_kind":"pith_short_8","alias_value":"CZ657XJE","created_at":"2026-05-18T12:31:10Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2017:CZ657XJEO3D3UEQDEOCOZVVTT7","target":"record","payload":{"canonical_record":{"source":{"id":"1712.01765","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2017-12-05T17:13:48Z","cross_cats_sorted":[],"title_canon_sha256":"957c6654ef557ae0e1ba79f4e78ecca003282d3c912139fec67af464ecc48139","abstract_canon_sha256":"34f691ab510170bc42e5bf5340bddb318137a3a262807aa31bfe716aa2a3723f"},"schema_version":"1.0"},"canonical_sha256":"167ddfdd2476c7ba12032384ecd6b39ff2a5fdce22528b2321d05e6553343181","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T00:28:47.616642Z","signature_b64":"1DlqRg5618RRemsdrGcaH5mdfj7pVtLvZsmZ/tdEZGs1spMeel2CKxoda9UrRvN0bNLFUWyovaZ/GiW0gHV8Aw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"167ddfdd2476c7ba12032384ecd6b39ff2a5fdce22528b2321d05e6553343181","last_reissued_at":"2026-05-18T00:28:47.615963Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T00:28:47.615963Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1712.01765","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T00:28:47Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jHPYGyyVXqtCRguqyIvprfZmNzDEj9feDjJEi9ZWqiN7BNSMiDxkBWjS2095qu0uXB2aXdiwCeSuEY7uw3AVAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-05T21:13:07.518639Z"},"content_sha256":"f58ce4791473e48f855600e0a5c41a16f2eb92d6fc10d58d89f110a9ecbb78f6","schema_version":"1.0","event_id":"sha256:f58ce4791473e48f855600e0a5c41a16f2eb92d6fc10d58d89f110a9ecbb78f6"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2017:CZ657XJEO3D3UEQDEOCOZVVTT7","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Best-Worst Scaling More Reliable than Rating Scales: A Case Study on Sentiment Intensity Annotation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Saif M. Mohammad, Svetlana Kiritchenko","submitted_at":"2017-12-05T17:13:48Z","abstract_excerpt":"Rating scales are a widely used method for data annotation; however, they present several challenges, such as difficulty in maintaining inter- and intra-annotator consistency. Best-worst scaling (BWS) is an alternative method of annotation that is claimed to produce high-quality annotations while keeping the required number of annotations similar to that of rating scales. However, the veracity of this claim has never been systematically established. Here for the first time, we set up an experiment that directly compares the rating scale method with BWS. We show that with the same total number "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1712.01765","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T00:28:47Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"2inhnIBzlVF4gNChy7H/bgYaN5BlR4SgQpOUfRmXt1062isW4Hsd1qVYNLrODKW1FM+sgyzG6gGGIMUejLErBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-05T21:13:07.518996Z"},"content_sha256":"bc0d6cae2d70770726d1f975570f2388e76a4f76883b13ee322e512bf9908209","schema_version":"1.0","event_id":"sha256:bc0d6cae2d70770726d1f975570f2388e76a4f76883b13ee322e512bf9908209"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/CZ657XJEO3D3UEQDEOCOZVVTT7/bundle.json","state_url":"https://pith.science/pith/CZ657XJEO3D3UEQDEOCOZVVTT7/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/CZ657XJEO3D3UEQDEOCOZVVTT7/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-06-05T21:13:07Z","links":{"resolver":"https://pith.science/pith/CZ657XJEO3D3UEQDEOCOZVVTT7","bundle":"https://pith.science/pith/CZ657XJEO3D3UEQDEOCOZVVTT7/bundle.json","state":"https://pith.science/pith/CZ657XJEO3D3UEQDEOCOZVVTT7/state.json","well_known_bundle":"https://pith.science/.well-known/pith/CZ657XJEO3D3UEQDEOCOZVVTT7/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2017:CZ657XJEO3D3UEQDEOCOZVVTT7","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"34f691ab510170bc42e5bf5340bddb318137a3a262807aa31bfe716aa2a3723f","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2017-12-05T17:13:48Z","title_canon_sha256":"957c6654ef557ae0e1ba79f4e78ecca003282d3c912139fec67af464ecc48139"},"schema_version":"1.0","source":{"id":"1712.01765","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1712.01765","created_at":"2026-05-18T00:28:47Z"},{"alias_kind":"arxiv_version","alias_value":"1712.01765v1","created_at":"2026-05-18T00:28:47Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1712.01765","created_at":"2026-05-18T00:28:47Z"},{"alias_kind":"pith_short_12","alias_value":"CZ657XJEO3D3","created_at":"2026-05-18T12:31:10Z"},{"alias_kind":"pith_short_16","alias_value":"CZ657XJEO3D3UEQD","created_at":"2026-05-18T12:31:10Z"},{"alias_kind":"pith_short_8","alias_value":"CZ657XJE","created_at":"2026-05-18T12:31:10Z"}],"graph_snapshots":[{"event_id":"sha256:bc0d6cae2d70770726d1f975570f2388e76a4f76883b13ee322e512bf9908209","target":"graph","created_at":"2026-05-18T00:28:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"Rating scales are a widely used method for data annotation; however, they present several challenges, such as difficulty in maintaining inter- and intra-annotator consistency. Best-worst scaling (BWS) is an alternative method of annotation that is claimed to produce high-quality annotations while keeping the required number of annotations similar to that of rating scales. However, the veracity of this claim has never been systematically established. Here for the first time, we set up an experiment that directly compares the rating scale method with BWS. We show that with the same total number ","authors_text":"Saif M. Mohammad, Svetlana Kiritchenko","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2017-12-05T17:13:48Z","title":"Best-Worst Scaling More Reliable than Rating Scales: A Case Study on Sentiment Intensity Annotation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1712.01765","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:f58ce4791473e48f855600e0a5c41a16f2eb92d6fc10d58d89f110a9ecbb78f6","target":"record","created_at":"2026-05-18T00:28:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"34f691ab510170bc42e5bf5340bddb318137a3a262807aa31bfe716aa2a3723f","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2017-12-05T17:13:48Z","title_canon_sha256":"957c6654ef557ae0e1ba79f4e78ecca003282d3c912139fec67af464ecc48139"},"schema_version":"1.0","source":{"id":"1712.01765","kind":"arxiv","version":1}},"canonical_sha256":"167ddfdd2476c7ba12032384ecd6b39ff2a5fdce22528b2321d05e6553343181","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"167ddfdd2476c7ba12032384ecd6b39ff2a5fdce22528b2321d05e6553343181","first_computed_at":"2026-05-18T00:28:47.615963Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-18T00:28:47.615963Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"1DlqRg5618RRemsdrGcaH5mdfj7pVtLvZsmZ/tdEZGs1spMeel2CKxoda9UrRvN0bNLFUWyovaZ/GiW0gHV8Aw==","signature_status":"signed_v1","signed_at":"2026-05-18T00:28:47.616642Z","signed_message":"canonical_sha256_bytes"},"source_id":"1712.01765","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:f58ce4791473e48f855600e0a5c41a16f2eb92d6fc10d58d89f110a9ecbb78f6","sha256:bc0d6cae2d70770726d1f975570f2388e76a4f76883b13ee322e512bf9908209"],"state_sha256":"622faa342b899b6db74411702a5dea917b0d50fb75fcff6a5a5a0e18fd526c00"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"sR8Z6zNilb5TiJQhrj8N8vnuZvKwy1fiDs+QemGsCE4Ho5KveCuftSrWk5JNKiLGCxg/ezx6NwN4/3r8GBoQDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-06-05T21:13:07.521152Z","bundle_sha256":"06dccf465301bcea9aae1483f3091d4b5e65f525961494e9f2965bec7ec6022f"}}