{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2018:6S6GWK5PU5FQNQLEHTGKZYAYJP","short_pith_number":"pith:6S6GWK5P","canonical_record":{"source":{"id":"1806.01170","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2018-06-04T16:10:19Z","cross_cats_sorted":[],"title_canon_sha256":"91bc1f44b2315a3817514076502d9e3c17c239d03e18eb75bd974f04cf216a69","abstract_canon_sha256":"7a239e769ba71ff6bb5d2a6aeaa66e49244376322cbf65e188165297b5482231"},"schema_version":"1.0"},"canonical_sha256":"f4bc6b2bafa74b06c1643cccace0184bf29e7a36f9f4cd5e2301f0c963b80f27","source":{"kind":"arxiv","id":"1806.01170","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1806.01170","created_at":"2026-05-18T00:14:17Z"},{"alias_kind":"arxiv_version","alias_value":"1806.01170v1","created_at":"2026-05-18T00:14:17Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1806.01170","created_at":"2026-05-18T00:14:17Z"},{"alias_kind":"pith_short_12","alias_value":"6S6GWK5PU5FQ","created_at":"2026-05-18T12:32:11Z"},{"alias_kind":"pith_short_16","alias_value":"6S6GWK5PU5FQNQLE","created_at":"2026-05-18T12:32:11Z"},{"alias_kind":"pith_short_8","alias_value":"6S6GWK5P","created_at":"2026-05-18T12:32:11Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2018:6S6GWK5PU5FQNQLEHTGKZYAYJP","target":"record","payload":{"canonical_record":{"source":{"id":"1806.01170","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2018-06-04T16:10:19Z","cross_cats_sorted":[],"title_canon_sha256":"91bc1f44b2315a3817514076502d9e3c17c239d03e18eb75bd974f04cf216a69","abstract_canon_sha256":"7a239e769ba71ff6bb5d2a6aeaa66e49244376322cbf65e188165297b5482231"},"schema_version":"1.0"},"canonical_sha256":"f4bc6b2bafa74b06c1643cccace0184bf29e7a36f9f4cd5e2301f0c963b80f27","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T00:14:17.237747Z","signature_b64":"0nsK+sHNwZErrfh85KYHSwzJAuAK7FUvsdSV9bzIfJFkAz/U8COWGEhrSnszp3XoDlx3xLJ6IJe/4+xJaAzKDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f4bc6b2bafa74b06c1643cccace0184bf29e7a36f9f4cd5e2301f0c963b80f27","last_reissued_at":"2026-05-18T00:14:17.237232Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T00:14:17.237232Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1806.01170","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T00:14:17Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Il/t0dqcK7CqEtsnxkG2bQcNjU/jgPgQ32CwoDUS9bWsrCGSpgQW+afn8ebg/wP5GCwCQnmsHluWz/urpQvEDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-04T16:54:58.145376Z"},"content_sha256":"ab955e6f877831f52ef33da1fa93bc0ded154b3c5b65ec9d869db7049b83e848","schema_version":"1.0","event_id":"sha256:ab955e6f877831f52ef33da1fa93bc0ded154b3c5b65ec9d869db7049b83e848"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2018:6S6GWK5PU5FQNQLEHTGKZYAYJP","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Efficient Online Scalar Annotation with Bounded Support","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Benjamin Van Durme, Keisuke Sakaguchi","submitted_at":"2018-06-04T16:10:19Z","abstract_excerpt":"We describe a novel method for efficiently eliciting scalar annotations for dataset construction and system quality estimation by human judgments. We contrast direct assessment (annotators assign scores to items directly), online pairwise ranking aggregation (scores derive from annotator comparison of items), and a hybrid approach (EASL: Efficient Annotation of Scalar Labels) proposed here. Our proposal leads to increased correlation with ground truth, at far greater annotator efficiency, suggesting this strategy as an improved mechanism for dataset creation and manual system evaluation."},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1806.01170","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T00:14:17Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"fOBvfisU+QVoTDAcfU8aOPMq74DysquBtOuW/2YLyJLdd+oiPUeMfKgACHa4E+g821DXPjHHtSNZnLwWlRxoCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-04T16:54:58.145769Z"},"content_sha256":"d32a591d461af6ef8e6a949fe468941c79d944340912643423c9b5da8da80177","schema_version":"1.0","event_id":"sha256:d32a591d461af6ef8e6a949fe468941c79d944340912643423c9b5da8da80177"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/6S6GWK5PU5FQNQLEHTGKZYAYJP/bundle.json","state_url":"https://pith.science/pith/6S6GWK5PU5FQNQLEHTGKZYAYJP/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/6S6GWK5PU5FQNQLEHTGKZYAYJP/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-06-04T16:54:58Z","links":{"resolver":"https://pith.science/pith/6S6GWK5PU5FQNQLEHTGKZYAYJP","bundle":"https://pith.science/pith/6S6GWK5PU5FQNQLEHTGKZYAYJP/bundle.json","state":"https://pith.science/pith/6S6GWK5PU5FQNQLEHTGKZYAYJP/state.json","well_known_bundle":"https://pith.science/.well-known/pith/6S6GWK5PU5FQNQLEHTGKZYAYJP/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2018:6S6GWK5PU5FQNQLEHTGKZYAYJP","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7a239e769ba71ff6bb5d2a6aeaa66e49244376322cbf65e188165297b5482231","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2018-06-04T16:10:19Z","title_canon_sha256":"91bc1f44b2315a3817514076502d9e3c17c239d03e18eb75bd974f04cf216a69"},"schema_version":"1.0","source":{"id":"1806.01170","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1806.01170","created_at":"2026-05-18T00:14:17Z"},{"alias_kind":"arxiv_version","alias_value":"1806.01170v1","created_at":"2026-05-18T00:14:17Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1806.01170","created_at":"2026-05-18T00:14:17Z"},{"alias_kind":"pith_short_12","alias_value":"6S6GWK5PU5FQ","created_at":"2026-05-18T12:32:11Z"},{"alias_kind":"pith_short_16","alias_value":"6S6GWK5PU5FQNQLE","created_at":"2026-05-18T12:32:11Z"},{"alias_kind":"pith_short_8","alias_value":"6S6GWK5P","created_at":"2026-05-18T12:32:11Z"}],"graph_snapshots":[{"event_id":"sha256:d32a591d461af6ef8e6a949fe468941c79d944340912643423c9b5da8da80177","target":"graph","created_at":"2026-05-18T00:14:17Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"We describe a novel method for efficiently eliciting scalar annotations for dataset construction and system quality estimation by human judgments. We contrast direct assessment (annotators assign scores to items directly), online pairwise ranking aggregation (scores derive from annotator comparison of items), and a hybrid approach (EASL: Efficient Annotation of Scalar Labels) proposed here. Our proposal leads to increased correlation with ground truth, at far greater annotator efficiency, suggesting this strategy as an improved mechanism for dataset creation and manual system evaluation.","authors_text":"Benjamin Van Durme, Keisuke Sakaguchi","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2018-06-04T16:10:19Z","title":"Efficient Online Scalar Annotation with Bounded Support"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1806.01170","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:ab955e6f877831f52ef33da1fa93bc0ded154b3c5b65ec9d869db7049b83e848","target":"record","created_at":"2026-05-18T00:14:17Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7a239e769ba71ff6bb5d2a6aeaa66e49244376322cbf65e188165297b5482231","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2018-06-04T16:10:19Z","title_canon_sha256":"91bc1f44b2315a3817514076502d9e3c17c239d03e18eb75bd974f04cf216a69"},"schema_version":"1.0","source":{"id":"1806.01170","kind":"arxiv","version":1}},"canonical_sha256":"f4bc6b2bafa74b06c1643cccace0184bf29e7a36f9f4cd5e2301f0c963b80f27","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"f4bc6b2bafa74b06c1643cccace0184bf29e7a36f9f4cd5e2301f0c963b80f27","first_computed_at":"2026-05-18T00:14:17.237232Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-18T00:14:17.237232Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"0nsK+sHNwZErrfh85KYHSwzJAuAK7FUvsdSV9bzIfJFkAz/U8COWGEhrSnszp3XoDlx3xLJ6IJe/4+xJaAzKDA==","signature_status":"signed_v1","signed_at":"2026-05-18T00:14:17.237747Z","signed_message":"canonical_sha256_bytes"},"source_id":"1806.01170","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:ab955e6f877831f52ef33da1fa93bc0ded154b3c5b65ec9d869db7049b83e848","sha256:d32a591d461af6ef8e6a949fe468941c79d944340912643423c9b5da8da80177"],"state_sha256":"6fc895d119c3ad5202e5f5c40a910a32330699ee75b4a87093d6be1459802656"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Sh6OzJ522gwlN/QUxyJj6Hip6nTQTGrh1Bcf58xM+aaI1ELiIJ6MGhkaeR2V3ilJxGQDOLmWIvNxtVOdJnRKDA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-06-04T16:54:58.147667Z","bundle_sha256":"61dab4dedee0fa1be6d29b8e3d72fa8b4ca0f5d210815e0f51c86aa777e0070c"}}