{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:2EHOVKUPTAOOJMKJBKE7XQM4FW","short_pith_number":"pith:2EHOVKUP","canonical_record":{"source":{"id":"2211.00981","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2022-11-02T09:38:38Z","cross_cats_sorted":[],"title_canon_sha256":"0082004613e7bf04f71bcddec9bacca6e7a25e6dcfdcfd58e2a60a3f91db12f0","abstract_canon_sha256":"ccd0b4746e3a0587dde6d4f87ece39e433e49ef9e37219c94a52e2e434ac3950"},"schema_version":"1.0"},"canonical_sha256":"d10eeaaa8f981ce4b1490a89fbc19c2d9394edb8213b57a31bd15016c92c133a","source":{"kind":"arxiv","id":"2211.00981","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2211.00981","created_at":"2026-07-05T05:12:37Z"},{"alias_kind":"arxiv_version","alias_value":"2211.00981v1","created_at":"2026-07-05T05:12:37Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2211.00981","created_at":"2026-07-05T05:12:37Z"},{"alias_kind":"pith_short_12","alias_value":"2EHOVKUPTAOO","created_at":"2026-07-05T05:12:37Z"},{"alias_kind":"pith_short_16","alias_value":"2EHOVKUPTAOOJMKJ","created_at":"2026-07-05T05:12:37Z"},{"alias_kind":"pith_short_8","alias_value":"2EHOVKUP","created_at":"2026-07-05T05:12:37Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:2EHOVKUPTAOOJMKJBKE7XQM4FW","target":"record","payload":{"canonical_record":{"source":{"id":"2211.00981","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2022-11-02T09:38:38Z","cross_cats_sorted":[],"title_canon_sha256":"0082004613e7bf04f71bcddec9bacca6e7a25e6dcfdcfd58e2a60a3f91db12f0","abstract_canon_sha256":"ccd0b4746e3a0587dde6d4f87ece39e433e49ef9e37219c94a52e2e434ac3950"},"schema_version":"1.0"},"canonical_sha256":"d10eeaaa8f981ce4b1490a89fbc19c2d9394edb8213b57a31bd15016c92c133a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:12:37.090820Z","signature_b64":"hRaVpUhsXObLzYKk31bULeqf0BKlqNtOQgJx0mwZeLVyZGz9NeaNlIO9vkgbTor6BG/TEG5bXh2585+xBbh/Ag==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d10eeaaa8f981ce4b1490a89fbc19c2d9394edb8213b57a31bd15016c92c133a","last_reissued_at":"2026-07-05T05:12:37.090428Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:12:37.090428Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2211.00981","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:12:37Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"MkGv3RQUwJrd+gtb9eB5SiqtDumwM4ijNpvO85FLuRTnLx7CR329IWRkA35JVVBiT73xvRpLum52AUz2TAMuCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T12:24:36.108242Z"},"content_sha256":"aab342dfabcca36c2be1d354ba805141f9f464e989e54779a9292036032bbf72","schema_version":"1.0","event_id":"sha256:aab342dfabcca36c2be1d354ba805141f9f464e989e54779a9292036032bbf72"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:2EHOVKUPTAOOJMKJBKE7XQM4FW","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Relevance Assessments for Web Search Evaluation: Should We Randomise or Prioritise the Pooled Documents? (CORRECTED VERSION)","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.IR","authors_text":"Sijie Tao, Tetsuya Sakai, Zhaohao Zeng","submitted_at":"2022-11-02T09:38:38Z","abstract_excerpt":"In the context of depth-$k$ pooling for constructing web search test collections, we compare two approaches to ordering pooled documents for relevance assessors: the prioritisation strategy (PRI) used widely at NTCIR, and the simple randomisation strategy (RND). In order to address research questions regarding PRI and RND, we have constructed and released the WWW3E8 data set, which contains eight independent relevance labels for 32,375 topic-document pairs, i.e., a total of 259,000 labels. Four of the eight relevance labels were obtained from PRI-based pools; the other four were obtained from "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2211.00981","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2211.00981/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:12:37Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Ykc1DVIaVGVe7IGLkKyb16J1+8Y1yXvp7gWFyDtWpw+OQjZVCpMAjL2wa0o2KOZT+asZMPJDdJK/CM3NWljNBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T12:24:36.108630Z"},"content_sha256":"c1e2e9cc314306c9a014ea686a751f168eacb5598463f88ebf48fc410b92a09d","schema_version":"1.0","event_id":"sha256:c1e2e9cc314306c9a014ea686a751f168eacb5598463f88ebf48fc410b92a09d"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/2EHOVKUPTAOOJMKJBKE7XQM4FW/bundle.json","state_url":"https://pith.science/pith/2EHOVKUPTAOOJMKJBKE7XQM4FW/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/2EHOVKUPTAOOJMKJBKE7XQM4FW/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T12:24:36Z","links":{"resolver":"https://pith.science/pith/2EHOVKUPTAOOJMKJBKE7XQM4FW","bundle":"https://pith.science/pith/2EHOVKUPTAOOJMKJBKE7XQM4FW/bundle.json","state":"https://pith.science/pith/2EHOVKUPTAOOJMKJBKE7XQM4FW/state.json","well_known_bundle":"https://pith.science/.well-known/pith/2EHOVKUPTAOOJMKJBKE7XQM4FW/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:2EHOVKUPTAOOJMKJBKE7XQM4FW","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"ccd0b4746e3a0587dde6d4f87ece39e433e49ef9e37219c94a52e2e434ac3950","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2022-11-02T09:38:38Z","title_canon_sha256":"0082004613e7bf04f71bcddec9bacca6e7a25e6dcfdcfd58e2a60a3f91db12f0"},"schema_version":"1.0","source":{"id":"2211.00981","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2211.00981","created_at":"2026-07-05T05:12:37Z"},{"alias_kind":"arxiv_version","alias_value":"2211.00981v1","created_at":"2026-07-05T05:12:37Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2211.00981","created_at":"2026-07-05T05:12:37Z"},{"alias_kind":"pith_short_12","alias_value":"2EHOVKUPTAOO","created_at":"2026-07-05T05:12:37Z"},{"alias_kind":"pith_short_16","alias_value":"2EHOVKUPTAOOJMKJ","created_at":"2026-07-05T05:12:37Z"},{"alias_kind":"pith_short_8","alias_value":"2EHOVKUP","created_at":"2026-07-05T05:12:37Z"}],"graph_snapshots":[{"event_id":"sha256:c1e2e9cc314306c9a014ea686a751f168eacb5598463f88ebf48fc410b92a09d","target":"graph","created_at":"2026-07-05T05:12:37Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2211.00981/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In the context of depth-$k$ pooling for constructing web search test collections, we compare two approaches to ordering pooled documents for relevance assessors: the prioritisation strategy (PRI) used widely at NTCIR, and the simple randomisation strategy (RND). In order to address research questions regarding PRI and RND, we have constructed and released the WWW3E8 data set, which contains eight independent relevance labels for 32,375 topic-document pairs, i.e., a total of 259,000 labels. Four of the eight relevance labels were obtained from PRI-based pools; the other four were obtained from ","authors_text":"Sijie Tao, Tetsuya Sakai, Zhaohao Zeng","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2022-11-02T09:38:38Z","title":"Relevance Assessments for Web Search Evaluation: Should We Randomise or Prioritise the Pooled Documents? (CORRECTED VERSION)"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2211.00981","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:aab342dfabcca36c2be1d354ba805141f9f464e989e54779a9292036032bbf72","target":"record","created_at":"2026-07-05T05:12:37Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"ccd0b4746e3a0587dde6d4f87ece39e433e49ef9e37219c94a52e2e434ac3950","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2022-11-02T09:38:38Z","title_canon_sha256":"0082004613e7bf04f71bcddec9bacca6e7a25e6dcfdcfd58e2a60a3f91db12f0"},"schema_version":"1.0","source":{"id":"2211.00981","kind":"arxiv","version":1}},"canonical_sha256":"d10eeaaa8f981ce4b1490a89fbc19c2d9394edb8213b57a31bd15016c92c133a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"d10eeaaa8f981ce4b1490a89fbc19c2d9394edb8213b57a31bd15016c92c133a","first_computed_at":"2026-07-05T05:12:37.090428Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:12:37.090428Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"hRaVpUhsXObLzYKk31bULeqf0BKlqNtOQgJx0mwZeLVyZGz9NeaNlIO9vkgbTor6BG/TEG5bXh2585+xBbh/Ag==","signature_status":"signed_v1","signed_at":"2026-07-05T05:12:37.090820Z","signed_message":"canonical_sha256_bytes"},"source_id":"2211.00981","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:aab342dfabcca36c2be1d354ba805141f9f464e989e54779a9292036032bbf72","sha256:c1e2e9cc314306c9a014ea686a751f168eacb5598463f88ebf48fc410b92a09d"],"state_sha256":"e99d3875f42141b535dfcace3b5d0a6108bb2767a77fb61c6790faf7db269011"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"13jepD7V9U0/F2payzd82LETo7h/MoKRTRsjATL8lD5IowEW9tyTLswVBHNgRe83AVVoeP6nT8HO2CDQZ11vCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T12:24:36.110504Z","bundle_sha256":"1944f9643fb0c414c561732afd0df5fa47819635d73fe90a2cc908ffc26a08d0"}}