{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2015:NRWPKVIGO4OJUIGIRRDRDQAPZ7","short_pith_number":"pith:NRWPKVIG","canonical_record":{"source":{"id":"1511.07237","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2015-11-23T14:31:43Z","cross_cats_sorted":[],"title_canon_sha256":"995242d0fbe75dfe67b6d766d2d37b304b59aa2413eb5d9fae5e065c247c5ce1","abstract_canon_sha256":"3b944dd4fc2db172cd985d1b3098a98aea1112125b985e4dbb484bc7860a3f00"},"schema_version":"1.0"},"canonical_sha256":"6c6cf55506771c9a20c88c4711c00fcffa22c783e62018254c448fd0c24dd8b0","source":{"kind":"arxiv","id":"1511.07237","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1511.07237","created_at":"2026-05-18T01:26:11Z"},{"alias_kind":"arxiv_version","alias_value":"1511.07237v1","created_at":"2026-05-18T01:26:11Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1511.07237","created_at":"2026-05-18T01:26:11Z"},{"alias_kind":"pith_short_12","alias_value":"NRWPKVIGO4OJ","created_at":"2026-05-18T12:29:34Z"},{"alias_kind":"pith_short_16","alias_value":"NRWPKVIGO4OJUIGI","created_at":"2026-05-18T12:29:34Z"},{"alias_kind":"pith_short_8","alias_value":"NRWPKVIG","created_at":"2026-05-18T12:29:34Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2015:NRWPKVIGO4OJUIGIRRDRDQAPZ7","target":"record","payload":{"canonical_record":{"source":{"id":"1511.07237","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2015-11-23T14:31:43Z","cross_cats_sorted":[],"title_canon_sha256":"995242d0fbe75dfe67b6d766d2d37b304b59aa2413eb5d9fae5e065c247c5ce1","abstract_canon_sha256":"3b944dd4fc2db172cd985d1b3098a98aea1112125b985e4dbb484bc7860a3f00"},"schema_version":"1.0"},"canonical_sha256":"6c6cf55506771c9a20c88c4711c00fcffa22c783e62018254c448fd0c24dd8b0","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T01:26:11.878081Z","signature_b64":"DYr+WT5xFWCSEptmT0RzI1ie7P0mTsdmF1rbjl+vQsaBSIJU5/vA6HDx+ak1rhQ3mPhxBzwGzWz0J1DiasuhCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6c6cf55506771c9a20c88c4711c00fcffa22c783e62018254c448fd0c24dd8b0","last_reissued_at":"2026-05-18T01:26:11.877525Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T01:26:11.877525Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1511.07237","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T01:26:11Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"WjJX4F8euaxLlrnJaXWoABF1PqaEZAWeQrpUon1zt9vlLlZs2KRcjgB2nd9RNNd7x58Nr/2kqKymaoahFCh3Cg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-30T15:02:34.663739Z"},"content_sha256":"475af1bd344baf5e19025f151215d7e358c1cd13266d87e218f402ba14886f24","schema_version":"1.0","event_id":"sha256:475af1bd344baf5e19025f151215d7e358c1cd13266d87e218f402ba14886f24"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2015:NRWPKVIGO4OJUIGIRRDRDQAPZ7","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Predicting Relevance based on Assessor Disagreement: Analysis and Practical Applications for Search Evaluation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.IR","authors_text":"Chris Develder, Djoerd Hiemstra, Dong Nguyen, Robin Aly, Thomas Demeester","submitted_at":"2015-11-23T14:31:43Z","abstract_excerpt":"Evaluation of search engines relies on assessments of search results for selected test queries, from which we would ideally like to draw conclusions in terms of relevance of the results for general (e.g., future, unknown) users. In practice however, most evaluation scenarios only allow us to conclusively determine the relevance towards the particular assessor that provided the judgments. A factor that cannot be ignored when extending conclusions made from assessors towards users, is the possible disagreement on relevance, assuming that a single gold truth label does not exist. This paper prese"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1511.07237","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T01:26:11Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"oYqiQHYA835+0abmnD4aS1k+Ae250JN41/5DWETzztppR2eiEoqQW+IHxxl6Qs7w1udUg7CmuHVsBYBwAcS4DA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-30T15:02:34.664084Z"},"content_sha256":"550596c8f8f1334cdb6e8049970091d78034b5fd9857b09b3bf0b52c8e659695","schema_version":"1.0","event_id":"sha256:550596c8f8f1334cdb6e8049970091d78034b5fd9857b09b3bf0b52c8e659695"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/NRWPKVIGO4OJUIGIRRDRDQAPZ7/bundle.json","state_url":"https://pith.science/pith/NRWPKVIGO4OJUIGIRRDRDQAPZ7/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/NRWPKVIGO4OJUIGIRRDRDQAPZ7/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-05-30T15:02:34Z","links":{"resolver":"https://pith.science/pith/NRWPKVIGO4OJUIGIRRDRDQAPZ7","bundle":"https://pith.science/pith/NRWPKVIGO4OJUIGIRRDRDQAPZ7/bundle.json","state":"https://pith.science/pith/NRWPKVIGO4OJUIGIRRDRDQAPZ7/state.json","well_known_bundle":"https://pith.science/.well-known/pith/NRWPKVIGO4OJUIGIRRDRDQAPZ7/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2015:NRWPKVIGO4OJUIGIRRDRDQAPZ7","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"3b944dd4fc2db172cd985d1b3098a98aea1112125b985e4dbb484bc7860a3f00","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2015-11-23T14:31:43Z","title_canon_sha256":"995242d0fbe75dfe67b6d766d2d37b304b59aa2413eb5d9fae5e065c247c5ce1"},"schema_version":"1.0","source":{"id":"1511.07237","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1511.07237","created_at":"2026-05-18T01:26:11Z"},{"alias_kind":"arxiv_version","alias_value":"1511.07237v1","created_at":"2026-05-18T01:26:11Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1511.07237","created_at":"2026-05-18T01:26:11Z"},{"alias_kind":"pith_short_12","alias_value":"NRWPKVIGO4OJ","created_at":"2026-05-18T12:29:34Z"},{"alias_kind":"pith_short_16","alias_value":"NRWPKVIGO4OJUIGI","created_at":"2026-05-18T12:29:34Z"},{"alias_kind":"pith_short_8","alias_value":"NRWPKVIG","created_at":"2026-05-18T12:29:34Z"}],"graph_snapshots":[{"event_id":"sha256:550596c8f8f1334cdb6e8049970091d78034b5fd9857b09b3bf0b52c8e659695","target":"graph","created_at":"2026-05-18T01:26:11Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"Evaluation of search engines relies on assessments of search results for selected test queries, from which we would ideally like to draw conclusions in terms of relevance of the results for general (e.g., future, unknown) users. In practice however, most evaluation scenarios only allow us to conclusively determine the relevance towards the particular assessor that provided the judgments. A factor that cannot be ignored when extending conclusions made from assessors towards users, is the possible disagreement on relevance, assuming that a single gold truth label does not exist. This paper prese","authors_text":"Chris Develder, Djoerd Hiemstra, Dong Nguyen, Robin Aly, Thomas Demeester","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2015-11-23T14:31:43Z","title":"Predicting Relevance based on Assessor Disagreement: Analysis and Practical Applications for Search Evaluation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1511.07237","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:475af1bd344baf5e19025f151215d7e358c1cd13266d87e218f402ba14886f24","target":"record","created_at":"2026-05-18T01:26:11Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"3b944dd4fc2db172cd985d1b3098a98aea1112125b985e4dbb484bc7860a3f00","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2015-11-23T14:31:43Z","title_canon_sha256":"995242d0fbe75dfe67b6d766d2d37b304b59aa2413eb5d9fae5e065c247c5ce1"},"schema_version":"1.0","source":{"id":"1511.07237","kind":"arxiv","version":1}},"canonical_sha256":"6c6cf55506771c9a20c88c4711c00fcffa22c783e62018254c448fd0c24dd8b0","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"6c6cf55506771c9a20c88c4711c00fcffa22c783e62018254c448fd0c24dd8b0","first_computed_at":"2026-05-18T01:26:11.877525Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-18T01:26:11.877525Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"DYr+WT5xFWCSEptmT0RzI1ie7P0mTsdmF1rbjl+vQsaBSIJU5/vA6HDx+ak1rhQ3mPhxBzwGzWz0J1DiasuhCw==","signature_status":"signed_v1","signed_at":"2026-05-18T01:26:11.878081Z","signed_message":"canonical_sha256_bytes"},"source_id":"1511.07237","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:475af1bd344baf5e19025f151215d7e358c1cd13266d87e218f402ba14886f24","sha256:550596c8f8f1334cdb6e8049970091d78034b5fd9857b09b3bf0b52c8e659695"],"state_sha256":"1245128acd5455939f3c1acda0289bc7f5ae701fbbab6ce837c47c03e3d94c42"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"CISV60p317WW+rtf6uc+QRW90uPy/myde/u0RRSj8iXi1a8t1U2zYWWYB7Ofo9DWl1GTfxKaWScfdlHdXuh7Ag==","signed_message":"bundle_sha256_bytes","signed_at":"2026-05-30T15:02:34.666139Z","bundle_sha256":"81f4d669280138301ac81122c732228cdeaef379343add9b53877988215f37fc"}}