{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2016:KEAULD4QFU23CDJNBRND5ZT66Y","short_pith_number":"pith:KEAULD4Q","canonical_record":{"source":{"id":"1604.07209","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2016-04-25T11:28:21Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"2a22273653837097b40416718a73c2ccc16539fc4a674c620057ec5d89285768","abstract_canon_sha256":"806d424e77538d1c80591a9c907476a1b873d281d46fea2cf9b0689dd76b8f58"},"schema_version":"1.0"},"canonical_sha256":"5101458f902d35b10d2d0c5a3ee67ef620843105cb1ec4b42b3fe4885edcce4d","source":{"kind":"arxiv","id":"1604.07209","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1604.07209","created_at":"2026-05-18T01:16:22Z"},{"alias_kind":"arxiv_version","alias_value":"1604.07209v1","created_at":"2026-05-18T01:16:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1604.07209","created_at":"2026-05-18T01:16:22Z"},{"alias_kind":"pith_short_12","alias_value":"KEAULD4QFU23","created_at":"2026-05-18T12:30:25Z"},{"alias_kind":"pith_short_16","alias_value":"KEAULD4QFU23CDJN","created_at":"2026-05-18T12:30:25Z"},{"alias_kind":"pith_short_8","alias_value":"KEAULD4Q","created_at":"2026-05-18T12:30:25Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2016:KEAULD4QFU23CDJNBRND5ZT66Y","target":"record","payload":{"canonical_record":{"source":{"id":"1604.07209","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2016-04-25T11:28:21Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"2a22273653837097b40416718a73c2ccc16539fc4a674c620057ec5d89285768","abstract_canon_sha256":"806d424e77538d1c80591a9c907476a1b873d281d46fea2cf9b0689dd76b8f58"},"schema_version":"1.0"},"canonical_sha256":"5101458f902d35b10d2d0c5a3ee67ef620843105cb1ec4b42b3fe4885edcce4d","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T01:16:22.760733Z","signature_b64":"LTbmoY1L7gM8OWhIxerMvuebNRPp7kaO/jK8do/BSc3axZT/qLXPv21zExJZ9jdApgsFCduWhRjU2YKb/4TfBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"5101458f902d35b10d2d0c5a3ee67ef620843105cb1ec4b42b3fe4885edcce4d","last_reissued_at":"2026-05-18T01:16:22.759979Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T01:16:22.759979Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1604.07209","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T01:16:22Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Z+enRvHaJ3+AU5sueLdwwotfXa+uyBvBaDIUpqydnW69d0WNkvFW7/wVpA5Bxe45JcN5H/u1lbaUVLJoDI2zCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-07T02:45:14.569525Z"},"content_sha256":"704b147659c2c9b9eff934a14dcb706acc86b847e24c17f05d79273ab269c6ab","schema_version":"1.0","event_id":"sha256:704b147659c2c9b9eff934a14dcb706acc86b847e24c17f05d79273ab269c6ab"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2016:KEAULD4QFU23CDJNBRND5ZT66Y","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Unbiased Comparative Evaluation of Ranking Functions","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.IR","authors_text":"Adith Swaminathan, Peter Frazier, Thorsten Joachims, Tobias Schnabel","submitted_at":"2016-04-25T11:28:21Z","abstract_excerpt":"Eliciting relevance judgments for ranking evaluation is labor-intensive and costly, motivating careful selection of which documents to judge. Unlike traditional approaches that make this selection deterministically, probabilistic sampling has shown intriguing promise since it enables the design of estimators that are provably unbiased even when reusing data with missing judgments. In this paper, we first unify and extend these sampling approaches by viewing the evaluation problem as a Monte Carlo estimation task that applies to a large number of common IR metrics. Drawing on the theoretical cl"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1604.07209","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T01:16:22Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Y88W5LxcHHqo3jNrt5UOycrJSw0dSVcjTUWr9iIuRZ/9rAUBMRA8Pvv4dWs3YW5t8y7acjFvbf8UFQsrWgrwBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-07T02:45:14.570228Z"},"content_sha256":"958f0300d3d75591b8678c668da4b2181a7ad437dd063e8f8c0c20c7adefcfbb","schema_version":"1.0","event_id":"sha256:958f0300d3d75591b8678c668da4b2181a7ad437dd063e8f8c0c20c7adefcfbb"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/KEAULD4QFU23CDJNBRND5ZT66Y/bundle.json","state_url":"https://pith.science/pith/KEAULD4QFU23CDJNBRND5ZT66Y/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/KEAULD4QFU23CDJNBRND5ZT66Y/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-06-07T02:45:14Z","links":{"resolver":"https://pith.science/pith/KEAULD4QFU23CDJNBRND5ZT66Y","bundle":"https://pith.science/pith/KEAULD4QFU23CDJNBRND5ZT66Y/bundle.json","state":"https://pith.science/pith/KEAULD4QFU23CDJNBRND5ZT66Y/state.json","well_known_bundle":"https://pith.science/.well-known/pith/KEAULD4QFU23CDJNBRND5ZT66Y/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2016:KEAULD4QFU23CDJNBRND5ZT66Y","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"806d424e77538d1c80591a9c907476a1b873d281d46fea2cf9b0689dd76b8f58","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2016-04-25T11:28:21Z","title_canon_sha256":"2a22273653837097b40416718a73c2ccc16539fc4a674c620057ec5d89285768"},"schema_version":"1.0","source":{"id":"1604.07209","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1604.07209","created_at":"2026-05-18T01:16:22Z"},{"alias_kind":"arxiv_version","alias_value":"1604.07209v1","created_at":"2026-05-18T01:16:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1604.07209","created_at":"2026-05-18T01:16:22Z"},{"alias_kind":"pith_short_12","alias_value":"KEAULD4QFU23","created_at":"2026-05-18T12:30:25Z"},{"alias_kind":"pith_short_16","alias_value":"KEAULD4QFU23CDJN","created_at":"2026-05-18T12:30:25Z"},{"alias_kind":"pith_short_8","alias_value":"KEAULD4Q","created_at":"2026-05-18T12:30:25Z"}],"graph_snapshots":[{"event_id":"sha256:958f0300d3d75591b8678c668da4b2181a7ad437dd063e8f8c0c20c7adefcfbb","target":"graph","created_at":"2026-05-18T01:16:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"Eliciting relevance judgments for ranking evaluation is labor-intensive and costly, motivating careful selection of which documents to judge. Unlike traditional approaches that make this selection deterministically, probabilistic sampling has shown intriguing promise since it enables the design of estimators that are provably unbiased even when reusing data with missing judgments. In this paper, we first unify and extend these sampling approaches by viewing the evaluation problem as a Monte Carlo estimation task that applies to a large number of common IR metrics. Drawing on the theoretical cl","authors_text":"Adith Swaminathan, Peter Frazier, Thorsten Joachims, Tobias Schnabel","cross_cats":["cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2016-04-25T11:28:21Z","title":"Unbiased Comparative Evaluation of Ranking Functions"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1604.07209","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:704b147659c2c9b9eff934a14dcb706acc86b847e24c17f05d79273ab269c6ab","target":"record","created_at":"2026-05-18T01:16:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"806d424e77538d1c80591a9c907476a1b873d281d46fea2cf9b0689dd76b8f58","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2016-04-25T11:28:21Z","title_canon_sha256":"2a22273653837097b40416718a73c2ccc16539fc4a674c620057ec5d89285768"},"schema_version":"1.0","source":{"id":"1604.07209","kind":"arxiv","version":1}},"canonical_sha256":"5101458f902d35b10d2d0c5a3ee67ef620843105cb1ec4b42b3fe4885edcce4d","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"5101458f902d35b10d2d0c5a3ee67ef620843105cb1ec4b42b3fe4885edcce4d","first_computed_at":"2026-05-18T01:16:22.759979Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-18T01:16:22.759979Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"LTbmoY1L7gM8OWhIxerMvuebNRPp7kaO/jK8do/BSc3axZT/qLXPv21zExJZ9jdApgsFCduWhRjU2YKb/4TfBw==","signature_status":"signed_v1","signed_at":"2026-05-18T01:16:22.760733Z","signed_message":"canonical_sha256_bytes"},"source_id":"1604.07209","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:704b147659c2c9b9eff934a14dcb706acc86b847e24c17f05d79273ab269c6ab","sha256:958f0300d3d75591b8678c668da4b2181a7ad437dd063e8f8c0c20c7adefcfbb"],"state_sha256":"fb6be2b8b616aa6abcd6c3f725ef7e87c1e209b40f5f83cbe3fa4f75bbc5d96f"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Dw89rVgCnZ+1UdLzV45gNNTRbaOgaFHEgbuUiKwICNTkvzyYBXKt9bD+NvC9O2J7gZeUC1naiXaS8qTGOOXKBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-06-07T02:45:14.573775Z","bundle_sha256":"c58c9bfd9685d2e5a025b581600ca5870ab53da31594bd868912bfd0e8350b37"}}