{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2012:HA6UWNQKOIJKSBSD675UYNBYO3","short_pith_number":"pith:HA6UWNQK","canonical_record":{"source":{"id":"1212.0960","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2012-12-05T08:15:36Z","cross_cats_sorted":["cs.IR","stat.ML"],"title_canon_sha256":"231671e6056569dafb0fc7bcca7c0ca7e2c250644095c93f70674d4b4f588c32","abstract_canon_sha256":"c88663130f251db0c7a14bd0c81b0bbdf80255353114cf198258e5db0563fb51"},"schema_version":"1.0"},"canonical_sha256":"383d4b360a7212a90643f7fb4c343876c56c39df533fef8f6145c2f5b509e68f","source":{"kind":"arxiv","id":"1212.0960","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1212.0960","created_at":"2026-05-18T03:39:06Z"},{"alias_kind":"arxiv_version","alias_value":"1212.0960v1","created_at":"2026-05-18T03:39:06Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1212.0960","created_at":"2026-05-18T03:39:06Z"},{"alias_kind":"pith_short_12","alias_value":"HA6UWNQKOIJK","created_at":"2026-05-18T12:27:09Z"},{"alias_kind":"pith_short_16","alias_value":"HA6UWNQKOIJKSBSD","created_at":"2026-05-18T12:27:09Z"},{"alias_kind":"pith_short_8","alias_value":"HA6UWNQK","created_at":"2026-05-18T12:27:09Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2012:HA6UWNQKOIJKSBSD675UYNBYO3","target":"record","payload":{"canonical_record":{"source":{"id":"1212.0960","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2012-12-05T08:15:36Z","cross_cats_sorted":["cs.IR","stat.ML"],"title_canon_sha256":"231671e6056569dafb0fc7bcca7c0ca7e2c250644095c93f70674d4b4f588c32","abstract_canon_sha256":"c88663130f251db0c7a14bd0c81b0bbdf80255353114cf198258e5db0563fb51"},"schema_version":"1.0"},"canonical_sha256":"383d4b360a7212a90643f7fb4c343876c56c39df533fef8f6145c2f5b509e68f","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T03:39:06.838155Z","signature_b64":"QpWsuQPFokbbqpaEcl96u9124Q3Fk9z8PBQ981qhFQTN52Y9bY8owbW9ueMTNEryyZOpCn+EuGmxkWvvj5o0Cw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"383d4b360a7212a90643f7fb4c343876c56c39df533fef8f6145c2f5b509e68f","last_reissued_at":"2026-05-18T03:39:06.837478Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T03:39:06.837478Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1212.0960","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T03:39:06Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"PmVywbiafRyeKonyW1sywyvVfvAs3cNH/eIDJHcsPgP1SGRzrue/c0b6vgG9bVU/UOxrJMYOQVNrrl6hzFH8Dw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-03T18:39:58.156662Z"},"content_sha256":"47af3f40faf6c68ac8f5522b7940e5604e0e795a9ad6492dbbb598ac81818eda","schema_version":"1.0","event_id":"sha256:47af3f40faf6c68ac8f5522b7940e5604e0e795a9ad6492dbbb598ac81818eda"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2012:HA6UWNQKOIJKSBSD675UYNBYO3","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Evaluating Classifiers Without Expert Labels","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.IR","stat.ML"],"primary_cat":"cs.LG","authors_text":"Hyun Joon Jung, Matthew Lease","submitted_at":"2012-12-05T08:15:36Z","abstract_excerpt":"This paper considers the challenge of evaluating a set of classifiers, as done in shared task evaluations like the KDD Cup or NIST TREC, without expert labels. While expert labels provide the traditional cornerstone for evaluating statistical learners, limited or expensive access to experts represents a practical bottleneck. Instead, we seek methodology for estimating performance of the classifiers which is more scalable than expert labeling yet preserves high correlation with evaluation based on expert labels. We consider both: 1) using only labels automatically generated by the classifiers ("},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1212.0960","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T03:39:06Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"FI4FW+ak3FCunNtCArdjdhw3fhEf7H2sbiimtNBpjgkSl+lfXL0an9WE/yOObp8Qob3pw1EbnjEeuEwusWGDBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-03T18:39:58.157010Z"},"content_sha256":"331fd6b104c0979ba3865e4d7b91fcc1b7d2fda36e35a341f5bf13f345df5b01","schema_version":"1.0","event_id":"sha256:331fd6b104c0979ba3865e4d7b91fcc1b7d2fda36e35a341f5bf13f345df5b01"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/HA6UWNQKOIJKSBSD675UYNBYO3/bundle.json","state_url":"https://pith.science/pith/HA6UWNQKOIJKSBSD675UYNBYO3/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/HA6UWNQKOIJKSBSD675UYNBYO3/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-06-03T18:39:58Z","links":{"resolver":"https://pith.science/pith/HA6UWNQKOIJKSBSD675UYNBYO3","bundle":"https://pith.science/pith/HA6UWNQKOIJKSBSD675UYNBYO3/bundle.json","state":"https://pith.science/pith/HA6UWNQKOIJKSBSD675UYNBYO3/state.json","well_known_bundle":"https://pith.science/.well-known/pith/HA6UWNQKOIJKSBSD675UYNBYO3/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2012:HA6UWNQKOIJKSBSD675UYNBYO3","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c88663130f251db0c7a14bd0c81b0bbdf80255353114cf198258e5db0563fb51","cross_cats_sorted":["cs.IR","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2012-12-05T08:15:36Z","title_canon_sha256":"231671e6056569dafb0fc7bcca7c0ca7e2c250644095c93f70674d4b4f588c32"},"schema_version":"1.0","source":{"id":"1212.0960","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1212.0960","created_at":"2026-05-18T03:39:06Z"},{"alias_kind":"arxiv_version","alias_value":"1212.0960v1","created_at":"2026-05-18T03:39:06Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1212.0960","created_at":"2026-05-18T03:39:06Z"},{"alias_kind":"pith_short_12","alias_value":"HA6UWNQKOIJK","created_at":"2026-05-18T12:27:09Z"},{"alias_kind":"pith_short_16","alias_value":"HA6UWNQKOIJKSBSD","created_at":"2026-05-18T12:27:09Z"},{"alias_kind":"pith_short_8","alias_value":"HA6UWNQK","created_at":"2026-05-18T12:27:09Z"}],"graph_snapshots":[{"event_id":"sha256:331fd6b104c0979ba3865e4d7b91fcc1b7d2fda36e35a341f5bf13f345df5b01","target":"graph","created_at":"2026-05-18T03:39:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"This paper considers the challenge of evaluating a set of classifiers, as done in shared task evaluations like the KDD Cup or NIST TREC, without expert labels. While expert labels provide the traditional cornerstone for evaluating statistical learners, limited or expensive access to experts represents a practical bottleneck. Instead, we seek methodology for estimating performance of the classifiers which is more scalable than expert labeling yet preserves high correlation with evaluation based on expert labels. We consider both: 1) using only labels automatically generated by the classifiers (","authors_text":"Hyun Joon Jung, Matthew Lease","cross_cats":["cs.IR","stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2012-12-05T08:15:36Z","title":"Evaluating Classifiers Without Expert Labels"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1212.0960","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:47af3f40faf6c68ac8f5522b7940e5604e0e795a9ad6492dbbb598ac81818eda","target":"record","created_at":"2026-05-18T03:39:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c88663130f251db0c7a14bd0c81b0bbdf80255353114cf198258e5db0563fb51","cross_cats_sorted":["cs.IR","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2012-12-05T08:15:36Z","title_canon_sha256":"231671e6056569dafb0fc7bcca7c0ca7e2c250644095c93f70674d4b4f588c32"},"schema_version":"1.0","source":{"id":"1212.0960","kind":"arxiv","version":1}},"canonical_sha256":"383d4b360a7212a90643f7fb4c343876c56c39df533fef8f6145c2f5b509e68f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"383d4b360a7212a90643f7fb4c343876c56c39df533fef8f6145c2f5b509e68f","first_computed_at":"2026-05-18T03:39:06.837478Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-18T03:39:06.837478Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"QpWsuQPFokbbqpaEcl96u9124Q3Fk9z8PBQ981qhFQTN52Y9bY8owbW9ueMTNEryyZOpCn+EuGmxkWvvj5o0Cw==","signature_status":"signed_v1","signed_at":"2026-05-18T03:39:06.838155Z","signed_message":"canonical_sha256_bytes"},"source_id":"1212.0960","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:47af3f40faf6c68ac8f5522b7940e5604e0e795a9ad6492dbbb598ac81818eda","sha256:331fd6b104c0979ba3865e4d7b91fcc1b7d2fda36e35a341f5bf13f345df5b01"],"state_sha256":"5e1d7bb1e9933266fae1b8efbafcabb90f15748feb2b0fd116898788443af772"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jMgDrgrMvx3Df2kN45EulYT621MCDtWh7vl8BYlPBxOG2WoY29BVJ1TPddBp0j9ZFsYBA8Jp2ql8mc1UIgccDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-06-03T18:39:58.158946Z","bundle_sha256":"cf774bb9e75626a4252f528fb31139d1caa7c5a521f8f522a94179ef633e5bbc"}}