{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:AO76HISHDTITHZQOZOZVTCQUOP","short_pith_number":"pith:AO76HISH","canonical_record":{"source":{"id":"2405.05600","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2024-05-09T07:39:19Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"62316c9daf4568d624893f148a0b32641adb045ca23a72581fa7cdfd460ae1b3","abstract_canon_sha256":"5bb64748e8afa0d8ac67693ae6d98418935d533631dbf3925f32aca8779cdb4f"},"schema_version":"1.0"},"canonical_sha256":"03bfe3a2471cd133e60ecbb3598a1473d12116ea184a2bfaec4c944500bb9e46","source":{"kind":"arxiv","id":"2405.05600","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.05600","created_at":"2026-07-05T08:17:20Z"},{"alias_kind":"arxiv_version","alias_value":"2405.05600v1","created_at":"2026-07-05T08:17:20Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.05600","created_at":"2026-07-05T08:17:20Z"},{"alias_kind":"pith_short_12","alias_value":"AO76HISHDTIT","created_at":"2026-07-05T08:17:20Z"},{"alias_kind":"pith_short_16","alias_value":"AO76HISHDTITHZQO","created_at":"2026-07-05T08:17:20Z"},{"alias_kind":"pith_short_8","alias_value":"AO76HISH","created_at":"2026-07-05T08:17:20Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:AO76HISHDTITHZQOZOZVTCQUOP","target":"record","payload":{"canonical_record":{"source":{"id":"2405.05600","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2024-05-09T07:39:19Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"62316c9daf4568d624893f148a0b32641adb045ca23a72581fa7cdfd460ae1b3","abstract_canon_sha256":"5bb64748e8afa0d8ac67693ae6d98418935d533631dbf3925f32aca8779cdb4f"},"schema_version":"1.0"},"canonical_sha256":"03bfe3a2471cd133e60ecbb3598a1473d12116ea184a2bfaec4c944500bb9e46","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:17:20.407441Z","signature_b64":"kCyJf6Ea9PaE5Nwej2zFFYpVoeDZpDhfL+hEDatYfCwVz4yhTDeAONJxFOJU4IJwwt0kSCcczG83+003psMGCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"03bfe3a2471cd133e60ecbb3598a1473d12116ea184a2bfaec4c944500bb9e46","last_reissued_at":"2026-07-05T08:17:20.407023Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:17:20.407023Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2405.05600","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:17:20Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ypU1BvvDuutJ3eiATNUPMf/g/AJxrF4TrzxMw8yvl3RoioZHBEpsk5eMy8054T6OIhs36AtnDc5e42LD74P6DQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T09:47:42.929303Z"},"content_sha256":"c8381712711270e2192d992b9e60589439699ae3760037a58ad174b6c3b6007f","schema_version":"1.0","event_id":"sha256:c8381712711270e2192d992b9e60589439699ae3760037a58ad174b6c3b6007f"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:AO76HISHDTITHZQOZOZVTCQUOP","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Can We Use Large Language Models to Fill Relevance Judgment Holes?","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.IR","authors_text":"Chuan Meng, Leif Azzopardi, Mohammad Aliannejadi, Zahra Abbasiantaeb","submitted_at":"2024-05-09T07:39:19Z","abstract_excerpt":"Incomplete relevance judgments limit the re-usability of test collections. When new systems are compared against previous systems used to build the pool of judged documents, they often do so at a disadvantage due to the ``holes'' in test collection (i.e., pockets of un-assessed documents returned by the new system). In this paper, we take initial steps towards extending existing test collections by employing Large Language Models (LLM) to fill the holes by leveraging and grounding the method using existing human judgments. We explore this problem in the context of Conversational Search using T"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.05600","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.05600/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:17:20Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"abOrBhLCBWTvCSGguVB/9QVzHDrHZP6on1HGhGfQl+G1cKabkil21+ThQ3GcHsTF0bv25yCYsDXGb5FFV2pSBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T09:47:42.930202Z"},"content_sha256":"1c653dbee1d76c62d1171b852241b95b4a6d049ab5df11a85f9ad9cfe2a35684","schema_version":"1.0","event_id":"sha256:1c653dbee1d76c62d1171b852241b95b4a6d049ab5df11a85f9ad9cfe2a35684"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/AO76HISHDTITHZQOZOZVTCQUOP/bundle.json","state_url":"https://pith.science/pith/AO76HISHDTITHZQOZOZVTCQUOP/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/AO76HISHDTITHZQOZOZVTCQUOP/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T09:47:42Z","links":{"resolver":"https://pith.science/pith/AO76HISHDTITHZQOZOZVTCQUOP","bundle":"https://pith.science/pith/AO76HISHDTITHZQOZOZVTCQUOP/bundle.json","state":"https://pith.science/pith/AO76HISHDTITHZQOZOZVTCQUOP/state.json","well_known_bundle":"https://pith.science/.well-known/pith/AO76HISHDTITHZQOZOZVTCQUOP/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:AO76HISHDTITHZQOZOZVTCQUOP","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"5bb64748e8afa0d8ac67693ae6d98418935d533631dbf3925f32aca8779cdb4f","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2024-05-09T07:39:19Z","title_canon_sha256":"62316c9daf4568d624893f148a0b32641adb045ca23a72581fa7cdfd460ae1b3"},"schema_version":"1.0","source":{"id":"2405.05600","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.05600","created_at":"2026-07-05T08:17:20Z"},{"alias_kind":"arxiv_version","alias_value":"2405.05600v1","created_at":"2026-07-05T08:17:20Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.05600","created_at":"2026-07-05T08:17:20Z"},{"alias_kind":"pith_short_12","alias_value":"AO76HISHDTIT","created_at":"2026-07-05T08:17:20Z"},{"alias_kind":"pith_short_16","alias_value":"AO76HISHDTITHZQO","created_at":"2026-07-05T08:17:20Z"},{"alias_kind":"pith_short_8","alias_value":"AO76HISH","created_at":"2026-07-05T08:17:20Z"}],"graph_snapshots":[{"event_id":"sha256:1c653dbee1d76c62d1171b852241b95b4a6d049ab5df11a85f9ad9cfe2a35684","target":"graph","created_at":"2026-07-05T08:17:20Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.05600/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Incomplete relevance judgments limit the re-usability of test collections. When new systems are compared against previous systems used to build the pool of judged documents, they often do so at a disadvantage due to the ``holes'' in test collection (i.e., pockets of un-assessed documents returned by the new system). In this paper, we take initial steps towards extending existing test collections by employing Large Language Models (LLM) to fill the holes by leveraging and grounding the method using existing human judgments. We explore this problem in the context of Conversational Search using T","authors_text":"Chuan Meng, Leif Azzopardi, Mohammad Aliannejadi, Zahra Abbasiantaeb","cross_cats":["cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2024-05-09T07:39:19Z","title":"Can We Use Large Language Models to Fill Relevance Judgment Holes?"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.05600","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:c8381712711270e2192d992b9e60589439699ae3760037a58ad174b6c3b6007f","target":"record","created_at":"2026-07-05T08:17:20Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"5bb64748e8afa0d8ac67693ae6d98418935d533631dbf3925f32aca8779cdb4f","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2024-05-09T07:39:19Z","title_canon_sha256":"62316c9daf4568d624893f148a0b32641adb045ca23a72581fa7cdfd460ae1b3"},"schema_version":"1.0","source":{"id":"2405.05600","kind":"arxiv","version":1}},"canonical_sha256":"03bfe3a2471cd133e60ecbb3598a1473d12116ea184a2bfaec4c944500bb9e46","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"03bfe3a2471cd133e60ecbb3598a1473d12116ea184a2bfaec4c944500bb9e46","first_computed_at":"2026-07-05T08:17:20.407023Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:17:20.407023Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"kCyJf6Ea9PaE5Nwej2zFFYpVoeDZpDhfL+hEDatYfCwVz4yhTDeAONJxFOJU4IJwwt0kSCcczG83+003psMGCg==","signature_status":"signed_v1","signed_at":"2026-07-05T08:17:20.407441Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.05600","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:c8381712711270e2192d992b9e60589439699ae3760037a58ad174b6c3b6007f","sha256:1c653dbee1d76c62d1171b852241b95b4a6d049ab5df11a85f9ad9cfe2a35684"],"state_sha256":"999bf29eddaf15176259ca9fa92d71022027881565c2a95403b186711c20c4b4"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"gFxe2MuzaSETT4/1LzODu0DLo1pMaCrEVog4eTlOhOsu/YYOntQa5cARqpGWJm/DCm1ndFLJ9VorV8RwqYcFCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T09:47:42.935757Z","bundle_sha256":"8807c18e8a9a955dc4e55510378e1296f37065af3b4c144f26c442a252447308"}}