{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:SWXFQ3FD4CO4KDQIFS2RIPKA6G","short_pith_number":"pith:SWXFQ3FD","canonical_record":{"source":{"id":"2502.20937","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2025-02-28T10:46:56Z","cross_cats_sorted":[],"title_canon_sha256":"3a3b194f58c019540314aab84d4e146596f5a33e73a1a31fbda98396ed904eec","abstract_canon_sha256":"88b512b1a8598e0d00a69aa4d41d6ac0dfe7517e62fe3879bff488f7c54e67c6"},"schema_version":"1.0"},"canonical_sha256":"95ae586ca3e09dc50e082cb5143d40f1a9e5c9fcc99a935f526c77ae490dce8a","source":{"kind":"arxiv","id":"2502.20937","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2502.20937","created_at":"2026-07-05T11:06:28Z"},{"alias_kind":"arxiv_version","alias_value":"2502.20937v2","created_at":"2026-07-05T11:06:28Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.20937","created_at":"2026-07-05T11:06:28Z"},{"alias_kind":"pith_short_12","alias_value":"SWXFQ3FD4CO4","created_at":"2026-07-05T11:06:28Z"},{"alias_kind":"pith_short_16","alias_value":"SWXFQ3FD4CO4KDQI","created_at":"2026-07-05T11:06:28Z"},{"alias_kind":"pith_short_8","alias_value":"SWXFQ3FD","created_at":"2026-07-05T11:06:28Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:SWXFQ3FD4CO4KDQIFS2RIPKA6G","target":"record","payload":{"canonical_record":{"source":{"id":"2502.20937","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2025-02-28T10:46:56Z","cross_cats_sorted":[],"title_canon_sha256":"3a3b194f58c019540314aab84d4e146596f5a33e73a1a31fbda98396ed904eec","abstract_canon_sha256":"88b512b1a8598e0d00a69aa4d41d6ac0dfe7517e62fe3879bff488f7c54e67c6"},"schema_version":"1.0"},"canonical_sha256":"95ae586ca3e09dc50e082cb5143d40f1a9e5c9fcc99a935f526c77ae490dce8a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:06:28.995892Z","signature_b64":"qZYxJbgofBdUychL5rNySvrdTHtzsRG6oX42kiRhVA+xq9FYX7MCSu7c75AZQbTHKoRY8LfG4SUO3H+Ag11fBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"95ae586ca3e09dc50e082cb5143d40f1a9e5c9fcc99a935f526c77ae490dce8a","last_reissued_at":"2026-07-05T11:06:28.995287Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:06:28.995287Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2502.20937","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:06:28Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QW69830v8x0kuZIH0IdWYw/F6yHLYC1DkhfaVOeSlP/9wC79YVCSAg7H5AsjkQ6wLMfug+gtDsNEvS9qUw38Cg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T14:52:58.928827Z"},"content_sha256":"b0609cbafbaf044fe779a652e23bf62a466fe068f67af2272d2b56f076d2f54d","schema_version":"1.0","event_id":"sha256:b0609cbafbaf044fe779a652e23bf62a466fe068f67af2272d2b56f076d2f54d"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:SWXFQ3FD4CO4KDQIFS2RIPKA6G","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Variations in Relevance Judgments and the Shelf Life of Test Collections","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.IR","authors_text":"Andrew Parry, Eugene Yang, Ferdinand Schlatt, Guglielmo Faggioli, Harrisen Scells, Maik Fr\\\"obe, Saber Zerhoudi, Sean MacAvaney","submitted_at":"2025-02-28T10:46:56Z","abstract_excerpt":"The fundamental property of Cranfield-style evaluations, that system rankings are stable even when assessors disagree on individual relevance decisions, was validated on traditional test collections. However, the paradigm shift towards neural retrieval models affected the characteristics of modern test collections, e.g., documents are short, judged with four grades of relevance, and information needs have no descriptions or narratives. Under these changes, it is unclear whether assessor disagreement remains negligible for system comparisons. We investigate this aspect under the additional cond"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.20937","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2502.20937/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:06:28Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"tgB+DFCjikldPD3n3otdnYFF7T8V7EAviO8wPahNEQlpWBMdhL2KQiZDH6lcYkXBxVuTnB/aq9R7zDv4YeW6Ag==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T14:52:58.929970Z"},"content_sha256":"e273fbbce986d6483008ac5f90adaf29c78b7e7422bffeb53d33eeb47cddd536","schema_version":"1.0","event_id":"sha256:e273fbbce986d6483008ac5f90adaf29c78b7e7422bffeb53d33eeb47cddd536"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/SWXFQ3FD4CO4KDQIFS2RIPKA6G/bundle.json","state_url":"https://pith.science/pith/SWXFQ3FD4CO4KDQIFS2RIPKA6G/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/SWXFQ3FD4CO4KDQIFS2RIPKA6G/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T14:52:58Z","links":{"resolver":"https://pith.science/pith/SWXFQ3FD4CO4KDQIFS2RIPKA6G","bundle":"https://pith.science/pith/SWXFQ3FD4CO4KDQIFS2RIPKA6G/bundle.json","state":"https://pith.science/pith/SWXFQ3FD4CO4KDQIFS2RIPKA6G/state.json","well_known_bundle":"https://pith.science/.well-known/pith/SWXFQ3FD4CO4KDQIFS2RIPKA6G/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:SWXFQ3FD4CO4KDQIFS2RIPKA6G","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"88b512b1a8598e0d00a69aa4d41d6ac0dfe7517e62fe3879bff488f7c54e67c6","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2025-02-28T10:46:56Z","title_canon_sha256":"3a3b194f58c019540314aab84d4e146596f5a33e73a1a31fbda98396ed904eec"},"schema_version":"1.0","source":{"id":"2502.20937","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2502.20937","created_at":"2026-07-05T11:06:28Z"},{"alias_kind":"arxiv_version","alias_value":"2502.20937v2","created_at":"2026-07-05T11:06:28Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.20937","created_at":"2026-07-05T11:06:28Z"},{"alias_kind":"pith_short_12","alias_value":"SWXFQ3FD4CO4","created_at":"2026-07-05T11:06:28Z"},{"alias_kind":"pith_short_16","alias_value":"SWXFQ3FD4CO4KDQI","created_at":"2026-07-05T11:06:28Z"},{"alias_kind":"pith_short_8","alias_value":"SWXFQ3FD","created_at":"2026-07-05T11:06:28Z"}],"graph_snapshots":[{"event_id":"sha256:e273fbbce986d6483008ac5f90adaf29c78b7e7422bffeb53d33eeb47cddd536","target":"graph","created_at":"2026-07-05T11:06:28Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2502.20937/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The fundamental property of Cranfield-style evaluations, that system rankings are stable even when assessors disagree on individual relevance decisions, was validated on traditional test collections. However, the paradigm shift towards neural retrieval models affected the characteristics of modern test collections, e.g., documents are short, judged with four grades of relevance, and information needs have no descriptions or narratives. Under these changes, it is unclear whether assessor disagreement remains negligible for system comparisons. We investigate this aspect under the additional cond","authors_text":"Andrew Parry, Eugene Yang, Ferdinand Schlatt, Guglielmo Faggioli, Harrisen Scells, Maik Fr\\\"obe, Saber Zerhoudi, Sean MacAvaney","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2025-02-28T10:46:56Z","title":"Variations in Relevance Judgments and the Shelf Life of Test Collections"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.20937","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b0609cbafbaf044fe779a652e23bf62a466fe068f67af2272d2b56f076d2f54d","target":"record","created_at":"2026-07-05T11:06:28Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"88b512b1a8598e0d00a69aa4d41d6ac0dfe7517e62fe3879bff488f7c54e67c6","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2025-02-28T10:46:56Z","title_canon_sha256":"3a3b194f58c019540314aab84d4e146596f5a33e73a1a31fbda98396ed904eec"},"schema_version":"1.0","source":{"id":"2502.20937","kind":"arxiv","version":2}},"canonical_sha256":"95ae586ca3e09dc50e082cb5143d40f1a9e5c9fcc99a935f526c77ae490dce8a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"95ae586ca3e09dc50e082cb5143d40f1a9e5c9fcc99a935f526c77ae490dce8a","first_computed_at":"2026-07-05T11:06:28.995287Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:06:28.995287Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"qZYxJbgofBdUychL5rNySvrdTHtzsRG6oX42kiRhVA+xq9FYX7MCSu7c75AZQbTHKoRY8LfG4SUO3H+Ag11fBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T11:06:28.995892Z","signed_message":"canonical_sha256_bytes"},"source_id":"2502.20937","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b0609cbafbaf044fe779a652e23bf62a466fe068f67af2272d2b56f076d2f54d","sha256:e273fbbce986d6483008ac5f90adaf29c78b7e7422bffeb53d33eeb47cddd536"],"state_sha256":"8470de37812209d06978512e0bca0d3f244ea57756241cc98321733aea85cde5"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"PlluCK9yK8gKwjvUjRZeFnQ6yKHz9i5rKi4w+IGfocLOdsQw5kiX//FJ4XN0URlAgpH12Jv89BNlU0keDL38Ag==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T14:52:58.936365Z","bundle_sha256":"06a125b9deec12d7b66d4e0de47f345f781c477773cb7819153d29e3543121ca"}}