{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2020:FW6JTPTO72OJG2NRRYOM2AOURQ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"2330d1e7b54ad27e4e93c4d8028ef5c5c019478bf979ccd01baca46553d88b9a","cross_cats_sorted":["cs.CL","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2020-04-28T13:22:26Z","title_canon_sha256":"c45bff83b0453bce363f12668e59a2f6829055b0ac7ae654db3b8a9f59c5b49f"},"schema_version":"1.0","source":{"id":"2004.13486","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2004.13486","created_at":"2026-07-05T00:59:00Z"},{"alias_kind":"arxiv_version","alias_value":"2004.13486v1","created_at":"2026-07-05T00:59:00Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2004.13486","created_at":"2026-07-05T00:59:00Z"},{"alias_kind":"pith_short_12","alias_value":"FW6JTPTO72OJ","created_at":"2026-07-05T00:59:00Z"},{"alias_kind":"pith_short_16","alias_value":"FW6JTPTO72OJG2NR","created_at":"2026-07-05T00:59:00Z"},{"alias_kind":"pith_short_8","alias_value":"FW6JTPTO","created_at":"2026-07-05T00:59:00Z"}],"graph_snapshots":[{"event_id":"sha256:09ba93607d3381e6ba2ce011ce3f8118f2509e3e0f09e552ca578d2a9fda1b1c","target":"graph","created_at":"2026-07-05T00:59:00Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2004.13486/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"As deep learning based models are increasingly being used for information retrieval (IR), a major challenge is to ensure the availability of test collections for measuring their quality. Test collections are generated based on pooling results of various retrieval systems, but until recently this did not include deep learning systems. This raises a major challenge for reusable evaluation: Since deep learning based models use external resources (e.g. word embeddings) and advanced representations as opposed to traditional methods that are mainly based on lexical similarity, they may return differ","authors_text":"Bhaskar Mitra, Daniel Campos, Emine Yilmaz, Nick Craswell","cross_cats":["cs.CL","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2020-04-28T13:22:26Z","title":"On the Reliability of Test Collections for Evaluating Systems of Different Types"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2004.13486","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:072870be24e97dce4c0e487b757dacf0d41001b0b93a97923b2e28a51c9812c0","target":"record","created_at":"2026-07-05T00:59:00Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"2330d1e7b54ad27e4e93c4d8028ef5c5c019478bf979ccd01baca46553d88b9a","cross_cats_sorted":["cs.CL","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2020-04-28T13:22:26Z","title_canon_sha256":"c45bff83b0453bce363f12668e59a2f6829055b0ac7ae654db3b8a9f59c5b49f"},"schema_version":"1.0","source":{"id":"2004.13486","kind":"arxiv","version":1}},"canonical_sha256":"2dbc99be6efe9c9369b18e1ccd01d48c22521debb2bbac1910b9360665313167","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"2dbc99be6efe9c9369b18e1ccd01d48c22521debb2bbac1910b9360665313167","first_computed_at":"2026-07-05T00:59:00.013395Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T00:59:00.013395Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"q0LAJKO3vy9Sf9GazHehElRl85+KLx32Bj4nDHEpqVjVbC6P2lSnYKXcmfsdHCglKDEqjGWwpNhgGXwWeRgZAg==","signature_status":"signed_v1","signed_at":"2026-07-05T00:59:00.013754Z","signed_message":"canonical_sha256_bytes"},"source_id":"2004.13486","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:072870be24e97dce4c0e487b757dacf0d41001b0b93a97923b2e28a51c9812c0","sha256:09ba93607d3381e6ba2ce011ce3f8118f2509e3e0f09e552ca578d2a9fda1b1c"],"state_sha256":"27b39a6df8a3752c54fe7c7b6574a94dbbc1cc1ab3a4c490bc8466a2446c448c"}