{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2021:ODI3RVT6FZTBGOYWBGJGE4N6F6","short_pith_number":"pith:ODI3RVT6","schema_version":"1.0","canonical_sha256":"70d1b8d67e2e66133b1609926271be2fad75658dd865a1484000c937864e0367","source":{"kind":"arxiv","id":"2105.07975","version":1},"attestation_state":"computed","paper":{"title":"How Deep is your Learning: the DL-HARD Annotated Deep Learning Dataset","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.IR","authors_text":"Andrew Yates, Iain Mackie, Jeffery Dalton","submitted_at":"2021-05-17T15:58:44Z","abstract_excerpt":"Deep Learning Hard (DL-HARD) is a new annotated dataset designed to more effectively evaluate neural ranking models on complex topics. It builds on TREC Deep Learning (DL) topics by extensively annotating them with question intent categories, answer types, wikified entities, topic categories, and result type metadata from a commercial web search engine. Based on this data, we introduce a framework for identifying challenging queries. DL-HARD contains fifty topics from the official DL 2019/2020 evaluation benchmark, half of which are newly and independently assessed. We perform experiments usin"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2105.07975","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2021-05-17T15:58:44Z","cross_cats_sorted":[],"title_canon_sha256":"95e7674f989d32917fe820ce4fa84feb9d8f4bca8475e3d0aefde7d08f790b9d","abstract_canon_sha256":"deb04b6ae82997aadfdf9d8bef8bf13dc9ea6434ca2e491de3ce59c69f9d8656"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:40:46.719892Z","signature_b64":"5UOY4peP4Dda8GaLH6xOtb3Offs8v2MNoeDfyO2PgkGue19KM0kZnv3STOwA3bnYEoDTPwBO5sYTUEVxRcOTDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"70d1b8d67e2e66133b1609926271be2fad75658dd865a1484000c937864e0367","last_reissued_at":"2026-07-05T02:40:46.719384Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:40:46.719384Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"How Deep is your Learning: the DL-HARD Annotated Deep Learning Dataset","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.IR","authors_text":"Andrew Yates, Iain Mackie, Jeffery Dalton","submitted_at":"2021-05-17T15:58:44Z","abstract_excerpt":"Deep Learning Hard (DL-HARD) is a new annotated dataset designed to more effectively evaluate neural ranking models on complex topics. It builds on TREC Deep Learning (DL) topics by extensively annotating them with question intent categories, answer types, wikified entities, topic categories, and result type metadata from a commercial web search engine. Based on this data, we introduce a framework for identifying challenging queries. DL-HARD contains fifty topics from the official DL 2019/2020 evaluation benchmark, half of which are newly and independently assessed. We perform experiments usin"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2105.07975","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2105.07975/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2105.07975","created_at":"2026-07-05T02:40:46.719440+00:00"},{"alias_kind":"arxiv_version","alias_value":"2105.07975v1","created_at":"2026-07-05T02:40:46.719440+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2105.07975","created_at":"2026-07-05T02:40:46.719440+00:00"},{"alias_kind":"pith_short_12","alias_value":"ODI3RVT6FZTB","created_at":"2026-07-05T02:40:46.719440+00:00"},{"alias_kind":"pith_short_16","alias_value":"ODI3RVT6FZTBGOYW","created_at":"2026-07-05T02:40:46.719440+00:00"},{"alias_kind":"pith_short_8","alias_value":"ODI3RVT6","created_at":"2026-07-05T02:40:46.719440+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/ODI3RVT6FZTBGOYWBGJGE4N6F6","json":"https://pith.science/pith/ODI3RVT6FZTBGOYWBGJGE4N6F6.json","graph_json":"https://pith.science/api/pith-number/ODI3RVT6FZTBGOYWBGJGE4N6F6/graph.json","events_json":"https://pith.science/api/pith-number/ODI3RVT6FZTBGOYWBGJGE4N6F6/events.json","paper":"https://pith.science/paper/ODI3RVT6"},"agent_actions":{"view_html":"https://pith.science/pith/ODI3RVT6FZTBGOYWBGJGE4N6F6","download_json":"https://pith.science/pith/ODI3RVT6FZTBGOYWBGJGE4N6F6.json","view_paper":"https://pith.science/paper/ODI3RVT6","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2105.07975&json=true","fetch_graph":"https://pith.science/api/pith-number/ODI3RVT6FZTBGOYWBGJGE4N6F6/graph.json","fetch_events":"https://pith.science/api/pith-number/ODI3RVT6FZTBGOYWBGJGE4N6F6/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/ODI3RVT6FZTBGOYWBGJGE4N6F6/action/timestamp_anchor","attest_storage":"https://pith.science/pith/ODI3RVT6FZTBGOYWBGJGE4N6F6/action/storage_attestation","attest_author":"https://pith.science/pith/ODI3RVT6FZTBGOYWBGJGE4N6F6/action/author_attestation","sign_citation":"https://pith.science/pith/ODI3RVT6FZTBGOYWBGJGE4N6F6/action/citation_signature","submit_replication":"https://pith.science/pith/ODI3RVT6FZTBGOYWBGJGE4N6F6/action/replication_record"}},"created_at":"2026-07-05T02:40:46.719440+00:00","updated_at":"2026-07-05T02:40:46.719440+00:00"}