{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2017:EIBI3PUWBOZCMX2IHCRKJ33WGH","short_pith_number":"pith:EIBI3PUW","canonical_record":{"source":{"id":"1702.04811","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2017-02-15T23:04:09Z","cross_cats_sorted":[],"title_canon_sha256":"767e4182271504a7574ca941e84ac41b67364b430b9f62ef7605b2826d66f880","abstract_canon_sha256":"99db2aad37e5e8e239a5832d089e269029cd07d506ae9b3ca309e789e95fc0a2"},"schema_version":"1.0"},"canonical_sha256":"22028dbe960bb2265f4838a2a4ef7631c2840cef9cc028377786db8e254f8895","source":{"kind":"arxiv","id":"1702.04811","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1702.04811","created_at":"2026-05-18T00:06:16Z"},{"alias_kind":"arxiv_version","alias_value":"1702.04811v3","created_at":"2026-05-18T00:06:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1702.04811","created_at":"2026-05-18T00:06:16Z"},{"alias_kind":"pith_short_12","alias_value":"EIBI3PUWBOZC","created_at":"2026-05-18T12:31:12Z"},{"alias_kind":"pith_short_16","alias_value":"EIBI3PUWBOZCMX2I","created_at":"2026-05-18T12:31:12Z"},{"alias_kind":"pith_short_8","alias_value":"EIBI3PUW","created_at":"2026-05-18T12:31:12Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2017:EIBI3PUWBOZCMX2IHCRKJ33WGH","target":"record","payload":{"canonical_record":{"source":{"id":"1702.04811","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2017-02-15T23:04:09Z","cross_cats_sorted":[],"title_canon_sha256":"767e4182271504a7574ca941e84ac41b67364b430b9f62ef7605b2826d66f880","abstract_canon_sha256":"99db2aad37e5e8e239a5832d089e269029cd07d506ae9b3ca309e789e95fc0a2"},"schema_version":"1.0"},"canonical_sha256":"22028dbe960bb2265f4838a2a4ef7631c2840cef9cc028377786db8e254f8895","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T00:06:16.427417Z","signature_b64":"DVxuRO43Ii7pPXbAOQ/ymt3D4GQsVWFRsdqn+b6ctfTM3Gn2nf4I4PQkIN9AtZ8DQ0h57NAKrD72wN2BCt5ADA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"22028dbe960bb2265f4838a2a4ef7631c2840cef9cc028377786db8e254f8895","last_reissued_at":"2026-05-18T00:06:16.426909Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T00:06:16.426909Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1702.04811","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T00:06:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"H50OSGD6NHH2+bMGMUasWrQF74Bsqtsi7r4MLwlRxFH52FAFpGQEsmuGTRpcz93QmEybFSTDl6K79WBYtKsRAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-26T17:27:34.136429Z"},"content_sha256":"128680db64d52d902a0915b8fc5ba24379d8f9a1eb7170a7919452884e46e1b4","schema_version":"1.0","event_id":"sha256:128680db64d52d902a0915b8fc5ba24379d8f9a1eb7170a7919452884e46e1b4"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2017:EIBI3PUWBOZCMX2IHCRKJ33WGH","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Understanding Deep Learning Performance through an Examination of Test Set Difficulty: A Psychometric Case Study","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Hao Wu, Hong Yu, John P. Lalor, Tsendsuren Munkhdalai","submitted_at":"2017-02-15T23:04:09Z","abstract_excerpt":"Interpreting the performance of deep learning models beyond test set accuracy is challenging. Characteristics of individual data points are often not considered during evaluation, and each data point is treated equally. We examine the impact of a test set question's difficulty to determine if there is a relationship between difficulty and performance. We model difficulty using well-studied psychometric methods on human response patterns. Experiments on Natural Language Inference (NLI) and Sentiment Analysis (SA) show that the likelihood of answering a question correctly is impacted by the ques"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1702.04811","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T00:06:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"iL9mK22mDjDXfMjOJf/LWk+CobdE/w6ZiR7ILYRbdrGfz7WzCmiUuPT1J19cBw4HQ3dwF7p84EqgXK9m21tFCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-26T17:27:34.137099Z"},"content_sha256":"52b9d71ff74c0ab4aef7427bb7951f574d99433ea65bb5b438cdf55794362d3f","schema_version":"1.0","event_id":"sha256:52b9d71ff74c0ab4aef7427bb7951f574d99433ea65bb5b438cdf55794362d3f"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/EIBI3PUWBOZCMX2IHCRKJ33WGH/bundle.json","state_url":"https://pith.science/pith/EIBI3PUWBOZCMX2IHCRKJ33WGH/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/EIBI3PUWBOZCMX2IHCRKJ33WGH/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-05-26T17:27:34Z","links":{"resolver":"https://pith.science/pith/EIBI3PUWBOZCMX2IHCRKJ33WGH","bundle":"https://pith.science/pith/EIBI3PUWBOZCMX2IHCRKJ33WGH/bundle.json","state":"https://pith.science/pith/EIBI3PUWBOZCMX2IHCRKJ33WGH/state.json","well_known_bundle":"https://pith.science/.well-known/pith/EIBI3PUWBOZCMX2IHCRKJ33WGH/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2017:EIBI3PUWBOZCMX2IHCRKJ33WGH","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"99db2aad37e5e8e239a5832d089e269029cd07d506ae9b3ca309e789e95fc0a2","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2017-02-15T23:04:09Z","title_canon_sha256":"767e4182271504a7574ca941e84ac41b67364b430b9f62ef7605b2826d66f880"},"schema_version":"1.0","source":{"id":"1702.04811","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1702.04811","created_at":"2026-05-18T00:06:16Z"},{"alias_kind":"arxiv_version","alias_value":"1702.04811v3","created_at":"2026-05-18T00:06:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1702.04811","created_at":"2026-05-18T00:06:16Z"},{"alias_kind":"pith_short_12","alias_value":"EIBI3PUWBOZC","created_at":"2026-05-18T12:31:12Z"},{"alias_kind":"pith_short_16","alias_value":"EIBI3PUWBOZCMX2I","created_at":"2026-05-18T12:31:12Z"},{"alias_kind":"pith_short_8","alias_value":"EIBI3PUW","created_at":"2026-05-18T12:31:12Z"}],"graph_snapshots":[{"event_id":"sha256:52b9d71ff74c0ab4aef7427bb7951f574d99433ea65bb5b438cdf55794362d3f","target":"graph","created_at":"2026-05-18T00:06:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"Interpreting the performance of deep learning models beyond test set accuracy is challenging. Characteristics of individual data points are often not considered during evaluation, and each data point is treated equally. We examine the impact of a test set question's difficulty to determine if there is a relationship between difficulty and performance. We model difficulty using well-studied psychometric methods on human response patterns. Experiments on Natural Language Inference (NLI) and Sentiment Analysis (SA) show that the likelihood of answering a question correctly is impacted by the ques","authors_text":"Hao Wu, Hong Yu, John P. Lalor, Tsendsuren Munkhdalai","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2017-02-15T23:04:09Z","title":"Understanding Deep Learning Performance through an Examination of Test Set Difficulty: A Psychometric Case Study"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1702.04811","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:128680db64d52d902a0915b8fc5ba24379d8f9a1eb7170a7919452884e46e1b4","target":"record","created_at":"2026-05-18T00:06:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"99db2aad37e5e8e239a5832d089e269029cd07d506ae9b3ca309e789e95fc0a2","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2017-02-15T23:04:09Z","title_canon_sha256":"767e4182271504a7574ca941e84ac41b67364b430b9f62ef7605b2826d66f880"},"schema_version":"1.0","source":{"id":"1702.04811","kind":"arxiv","version":3}},"canonical_sha256":"22028dbe960bb2265f4838a2a4ef7631c2840cef9cc028377786db8e254f8895","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"22028dbe960bb2265f4838a2a4ef7631c2840cef9cc028377786db8e254f8895","first_computed_at":"2026-05-18T00:06:16.426909Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-18T00:06:16.426909Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"DVxuRO43Ii7pPXbAOQ/ymt3D4GQsVWFRsdqn+b6ctfTM3Gn2nf4I4PQkIN9AtZ8DQ0h57NAKrD72wN2BCt5ADA==","signature_status":"signed_v1","signed_at":"2026-05-18T00:06:16.427417Z","signed_message":"canonical_sha256_bytes"},"source_id":"1702.04811","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:128680db64d52d902a0915b8fc5ba24379d8f9a1eb7170a7919452884e46e1b4","sha256:52b9d71ff74c0ab4aef7427bb7951f574d99433ea65bb5b438cdf55794362d3f"],"state_sha256":"78a8d19393c4d009f196409c3c0bcc3510075f53d5e41a93ad086216d672c2a0"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"TJsZzTCdqc5a8VQuGD10puhJtgJKklKY4a6D95hyui2LUDgZIH7g/lGus8bMrcg6NQPEnlSXPvm6qarZf9A2Aw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-05-26T17:27:34.140757Z","bundle_sha256":"03c60387f33e7b6acd76da41cef610e6eba51e2543acc55a4d6f4de8f71ba2af"}}