{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:BPETLVCAKD6OSHVIYQHVZ4NZIK","short_pith_number":"pith:BPETLVCA","canonical_record":{"source":{"id":"2308.04137","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2023-08-08T08:50:27Z","cross_cats_sorted":["cs.CV"],"title_canon_sha256":"341b8c372bc5b8ec69de404a7139dfd577c6fc858e5eeb9de28d7adec2474b8b","abstract_canon_sha256":"f269d013e27fad83186820047eb17f9c394fc0101544c0a1bbe0022a98a23b61"},"schema_version":"1.0"},"canonical_sha256":"0bc935d44050fce91ea8c40f5cf1b9429af322ac40f88941b398555c496e3a76","source":{"kind":"arxiv","id":"2308.04137","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2308.04137","created_at":"2026-07-05T11:07:55Z"},{"alias_kind":"arxiv_version","alias_value":"2308.04137v3","created_at":"2026-07-05T11:07:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2308.04137","created_at":"2026-07-05T11:07:55Z"},{"alias_kind":"pith_short_12","alias_value":"BPETLVCAKD6O","created_at":"2026-07-05T11:07:55Z"},{"alias_kind":"pith_short_16","alias_value":"BPETLVCAKD6OSHVI","created_at":"2026-07-05T11:07:55Z"},{"alias_kind":"pith_short_8","alias_value":"BPETLVCA","created_at":"2026-07-05T11:07:55Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:BPETLVCAKD6OSHVIYQHVZ4NZIK","target":"record","payload":{"canonical_record":{"source":{"id":"2308.04137","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2023-08-08T08:50:27Z","cross_cats_sorted":["cs.CV"],"title_canon_sha256":"341b8c372bc5b8ec69de404a7139dfd577c6fc858e5eeb9de28d7adec2474b8b","abstract_canon_sha256":"f269d013e27fad83186820047eb17f9c394fc0101544c0a1bbe0022a98a23b61"},"schema_version":"1.0"},"canonical_sha256":"0bc935d44050fce91ea8c40f5cf1b9429af322ac40f88941b398555c496e3a76","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:07:55.674564Z","signature_b64":"duWXFcm80GNDLaGfp9lpBuMgfwYDyr7zk+wFvmiMDCULLAAxRjcOmLROBZW42AcJO4Rj1reo4vNx1hSdkgERBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0bc935d44050fce91ea8c40f5cf1b9429af322ac40f88941b398555c496e3a76","last_reissued_at":"2026-07-05T11:07:55.673749Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:07:55.673749Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2308.04137","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:07:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jASFA1n7w5zXvGQlAWOAdZdPN/FDkAMyOFYJBYzYw+3vSJyXfzrLa/ULCtw4UPxcE1hyNhfnOJxjhh3Fuch4Dg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-22T04:11:22.259469Z"},"content_sha256":"5bf248c9097b2134b823335d20a47a9242565d474b89eb8f494ceecddd92142c","schema_version":"1.0","event_id":"sha256:5bf248c9097b2134b823335d20a47a9242565d474b89eb8f494ceecddd92142c"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:BPETLVCAKD6OSHVIYQHVZ4NZIK","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"A Comprehensive Assessment Benchmark for Rigorously Evaluating Deep Learning Image Classifiers","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.CV"],"primary_cat":"cs.LG","authors_text":"Michael W. Spratling","submitted_at":"2023-08-08T08:50:27Z","abstract_excerpt":"Reliable and robust evaluation methods are a necessary first step towards developing machine learning models that are themselves robust and reliable. Unfortunately, current evaluation protocols typically used to assess classifiers fail to comprehensively evaluate performance as they tend to rely on limited types of test data, and ignore others. For example, using the standard test data fails to evaluate the predictions made by the classifier to samples from classes it was not trained on. On the other hand, testing with data containing samples from unknown classes fails to evaluate how well the"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2308.04137","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2308.04137/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:07:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"seE0ckaF+ud1LL2fXnNI+8iwmMxmBtf0Rxs7a/HiGoTZSAAqo2Bt1cF6dFZB69WxCoEPUuuLR621gnWJg5RtCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-22T04:11:22.259980Z"},"content_sha256":"35ac134c3f49e9f37152b0563ca07aca4f72b3d0cca621839da955f674f70b1f","schema_version":"1.0","event_id":"sha256:35ac134c3f49e9f37152b0563ca07aca4f72b3d0cca621839da955f674f70b1f"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/BPETLVCAKD6OSHVIYQHVZ4NZIK/bundle.json","state_url":"https://pith.science/pith/BPETLVCAKD6OSHVIYQHVZ4NZIK/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/BPETLVCAKD6OSHVIYQHVZ4NZIK/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-22T04:11:22Z","links":{"resolver":"https://pith.science/pith/BPETLVCAKD6OSHVIYQHVZ4NZIK","bundle":"https://pith.science/pith/BPETLVCAKD6OSHVIYQHVZ4NZIK/bundle.json","state":"https://pith.science/pith/BPETLVCAKD6OSHVIYQHVZ4NZIK/state.json","well_known_bundle":"https://pith.science/.well-known/pith/BPETLVCAKD6OSHVIYQHVZ4NZIK/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:BPETLVCAKD6OSHVIYQHVZ4NZIK","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"f269d013e27fad83186820047eb17f9c394fc0101544c0a1bbe0022a98a23b61","cross_cats_sorted":["cs.CV"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2023-08-08T08:50:27Z","title_canon_sha256":"341b8c372bc5b8ec69de404a7139dfd577c6fc858e5eeb9de28d7adec2474b8b"},"schema_version":"1.0","source":{"id":"2308.04137","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2308.04137","created_at":"2026-07-05T11:07:55Z"},{"alias_kind":"arxiv_version","alias_value":"2308.04137v3","created_at":"2026-07-05T11:07:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2308.04137","created_at":"2026-07-05T11:07:55Z"},{"alias_kind":"pith_short_12","alias_value":"BPETLVCAKD6O","created_at":"2026-07-05T11:07:55Z"},{"alias_kind":"pith_short_16","alias_value":"BPETLVCAKD6OSHVI","created_at":"2026-07-05T11:07:55Z"},{"alias_kind":"pith_short_8","alias_value":"BPETLVCA","created_at":"2026-07-05T11:07:55Z"}],"graph_snapshots":[{"event_id":"sha256:35ac134c3f49e9f37152b0563ca07aca4f72b3d0cca621839da955f674f70b1f","target":"graph","created_at":"2026-07-05T11:07:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2308.04137/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reliable and robust evaluation methods are a necessary first step towards developing machine learning models that are themselves robust and reliable. Unfortunately, current evaluation protocols typically used to assess classifiers fail to comprehensively evaluate performance as they tend to rely on limited types of test data, and ignore others. For example, using the standard test data fails to evaluate the predictions made by the classifier to samples from classes it was not trained on. On the other hand, testing with data containing samples from unknown classes fails to evaluate how well the","authors_text":"Michael W. Spratling","cross_cats":["cs.CV"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2023-08-08T08:50:27Z","title":"A Comprehensive Assessment Benchmark for Rigorously Evaluating Deep Learning Image Classifiers"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2308.04137","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5bf248c9097b2134b823335d20a47a9242565d474b89eb8f494ceecddd92142c","target":"record","created_at":"2026-07-05T11:07:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"f269d013e27fad83186820047eb17f9c394fc0101544c0a1bbe0022a98a23b61","cross_cats_sorted":["cs.CV"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2023-08-08T08:50:27Z","title_canon_sha256":"341b8c372bc5b8ec69de404a7139dfd577c6fc858e5eeb9de28d7adec2474b8b"},"schema_version":"1.0","source":{"id":"2308.04137","kind":"arxiv","version":3}},"canonical_sha256":"0bc935d44050fce91ea8c40f5cf1b9429af322ac40f88941b398555c496e3a76","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0bc935d44050fce91ea8c40f5cf1b9429af322ac40f88941b398555c496e3a76","first_computed_at":"2026-07-05T11:07:55.673749Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:07:55.673749Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"duWXFcm80GNDLaGfp9lpBuMgfwYDyr7zk+wFvmiMDCULLAAxRjcOmLROBZW42AcJO4Rj1reo4vNx1hSdkgERBw==","signature_status":"signed_v1","signed_at":"2026-07-05T11:07:55.674564Z","signed_message":"canonical_sha256_bytes"},"source_id":"2308.04137","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5bf248c9097b2134b823335d20a47a9242565d474b89eb8f494ceecddd92142c","sha256:35ac134c3f49e9f37152b0563ca07aca4f72b3d0cca621839da955f674f70b1f"],"state_sha256":"3c2e3f6eb1ee8d0a189762712965f132037c00e64b3aa43e5868049cb9b5ced1"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"D5tWF2mxIjoFe5BKS+WX/HieAeRUGsPgrr5d2maAIQxjFWLka0c/38RbAmln4i2YAFgFybmQaTEJE9q6+iuZAQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-22T04:11:22.264397Z","bundle_sha256":"22ae43c77ea5ca98fdd4d5e53997ce105592828a81cf3eed9d35142c9fb92037"}}