{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2020:V3OVDYMWC563LAZ7TOEH57NTAU","short_pith_number":"pith:V3OVDYMW","canonical_record":{"source":{"id":"2005.09241","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2020-05-19T06:45:50Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"8b7812493d7bdcd0b423bedfa184ef980e79c06a5d7b53ee5ec94bb2a6e85cb4","abstract_canon_sha256":"5a1a7afba3958782674926425b0f54576f978823e6bc448a691447d348815822"},"schema_version":"1.0"},"canonical_sha256":"aedd51e196177db5833f9b887efdb30531e59b43a57373fe0d2115c0e93d95eb","source":{"kind":"arxiv","id":"2005.09241","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2005.09241","created_at":"2026-07-05T01:03:48Z"},{"alias_kind":"arxiv_version","alias_value":"2005.09241v1","created_at":"2026-07-05T01:03:48Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2005.09241","created_at":"2026-07-05T01:03:48Z"},{"alias_kind":"pith_short_12","alias_value":"V3OVDYMWC563","created_at":"2026-07-05T01:03:48Z"},{"alias_kind":"pith_short_16","alias_value":"V3OVDYMWC563LAZ7","created_at":"2026-07-05T01:03:48Z"},{"alias_kind":"pith_short_8","alias_value":"V3OVDYMW","created_at":"2026-07-05T01:03:48Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2020:V3OVDYMWC563LAZ7TOEH57NTAU","target":"record","payload":{"canonical_record":{"source":{"id":"2005.09241","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2020-05-19T06:45:50Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"8b7812493d7bdcd0b423bedfa184ef980e79c06a5d7b53ee5ec94bb2a6e85cb4","abstract_canon_sha256":"5a1a7afba3958782674926425b0f54576f978823e6bc448a691447d348815822"},"schema_version":"1.0"},"canonical_sha256":"aedd51e196177db5833f9b887efdb30531e59b43a57373fe0d2115c0e93d95eb","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T01:03:48.341956Z","signature_b64":"xv39sWCuigiNWjieacfvJ9g5qZwQexNPR7eSkiFt+looKuhhWzrTkjqz4/QQ8HTyqcjrF5M/48mZt/vUkVhlBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"aedd51e196177db5833f9b887efdb30531e59b43a57373fe0d2115c0e93d95eb","last_reissued_at":"2026-07-05T01:03:48.341520Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T01:03:48.341520Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2005.09241","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:03:48Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"F4CFYaazQ345XPRNlqF6Wje/kAKGboL/yhqIpy07Fn1Gjc9qYsWC4orayijrCN9hWfvBbkJq7ser2JGYvx6fCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T14:25:18.898886Z"},"content_sha256":"0f36b68354e0a71f39e82793fd250a754b213532a47d80df2c24d693c728d728","schema_version":"1.0","event_id":"sha256:0f36b68354e0a71f39e82793fd250a754b213532a47d80df2c24d693c728d728"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2020:V3OVDYMWC563LAZ7TOEH57NTAU","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"On the Value of Out-of-Distribution Testing: An Example of Goodhart's Law","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CV","authors_text":"Anton van den Hengel, Christopher Kanan, Damien Teney, Ehsan Abbasnejad, Kushal Kafle, Robik Shrestha","submitted_at":"2020-05-19T06:45:50Z","abstract_excerpt":"Out-of-distribution (OOD) testing is increasingly popular for evaluating a machine learning system's ability to generalize beyond the biases of a training set. OOD benchmarks are designed to present a different joint distribution of data and labels between training and test time. VQA-CP has become the standard OOD benchmark for visual question answering, but we discovered three troubling practices in its current use. First, most published methods rely on explicit knowledge of the construction of the OOD splits. They often rely on ``inverting'' the distribution of labels, e.g. answering mostly "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2005.09241","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2005.09241/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:03:48Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"dPxzf9kUxkyxLK5lrMidvIQ5Vw6IziEOUW4fpBfDmfrE8BolPTSOdgNPo1Sl626fr3jxPsVEVt0b5JUkoC3DCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T14:25:18.899461Z"},"content_sha256":"ad18f361e0b807f709e6933d278b8fba17e77f8f07722c9986e9ac70fd74fbf8","schema_version":"1.0","event_id":"sha256:ad18f361e0b807f709e6933d278b8fba17e77f8f07722c9986e9ac70fd74fbf8"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/V3OVDYMWC563LAZ7TOEH57NTAU/bundle.json","state_url":"https://pith.science/pith/V3OVDYMWC563LAZ7TOEH57NTAU/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/V3OVDYMWC563LAZ7TOEH57NTAU/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-04T14:25:18Z","links":{"resolver":"https://pith.science/pith/V3OVDYMWC563LAZ7TOEH57NTAU","bundle":"https://pith.science/pith/V3OVDYMWC563LAZ7TOEH57NTAU/bundle.json","state":"https://pith.science/pith/V3OVDYMWC563LAZ7TOEH57NTAU/state.json","well_known_bundle":"https://pith.science/.well-known/pith/V3OVDYMWC563LAZ7TOEH57NTAU/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2020:V3OVDYMWC563LAZ7TOEH57NTAU","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"5a1a7afba3958782674926425b0f54576f978823e6bc448a691447d348815822","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2020-05-19T06:45:50Z","title_canon_sha256":"8b7812493d7bdcd0b423bedfa184ef980e79c06a5d7b53ee5ec94bb2a6e85cb4"},"schema_version":"1.0","source":{"id":"2005.09241","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2005.09241","created_at":"2026-07-05T01:03:48Z"},{"alias_kind":"arxiv_version","alias_value":"2005.09241v1","created_at":"2026-07-05T01:03:48Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2005.09241","created_at":"2026-07-05T01:03:48Z"},{"alias_kind":"pith_short_12","alias_value":"V3OVDYMWC563","created_at":"2026-07-05T01:03:48Z"},{"alias_kind":"pith_short_16","alias_value":"V3OVDYMWC563LAZ7","created_at":"2026-07-05T01:03:48Z"},{"alias_kind":"pith_short_8","alias_value":"V3OVDYMW","created_at":"2026-07-05T01:03:48Z"}],"graph_snapshots":[{"event_id":"sha256:ad18f361e0b807f709e6933d278b8fba17e77f8f07722c9986e9ac70fd74fbf8","target":"graph","created_at":"2026-07-05T01:03:48Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2005.09241/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Out-of-distribution (OOD) testing is increasingly popular for evaluating a machine learning system's ability to generalize beyond the biases of a training set. OOD benchmarks are designed to present a different joint distribution of data and labels between training and test time. VQA-CP has become the standard OOD benchmark for visual question answering, but we discovered three troubling practices in its current use. First, most published methods rely on explicit knowledge of the construction of the OOD splits. They often rely on ``inverting'' the distribution of labels, e.g. answering mostly ","authors_text":"Anton van den Hengel, Christopher Kanan, Damien Teney, Ehsan Abbasnejad, Kushal Kafle, Robik Shrestha","cross_cats":["cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2020-05-19T06:45:50Z","title":"On the Value of Out-of-Distribution Testing: An Example of Goodhart's Law"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2005.09241","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:0f36b68354e0a71f39e82793fd250a754b213532a47d80df2c24d693c728d728","target":"record","created_at":"2026-07-05T01:03:48Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"5a1a7afba3958782674926425b0f54576f978823e6bc448a691447d348815822","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2020-05-19T06:45:50Z","title_canon_sha256":"8b7812493d7bdcd0b423bedfa184ef980e79c06a5d7b53ee5ec94bb2a6e85cb4"},"schema_version":"1.0","source":{"id":"2005.09241","kind":"arxiv","version":1}},"canonical_sha256":"aedd51e196177db5833f9b887efdb30531e59b43a57373fe0d2115c0e93d95eb","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"aedd51e196177db5833f9b887efdb30531e59b43a57373fe0d2115c0e93d95eb","first_computed_at":"2026-07-05T01:03:48.341520Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T01:03:48.341520Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"xv39sWCuigiNWjieacfvJ9g5qZwQexNPR7eSkiFt+looKuhhWzrTkjqz4/QQ8HTyqcjrF5M/48mZt/vUkVhlBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T01:03:48.341956Z","signed_message":"canonical_sha256_bytes"},"source_id":"2005.09241","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:0f36b68354e0a71f39e82793fd250a754b213532a47d80df2c24d693c728d728","sha256:ad18f361e0b807f709e6933d278b8fba17e77f8f07722c9986e9ac70fd74fbf8"],"state_sha256":"c498b3db6d2030ace9b670e3e9ec769084a1053daffd83232259b52ffaa19d88"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"kslmKxy0KXd7UrL0VTScFvADkOuD7xjxwCN2l6ea+DOXDw8j2ztqCzqWXcXFiMD04/qng8f0LhIEcONWssRoCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-04T14:25:18.950287Z","bundle_sha256":"56c74e09afec95f2e1d123ec463db0bbf42bc76888d986e8a861259defceecaa"}}