{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:3ITQLI7XWZKZY3HGBGH2GFIUVH","short_pith_number":"pith:3ITQLI7X","canonical_record":{"source":{"id":"2110.04629","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-10-09T18:54:02Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"d4cc20e27bddc452f8e45f1e21e63ef7ba110e284274d25b9ddf2d573a59dc63","abstract_canon_sha256":"20cb46ad08c00363f7d3df7ab6b5314443d1a68ccdbaa8344ba2d06ee4deffaf"},"schema_version":"1.0"},"canonical_sha256":"da2705a3f7b6559c6ce6098fa31514a9ff6bf686edfbb92ce7c02166d8dea086","source":{"kind":"arxiv","id":"2110.04629","version":4},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2110.04629","created_at":"2026-07-05T05:12:26Z"},{"alias_kind":"arxiv_version","alias_value":"2110.04629v4","created_at":"2026-07-05T05:12:26Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2110.04629","created_at":"2026-07-05T05:12:26Z"},{"alias_kind":"pith_short_12","alias_value":"3ITQLI7XWZKZ","created_at":"2026-07-05T05:12:26Z"},{"alias_kind":"pith_short_16","alias_value":"3ITQLI7XWZKZY3HG","created_at":"2026-07-05T05:12:26Z"},{"alias_kind":"pith_short_8","alias_value":"3ITQLI7X","created_at":"2026-07-05T05:12:26Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:3ITQLI7XWZKZY3HGBGH2GFIUVH","target":"record","payload":{"canonical_record":{"source":{"id":"2110.04629","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-10-09T18:54:02Z","cross_cats_sorted":["cs.AI","stat.ML"],"title_canon_sha256":"d4cc20e27bddc452f8e45f1e21e63ef7ba110e284274d25b9ddf2d573a59dc63","abstract_canon_sha256":"20cb46ad08c00363f7d3df7ab6b5314443d1a68ccdbaa8344ba2d06ee4deffaf"},"schema_version":"1.0"},"canonical_sha256":"da2705a3f7b6559c6ce6098fa31514a9ff6bf686edfbb92ce7c02166d8dea086","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:12:26.450230Z","signature_b64":"8lz9xSUDYRUapO/OZkCfmIjBFl4c4r9yuNrVVgnAlvy1o+SMomynWlT92BsCMUE+7aHn6oC5F83vaqF9UqDcBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"da2705a3f7b6559c6ce6098fa31514a9ff6bf686edfbb92ce7c02166d8dea086","last_reissued_at":"2026-07-05T05:12:26.449788Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:12:26.449788Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2110.04629","source_version":4,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:12:26Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"5IURMDyxLVDHYHxS7x9sjaI7EuvkXVEa8iruipAwuFtE1r5Apr6NIQyN/6DwsF5wJpxh0yRlOzF3iLquoj0WDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-20T12:38:12.608637Z"},"content_sha256":"1a38821aae81f6b7c3a6084cffe3223118989796a299f3afcf6153d69cc807b4","schema_version":"1.0","event_id":"sha256:1a38821aae81f6b7c3a6084cffe3223118989796a299f3afcf6153d69cc807b4"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:3ITQLI7XWZKZY3HGBGH2GFIUVH","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"The Neural Testbed: Evaluating Joint Predictions","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","stat.ML"],"primary_cat":"cs.LG","authors_text":"Benjamin Van Roy, Botao Hao, Brendan O'Donoghue, Dieterich Lawson, Ian Osband, Morteza Ibrahimi, Seyed Mohammad Asghari, Vikranth Dwaracherla, Xiuyuan Lu, Zheng Wen","submitted_at":"2021-10-09T18:54:02Z","abstract_excerpt":"Predictive distributions quantify uncertainties ignored by point estimates. This paper introduces The Neural Testbed: an open-source benchmark for controlled and principled evaluation of agents that generate such predictions. Crucially, the testbed assesses agents not only on the quality of their marginal predictions per input, but also on their joint predictions across many inputs. We evaluate a range of agents using a simple neural network data generating process. Our results indicate that some popular Bayesian deep learning agents do not fare well with joint predictions, even when they can "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2110.04629","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2110.04629/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:12:26Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Q0JT0fCAFa+UzjX0wZIAun/mKDxU0AUbohl4cn++kdooSko2LpTCzSv92qBMFOzJjkxe69qBt4NFkG3fv538DQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-20T12:38:12.609158Z"},"content_sha256":"b483329d4c88717d8f3839997a83d02ab8bd0b49721712cc9298701e37184470","schema_version":"1.0","event_id":"sha256:b483329d4c88717d8f3839997a83d02ab8bd0b49721712cc9298701e37184470"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/3ITQLI7XWZKZY3HGBGH2GFIUVH/bundle.json","state_url":"https://pith.science/pith/3ITQLI7XWZKZY3HGBGH2GFIUVH/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/3ITQLI7XWZKZY3HGBGH2GFIUVH/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-20T12:38:12Z","links":{"resolver":"https://pith.science/pith/3ITQLI7XWZKZY3HGBGH2GFIUVH","bundle":"https://pith.science/pith/3ITQLI7XWZKZY3HGBGH2GFIUVH/bundle.json","state":"https://pith.science/pith/3ITQLI7XWZKZY3HGBGH2GFIUVH/state.json","well_known_bundle":"https://pith.science/.well-known/pith/3ITQLI7XWZKZY3HGBGH2GFIUVH/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:3ITQLI7XWZKZY3HGBGH2GFIUVH","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"20cb46ad08c00363f7d3df7ab6b5314443d1a68ccdbaa8344ba2d06ee4deffaf","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-10-09T18:54:02Z","title_canon_sha256":"d4cc20e27bddc452f8e45f1e21e63ef7ba110e284274d25b9ddf2d573a59dc63"},"schema_version":"1.0","source":{"id":"2110.04629","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2110.04629","created_at":"2026-07-05T05:12:26Z"},{"alias_kind":"arxiv_version","alias_value":"2110.04629v4","created_at":"2026-07-05T05:12:26Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2110.04629","created_at":"2026-07-05T05:12:26Z"},{"alias_kind":"pith_short_12","alias_value":"3ITQLI7XWZKZ","created_at":"2026-07-05T05:12:26Z"},{"alias_kind":"pith_short_16","alias_value":"3ITQLI7XWZKZY3HG","created_at":"2026-07-05T05:12:26Z"},{"alias_kind":"pith_short_8","alias_value":"3ITQLI7X","created_at":"2026-07-05T05:12:26Z"}],"graph_snapshots":[{"event_id":"sha256:b483329d4c88717d8f3839997a83d02ab8bd0b49721712cc9298701e37184470","target":"graph","created_at":"2026-07-05T05:12:26Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2110.04629/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Predictive distributions quantify uncertainties ignored by point estimates. This paper introduces The Neural Testbed: an open-source benchmark for controlled and principled evaluation of agents that generate such predictions. Crucially, the testbed assesses agents not only on the quality of their marginal predictions per input, but also on their joint predictions across many inputs. We evaluate a range of agents using a simple neural network data generating process. Our results indicate that some popular Bayesian deep learning agents do not fare well with joint predictions, even when they can ","authors_text":"Benjamin Van Roy, Botao Hao, Brendan O'Donoghue, Dieterich Lawson, Ian Osband, Morteza Ibrahimi, Seyed Mohammad Asghari, Vikranth Dwaracherla, Xiuyuan Lu, Zheng Wen","cross_cats":["cs.AI","stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-10-09T18:54:02Z","title":"The Neural Testbed: Evaluating Joint Predictions"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2110.04629","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1a38821aae81f6b7c3a6084cffe3223118989796a299f3afcf6153d69cc807b4","target":"record","created_at":"2026-07-05T05:12:26Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"20cb46ad08c00363f7d3df7ab6b5314443d1a68ccdbaa8344ba2d06ee4deffaf","cross_cats_sorted":["cs.AI","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-10-09T18:54:02Z","title_canon_sha256":"d4cc20e27bddc452f8e45f1e21e63ef7ba110e284274d25b9ddf2d573a59dc63"},"schema_version":"1.0","source":{"id":"2110.04629","kind":"arxiv","version":4}},"canonical_sha256":"da2705a3f7b6559c6ce6098fa31514a9ff6bf686edfbb92ce7c02166d8dea086","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"da2705a3f7b6559c6ce6098fa31514a9ff6bf686edfbb92ce7c02166d8dea086","first_computed_at":"2026-07-05T05:12:26.449788Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:12:26.449788Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"8lz9xSUDYRUapO/OZkCfmIjBFl4c4r9yuNrVVgnAlvy1o+SMomynWlT92BsCMUE+7aHn6oC5F83vaqF9UqDcBw==","signature_status":"signed_v1","signed_at":"2026-07-05T05:12:26.450230Z","signed_message":"canonical_sha256_bytes"},"source_id":"2110.04629","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1a38821aae81f6b7c3a6084cffe3223118989796a299f3afcf6153d69cc807b4","sha256:b483329d4c88717d8f3839997a83d02ab8bd0b49721712cc9298701e37184470"],"state_sha256":"a597e22df1b7834b15152deaed64c9ebc4ce1f2dc7a028d37ed41c59098a120b"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QlpwTJ7FTalApjtzRwcDA/ZDhn3/XWThKlPvFxSi5AKF7s2OtxzHxfRN/B5iWfJYEry6Lme/gekLDZwP/fmyCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-20T12:38:12.613833Z","bundle_sha256":"f8d3f3853331cc1e3b175edcdefce4f7d9c9c233986da1bbf610e09d415cfca0"}}