{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2018:OVTFYVE7PK55U6NQBTSSUZRZ7K","short_pith_number":"pith:OVTFYVE7","canonical_record":{"source":{"id":"1812.01647","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2018-12-04T19:39:53Z","cross_cats_sorted":["cs.CR","stat.ML"],"title_canon_sha256":"7a75396250e799f548a48e58739f29b06523483491e7856e7211b0e08051d02f","abstract_canon_sha256":"824e70075aff97c97f3f57aa829db2ab6d643f568c3bc6006c4ea24f79167936"},"schema_version":"1.0"},"canonical_sha256":"75665c549f7abbda79b00ce52a6639fab45230fec3dbae578ad17422938d3fb3","source":{"kind":"arxiv","id":"1812.01647","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1812.01647","created_at":"2026-05-17T23:59:00Z"},{"alias_kind":"arxiv_version","alias_value":"1812.01647v1","created_at":"2026-05-17T23:59:00Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1812.01647","created_at":"2026-05-17T23:59:00Z"},{"alias_kind":"pith_short_12","alias_value":"OVTFYVE7PK55","created_at":"2026-05-18T12:32:43Z"},{"alias_kind":"pith_short_16","alias_value":"OVTFYVE7PK55U6NQ","created_at":"2026-05-18T12:32:43Z"},{"alias_kind":"pith_short_8","alias_value":"OVTFYVE7","created_at":"2026-05-18T12:32:43Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2018:OVTFYVE7PK55U6NQBTSSUZRZ7K","target":"record","payload":{"canonical_record":{"source":{"id":"1812.01647","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2018-12-04T19:39:53Z","cross_cats_sorted":["cs.CR","stat.ML"],"title_canon_sha256":"7a75396250e799f548a48e58739f29b06523483491e7856e7211b0e08051d02f","abstract_canon_sha256":"824e70075aff97c97f3f57aa829db2ab6d643f568c3bc6006c4ea24f79167936"},"schema_version":"1.0"},"canonical_sha256":"75665c549f7abbda79b00ce52a6639fab45230fec3dbae578ad17422938d3fb3","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:59:00.998106Z","signature_b64":"3xlOYCcXR+KKgUbC0Aru/q5A9/8TNdzZP8oX/X/gm6LyiWSiusVd9tSw3H8j53rduvRIXXmLl1ZT2LAklEyYAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"75665c549f7abbda79b00ce52a6639fab45230fec3dbae578ad17422938d3fb3","last_reissued_at":"2026-05-17T23:59:00.997639Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:59:00.997639Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1812.01647","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:59:00Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"3mT4Y0+CrR9p4IOMyCfwjG7aQJGCmvwDhpzkr5sM6qJzHoP1Yo5+v6NdeMCj5gl8gyO3cx3Qsgv/cP5GEHGYCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-28T07:55:27.739680Z"},"content_sha256":"c5e1337606748249168629027623c037d5a46a052de1bbc7b46b6efb4a1fece2","schema_version":"1.0","event_id":"sha256:c5e1337606748249168629027623c037d5a46a052de1bbc7b46b6efb4a1fece2"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2018:OVTFYVE7PK55U6NQBTSSUZRZ7K","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Rigorous Agent Evaluation: An Adversarial Approach to Uncover Catastrophic Failures","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CR","stat.ML"],"primary_cat":"cs.LG","authors_text":"Ananya Kumar, Avraham Ruderman, Csaba Szepesvari, Jonathan Uesato, Keith Anderson, Krishmamurthy (Dj) Dvijotham, Nicolas Heess, Pushmeet Kohli, Tom Erez","submitted_at":"2018-12-04T19:39:53Z","abstract_excerpt":"This paper addresses the problem of evaluating learning systems in safety critical domains such as autonomous driving, where failures can have catastrophic consequences. We focus on two problems: searching for scenarios when learned agents fail and assessing their probability of failure. The standard method for agent evaluation in reinforcement learning, Vanilla Monte Carlo, can miss failures entirely, leading to the deployment of unsafe agents. We demonstrate this is an issue for current agents, where even matching the compute used for training is sometimes insufficient for evaluation. To add"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1812.01647","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:59:00Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"mjyEgkcWc3eD6umsu5ubj/i1MBGHOX0lO3YX5yKluYP4ckOQH+ySfBcRTS2CD3kpRNUUP1XLyN03ebRfeMjPDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-28T07:55:27.740034Z"},"content_sha256":"0be9ebc5e92da53bee2aedd8c75a2d77c4e4ea036eb11cb0213d0de84e5d66e2","schema_version":"1.0","event_id":"sha256:0be9ebc5e92da53bee2aedd8c75a2d77c4e4ea036eb11cb0213d0de84e5d66e2"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/OVTFYVE7PK55U6NQBTSSUZRZ7K/bundle.json","state_url":"https://pith.science/pith/OVTFYVE7PK55U6NQBTSSUZRZ7K/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/OVTFYVE7PK55U6NQBTSSUZRZ7K/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-05-28T07:55:27Z","links":{"resolver":"https://pith.science/pith/OVTFYVE7PK55U6NQBTSSUZRZ7K","bundle":"https://pith.science/pith/OVTFYVE7PK55U6NQBTSSUZRZ7K/bundle.json","state":"https://pith.science/pith/OVTFYVE7PK55U6NQBTSSUZRZ7K/state.json","well_known_bundle":"https://pith.science/.well-known/pith/OVTFYVE7PK55U6NQBTSSUZRZ7K/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2018:OVTFYVE7PK55U6NQBTSSUZRZ7K","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"824e70075aff97c97f3f57aa829db2ab6d643f568c3bc6006c4ea24f79167936","cross_cats_sorted":["cs.CR","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2018-12-04T19:39:53Z","title_canon_sha256":"7a75396250e799f548a48e58739f29b06523483491e7856e7211b0e08051d02f"},"schema_version":"1.0","source":{"id":"1812.01647","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1812.01647","created_at":"2026-05-17T23:59:00Z"},{"alias_kind":"arxiv_version","alias_value":"1812.01647v1","created_at":"2026-05-17T23:59:00Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1812.01647","created_at":"2026-05-17T23:59:00Z"},{"alias_kind":"pith_short_12","alias_value":"OVTFYVE7PK55","created_at":"2026-05-18T12:32:43Z"},{"alias_kind":"pith_short_16","alias_value":"OVTFYVE7PK55U6NQ","created_at":"2026-05-18T12:32:43Z"},{"alias_kind":"pith_short_8","alias_value":"OVTFYVE7","created_at":"2026-05-18T12:32:43Z"}],"graph_snapshots":[{"event_id":"sha256:0be9ebc5e92da53bee2aedd8c75a2d77c4e4ea036eb11cb0213d0de84e5d66e2","target":"graph","created_at":"2026-05-17T23:59:00Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"This paper addresses the problem of evaluating learning systems in safety critical domains such as autonomous driving, where failures can have catastrophic consequences. We focus on two problems: searching for scenarios when learned agents fail and assessing their probability of failure. The standard method for agent evaluation in reinforcement learning, Vanilla Monte Carlo, can miss failures entirely, leading to the deployment of unsafe agents. We demonstrate this is an issue for current agents, where even matching the compute used for training is sometimes insufficient for evaluation. To add","authors_text":"Ananya Kumar, Avraham Ruderman, Csaba Szepesvari, Jonathan Uesato, Keith Anderson, Krishmamurthy (Dj) Dvijotham, Nicolas Heess, Pushmeet Kohli, Tom Erez","cross_cats":["cs.CR","stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2018-12-04T19:39:53Z","title":"Rigorous Agent Evaluation: An Adversarial Approach to Uncover Catastrophic Failures"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1812.01647","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:c5e1337606748249168629027623c037d5a46a052de1bbc7b46b6efb4a1fece2","target":"record","created_at":"2026-05-17T23:59:00Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"824e70075aff97c97f3f57aa829db2ab6d643f568c3bc6006c4ea24f79167936","cross_cats_sorted":["cs.CR","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2018-12-04T19:39:53Z","title_canon_sha256":"7a75396250e799f548a48e58739f29b06523483491e7856e7211b0e08051d02f"},"schema_version":"1.0","source":{"id":"1812.01647","kind":"arxiv","version":1}},"canonical_sha256":"75665c549f7abbda79b00ce52a6639fab45230fec3dbae578ad17422938d3fb3","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"75665c549f7abbda79b00ce52a6639fab45230fec3dbae578ad17422938d3fb3","first_computed_at":"2026-05-17T23:59:00.997639Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-17T23:59:00.997639Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"3xlOYCcXR+KKgUbC0Aru/q5A9/8TNdzZP8oX/X/gm6LyiWSiusVd9tSw3H8j53rduvRIXXmLl1ZT2LAklEyYAA==","signature_status":"signed_v1","signed_at":"2026-05-17T23:59:00.998106Z","signed_message":"canonical_sha256_bytes"},"source_id":"1812.01647","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:c5e1337606748249168629027623c037d5a46a052de1bbc7b46b6efb4a1fece2","sha256:0be9ebc5e92da53bee2aedd8c75a2d77c4e4ea036eb11cb0213d0de84e5d66e2"],"state_sha256":"3690e1037a9d2f47c7d3a8c55016790e59af6e6c15ab272d3e0f5cb72878a7a5"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"KUzFmTEwRPW9VuXGEv98prptZkHc8ZeUSsuClEyWG2P6drTwVl7X2MyqC4TGIEszkzU8zYVYDPgABYpGVo7hBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-05-28T07:55:27.742040Z","bundle_sha256":"d0266a66fa0f7dfe5dda73a8b0a07921affaa21c52aba8dd75142ed447b7e5ad"}}