{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:FUDGDQPHQFNNZUDK2XPQGZX5QV","short_pith_number":"pith:FUDGDQPH","canonical_record":{"source":{"id":"2107.06857","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.MA","submitted_at":"2021-07-14T17:22:14Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"1bd9134067a1b4f9d901a3a4d50345083c7f1ed277c3da39758ed8de530fe39a","abstract_canon_sha256":"45e22d9a08b9a66d504abf6660c84482e1bfd9e96a30ca7125b98789ca00b1eb"},"schema_version":"1.0"},"canonical_sha256":"2d0661c1e7815adcd06ad5df0366fd854edae725e74a56196d3d9b7f9e211638","source":{"kind":"arxiv","id":"2107.06857","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2107.06857","created_at":"2026-07-05T02:58:02Z"},{"alias_kind":"arxiv_version","alias_value":"2107.06857v1","created_at":"2026-07-05T02:58:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2107.06857","created_at":"2026-07-05T02:58:02Z"},{"alias_kind":"pith_short_12","alias_value":"FUDGDQPHQFNN","created_at":"2026-07-05T02:58:02Z"},{"alias_kind":"pith_short_16","alias_value":"FUDGDQPHQFNNZUDK","created_at":"2026-07-05T02:58:02Z"},{"alias_kind":"pith_short_8","alias_value":"FUDGDQPH","created_at":"2026-07-05T02:58:02Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:FUDGDQPHQFNNZUDK2XPQGZX5QV","target":"record","payload":{"canonical_record":{"source":{"id":"2107.06857","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.MA","submitted_at":"2021-07-14T17:22:14Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"1bd9134067a1b4f9d901a3a4d50345083c7f1ed277c3da39758ed8de530fe39a","abstract_canon_sha256":"45e22d9a08b9a66d504abf6660c84482e1bfd9e96a30ca7125b98789ca00b1eb"},"schema_version":"1.0"},"canonical_sha256":"2d0661c1e7815adcd06ad5df0366fd854edae725e74a56196d3d9b7f9e211638","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:58:02.149278Z","signature_b64":"vuRIyYx84M/Ze+k3yHcy90V8RzKBSbSWsugp/Ixzh6Sj00w3uEzpPvWUt5HPkZmYTsmN3uTk1cO0yOYxgemqCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2d0661c1e7815adcd06ad5df0366fd854edae725e74a56196d3d9b7f9e211638","last_reissued_at":"2026-07-05T02:58:02.148867Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:58:02.148867Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2107.06857","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:58:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"10EoganSjKrLTFxEtFecrRqtfUGFxlOsEkD2/pVYlZ+BcI+WGovsiPK8I15Y/7u1vnNjuIR71p0EcrxNruZ/DA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-11T16:51:06.373488Z"},"content_sha256":"032af1a05604723105f9149745489eaa03ef7425655e78c5f19304b3ef986663","schema_version":"1.0","event_id":"sha256:032af1a05604723105f9149745489eaa03ef7425655e78c5f19304b3ef986663"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:FUDGDQPHQFNNZUDK2XPQGZX5QV","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Scalable Evaluation of Multi-Agent Reinforcement Learning with Melting Pot","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.MA","authors_text":"Alexander Sasha Vezhnevets, Charles Beattie, Edgar Du\\'e\\~nez-Guzm\\'an, Igor Mordatch, Jayd Matyas, Joel Z. Leibo, John P. Agapiou, Peter Sunehag, Raphael Koster, Thore Graepel","submitted_at":"2021-07-14T17:22:14Z","abstract_excerpt":"Existing evaluation suites for multi-agent reinforcement learning (MARL) do not assess generalization to novel situations as their primary objective (unlike supervised-learning benchmarks). Our contribution, Melting Pot, is a MARL evaluation suite that fills this gap, and uses reinforcement learning to reduce the human labor required to create novel test scenarios. This works because one agent's behavior constitutes (part of) another agent's environment. To demonstrate scalability, we have created over 80 unique test scenarios covering a broad range of research topics such as social dilemmas, "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2107.06857","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2107.06857/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:58:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QBb9c9GVF2c+iBecq5nhtqEbogwCQZ+klpzUZciFGzbEySd/2ItTgt1QymH0YklXnEbK2a+lpTybiRmNppBMBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-11T16:51:06.373858Z"},"content_sha256":"7809425ba1abd6e0c6eb0d8403779a181a7dadd5a1bdded512b64fb79ad292f6","schema_version":"1.0","event_id":"sha256:7809425ba1abd6e0c6eb0d8403779a181a7dadd5a1bdded512b64fb79ad292f6"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/FUDGDQPHQFNNZUDK2XPQGZX5QV/bundle.json","state_url":"https://pith.science/pith/FUDGDQPHQFNNZUDK2XPQGZX5QV/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/FUDGDQPHQFNNZUDK2XPQGZX5QV/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-11T16:51:06Z","links":{"resolver":"https://pith.science/pith/FUDGDQPHQFNNZUDK2XPQGZX5QV","bundle":"https://pith.science/pith/FUDGDQPHQFNNZUDK2XPQGZX5QV/bundle.json","state":"https://pith.science/pith/FUDGDQPHQFNNZUDK2XPQGZX5QV/state.json","well_known_bundle":"https://pith.science/.well-known/pith/FUDGDQPHQFNNZUDK2XPQGZX5QV/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:FUDGDQPHQFNNZUDK2XPQGZX5QV","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"45e22d9a08b9a66d504abf6660c84482e1bfd9e96a30ca7125b98789ca00b1eb","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.MA","submitted_at":"2021-07-14T17:22:14Z","title_canon_sha256":"1bd9134067a1b4f9d901a3a4d50345083c7f1ed277c3da39758ed8de530fe39a"},"schema_version":"1.0","source":{"id":"2107.06857","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2107.06857","created_at":"2026-07-05T02:58:02Z"},{"alias_kind":"arxiv_version","alias_value":"2107.06857v1","created_at":"2026-07-05T02:58:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2107.06857","created_at":"2026-07-05T02:58:02Z"},{"alias_kind":"pith_short_12","alias_value":"FUDGDQPHQFNN","created_at":"2026-07-05T02:58:02Z"},{"alias_kind":"pith_short_16","alias_value":"FUDGDQPHQFNNZUDK","created_at":"2026-07-05T02:58:02Z"},{"alias_kind":"pith_short_8","alias_value":"FUDGDQPH","created_at":"2026-07-05T02:58:02Z"}],"graph_snapshots":[{"event_id":"sha256:7809425ba1abd6e0c6eb0d8403779a181a7dadd5a1bdded512b64fb79ad292f6","target":"graph","created_at":"2026-07-05T02:58:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2107.06857/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Existing evaluation suites for multi-agent reinforcement learning (MARL) do not assess generalization to novel situations as their primary objective (unlike supervised-learning benchmarks). Our contribution, Melting Pot, is a MARL evaluation suite that fills this gap, and uses reinforcement learning to reduce the human labor required to create novel test scenarios. This works because one agent's behavior constitutes (part of) another agent's environment. To demonstrate scalability, we have created over 80 unique test scenarios covering a broad range of research topics such as social dilemmas, ","authors_text":"Alexander Sasha Vezhnevets, Charles Beattie, Edgar Du\\'e\\~nez-Guzm\\'an, Igor Mordatch, Jayd Matyas, Joel Z. Leibo, John P. Agapiou, Peter Sunehag, Raphael Koster, Thore Graepel","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.MA","submitted_at":"2021-07-14T17:22:14Z","title":"Scalable Evaluation of Multi-Agent Reinforcement Learning with Melting Pot"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2107.06857","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:032af1a05604723105f9149745489eaa03ef7425655e78c5f19304b3ef986663","target":"record","created_at":"2026-07-05T02:58:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"45e22d9a08b9a66d504abf6660c84482e1bfd9e96a30ca7125b98789ca00b1eb","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.MA","submitted_at":"2021-07-14T17:22:14Z","title_canon_sha256":"1bd9134067a1b4f9d901a3a4d50345083c7f1ed277c3da39758ed8de530fe39a"},"schema_version":"1.0","source":{"id":"2107.06857","kind":"arxiv","version":1}},"canonical_sha256":"2d0661c1e7815adcd06ad5df0366fd854edae725e74a56196d3d9b7f9e211638","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"2d0661c1e7815adcd06ad5df0366fd854edae725e74a56196d3d9b7f9e211638","first_computed_at":"2026-07-05T02:58:02.148867Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T02:58:02.148867Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"vuRIyYx84M/Ze+k3yHcy90V8RzKBSbSWsugp/Ixzh6Sj00w3uEzpPvWUt5HPkZmYTsmN3uTk1cO0yOYxgemqCg==","signature_status":"signed_v1","signed_at":"2026-07-05T02:58:02.149278Z","signed_message":"canonical_sha256_bytes"},"source_id":"2107.06857","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:032af1a05604723105f9149745489eaa03ef7425655e78c5f19304b3ef986663","sha256:7809425ba1abd6e0c6eb0d8403779a181a7dadd5a1bdded512b64fb79ad292f6"],"state_sha256":"efc42a1f3925119fb94bbd05fc065c35bb3c2d5c5a74342667b6dc4a9d403bd5"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"NCYDGka8xxHPZO4C8HhyUNtqy5Hi017eI1lznyfIsot3D0moqNrBUy88MyQh6EchPg0QFtH8ABGLCPkd5asTCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-11T16:51:06.375908Z","bundle_sha256":"83bb31de56aa4c29e64ba01ba923f1c8b2957d77651e2867decfffce9e176d1a"}}