{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2021:5WYF672CN4OZBDDS3JFOPH2BPP","short_pith_number":"pith:5WYF672C","schema_version":"1.0","canonical_sha256":"edb05f7f426f1d908c72da4ae79f417bde71825bc444d1580544c64d99214cc0","source":{"kind":"arxiv","id":"2106.00661","version":4},"attestation_state":"computed","paper":{"title":"Reward is enough for convex MDPs","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG","stat.ML"],"primary_cat":"cs.AI","authors_text":"Brendan O'Donoghue, Guillaume Desjardins, Satinder Singh, Tom Zahavy","submitted_at":"2021-06-01T17:46:25Z","abstract_excerpt":"Maximising a cumulative reward function that is Markov and stationary, i.e., defined over state-action pairs and independent of time, is sufficient to capture many kinds of goals in a Markov decision process (MDP). However, not all goals can be captured in this manner. In this paper we study convex MDPs in which goals are expressed as convex functions of the stationary distribution and show that they cannot be formulated using stationary reward functions. Convex MDPs generalize the standard reinforcement learning (RL) problem formulation to a larger framework that includes many supervised and "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2106.00661","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2021-06-01T17:46:25Z","cross_cats_sorted":["cs.LG","stat.ML"],"title_canon_sha256":"fbb48a49ac2eb9bbee3b24826632ae4b40de6c8fa943eaae32d055416ac1bc22","abstract_canon_sha256":"c2cae70be1949777cca97948941833871f1abe1e8473aa16c5301b06cfa67cec"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:16:44.700304Z","signature_b64":"VnIlrsc7umFdQefvkeIR19MAwQCNbw+OJisc9lt7SqVLd3CamMsqzEP01/XwkwGCkufZsIR5Qj7bwUAk0G0cCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"edb05f7f426f1d908c72da4ae79f417bde71825bc444d1580544c64d99214cc0","last_reissued_at":"2026-07-05T06:16:44.699829Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:16:44.699829Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Reward is enough for convex MDPs","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG","stat.ML"],"primary_cat":"cs.AI","authors_text":"Brendan O'Donoghue, Guillaume Desjardins, Satinder Singh, Tom Zahavy","submitted_at":"2021-06-01T17:46:25Z","abstract_excerpt":"Maximising a cumulative reward function that is Markov and stationary, i.e., defined over state-action pairs and independent of time, is sufficient to capture many kinds of goals in a Markov decision process (MDP). However, not all goals can be captured in this manner. In this paper we study convex MDPs in which goals are expressed as convex functions of the stationary distribution and show that they cannot be formulated using stationary reward functions. Convex MDPs generalize the standard reinforcement learning (RL) problem formulation to a larger framework that includes many supervised and "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2106.00661","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2106.00661/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2106.00661","created_at":"2026-07-05T06:16:44.699888+00:00"},{"alias_kind":"arxiv_version","alias_value":"2106.00661v4","created_at":"2026-07-05T06:16:44.699888+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2106.00661","created_at":"2026-07-05T06:16:44.699888+00:00"},{"alias_kind":"pith_short_12","alias_value":"5WYF672CN4OZ","created_at":"2026-07-05T06:16:44.699888+00:00"},{"alias_kind":"pith_short_16","alias_value":"5WYF672CN4OZBDDS","created_at":"2026-07-05T06:16:44.699888+00:00"},{"alias_kind":"pith_short_8","alias_value":"5WYF672C","created_at":"2026-07-05T06:16:44.699888+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/5WYF672CN4OZBDDS3JFOPH2BPP","json":"https://pith.science/pith/5WYF672CN4OZBDDS3JFOPH2BPP.json","graph_json":"https://pith.science/api/pith-number/5WYF672CN4OZBDDS3JFOPH2BPP/graph.json","events_json":"https://pith.science/api/pith-number/5WYF672CN4OZBDDS3JFOPH2BPP/events.json","paper":"https://pith.science/paper/5WYF672C"},"agent_actions":{"view_html":"https://pith.science/pith/5WYF672CN4OZBDDS3JFOPH2BPP","download_json":"https://pith.science/pith/5WYF672CN4OZBDDS3JFOPH2BPP.json","view_paper":"https://pith.science/paper/5WYF672C","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2106.00661&json=true","fetch_graph":"https://pith.science/api/pith-number/5WYF672CN4OZBDDS3JFOPH2BPP/graph.json","fetch_events":"https://pith.science/api/pith-number/5WYF672CN4OZBDDS3JFOPH2BPP/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/5WYF672CN4OZBDDS3JFOPH2BPP/action/timestamp_anchor","attest_storage":"https://pith.science/pith/5WYF672CN4OZBDDS3JFOPH2BPP/action/storage_attestation","attest_author":"https://pith.science/pith/5WYF672CN4OZBDDS3JFOPH2BPP/action/author_attestation","sign_citation":"https://pith.science/pith/5WYF672CN4OZBDDS3JFOPH2BPP/action/citation_signature","submit_replication":"https://pith.science/pith/5WYF672CN4OZBDDS3JFOPH2BPP/action/replication_record"}},"created_at":"2026-07-05T06:16:44.699888+00:00","updated_at":"2026-07-05T06:16:44.699888+00:00"}