{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2021:5I2EVWQPJNUWWVLF6NNUUYP4EV","short_pith_number":"pith:5I2EVWQP","schema_version":"1.0","canonical_sha256":"ea344ada0f4b696b5565f35b4a61fc256b8a4384142aeb42d39c9c1a9ce1c219","source":{"kind":"arxiv","id":"2111.00876","version":2},"attestation_state":"computed","paper":{"title":"On the Expressivity of Markov Reward","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Anna Harutyunyan, David Abel, Doina Precup, Mark K. Ho, Michael L. Littman, Satinder Singh, Will Dabney","submitted_at":"2021-11-01T12:12:16Z","abstract_excerpt":"Reward is the driving force for reinforcement-learning agents. This paper is dedicated to understanding the expressivity of reward as a way to capture tasks that we would want an agent to perform. We frame this study around three new abstract notions of \"task\" that might be desirable: (1) a set of acceptable behaviors, (2) a partial ordering over behaviors, or (3) a partial ordering over trajectories. Our main results prove that while reward can express many of these tasks, there exist instances of each task type that no Markov reward function can capture. We then provide a set of polynomial-t"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2111.00876","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2021-11-01T12:12:16Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"aa5bdd9d1ccf7f3c343e2ced15b164742395f8752ff357376cc7e679cd1b25f3","abstract_canon_sha256":"38b308f8c60fcdbfbba0cb038d13f81d49d918af723b8798ad75b0148e8e566e"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T03:49:23.859707Z","signature_b64":"G87fsILx25ggNi2jbN/CmPn+dvcfl9rVRi9yBrx32PFmA2rc+6nuFchxz7PrTxWAi3MBnMHvBqmf8q/q0hcAAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"ea344ada0f4b696b5565f35b4a61fc256b8a4384142aeb42d39c9c1a9ce1c219","last_reissued_at":"2026-07-05T03:49:23.859250Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T03:49:23.859250Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"On the Expressivity of Markov Reward","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Anna Harutyunyan, David Abel, Doina Precup, Mark K. Ho, Michael L. Littman, Satinder Singh, Will Dabney","submitted_at":"2021-11-01T12:12:16Z","abstract_excerpt":"Reward is the driving force for reinforcement-learning agents. This paper is dedicated to understanding the expressivity of reward as a way to capture tasks that we would want an agent to perform. We frame this study around three new abstract notions of \"task\" that might be desirable: (1) a set of acceptable behaviors, (2) a partial ordering over behaviors, or (3) a partial ordering over trajectories. Our main results prove that while reward can express many of these tasks, there exist instances of each task type that no Markov reward function can capture. We then provide a set of polynomial-t"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2111.00876","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2111.00876/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2111.00876","created_at":"2026-07-05T03:49:23.859306+00:00"},{"alias_kind":"arxiv_version","alias_value":"2111.00876v2","created_at":"2026-07-05T03:49:23.859306+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2111.00876","created_at":"2026-07-05T03:49:23.859306+00:00"},{"alias_kind":"pith_short_12","alias_value":"5I2EVWQPJNUW","created_at":"2026-07-05T03:49:23.859306+00:00"},{"alias_kind":"pith_short_16","alias_value":"5I2EVWQPJNUWWVLF","created_at":"2026-07-05T03:49:23.859306+00:00"},{"alias_kind":"pith_short_8","alias_value":"5I2EVWQP","created_at":"2026-07-05T03:49:23.859306+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.19328","citing_title":"UBP2: Uncertainty-Balanced Preference Planning for Efficient Preference-based Reinforcement Learning","ref_index":1,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/5I2EVWQPJNUWWVLF6NNUUYP4EV","json":"https://pith.science/pith/5I2EVWQPJNUWWVLF6NNUUYP4EV.json","graph_json":"https://pith.science/api/pith-number/5I2EVWQPJNUWWVLF6NNUUYP4EV/graph.json","events_json":"https://pith.science/api/pith-number/5I2EVWQPJNUWWVLF6NNUUYP4EV/events.json","paper":"https://pith.science/paper/5I2EVWQP"},"agent_actions":{"view_html":"https://pith.science/pith/5I2EVWQPJNUWWVLF6NNUUYP4EV","download_json":"https://pith.science/pith/5I2EVWQPJNUWWVLF6NNUUYP4EV.json","view_paper":"https://pith.science/paper/5I2EVWQP","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2111.00876&json=true","fetch_graph":"https://pith.science/api/pith-number/5I2EVWQPJNUWWVLF6NNUUYP4EV/graph.json","fetch_events":"https://pith.science/api/pith-number/5I2EVWQPJNUWWVLF6NNUUYP4EV/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/5I2EVWQPJNUWWVLF6NNUUYP4EV/action/timestamp_anchor","attest_storage":"https://pith.science/pith/5I2EVWQPJNUWWVLF6NNUUYP4EV/action/storage_attestation","attest_author":"https://pith.science/pith/5I2EVWQPJNUWWVLF6NNUUYP4EV/action/author_attestation","sign_citation":"https://pith.science/pith/5I2EVWQPJNUWWVLF6NNUUYP4EV/action/citation_signature","submit_replication":"https://pith.science/pith/5I2EVWQPJNUWWVLF6NNUUYP4EV/action/replication_record"}},"created_at":"2026-07-05T03:49:23.859306+00:00","updated_at":"2026-07-05T03:49:23.859306+00:00"}