{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:ANAIA3O77HUZJWBS3AFVBDYWQY","short_pith_number":"pith:ANAIA3O7","canonical_record":{"source":{"id":"2211.11602","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2022-11-21T16:00:31Z","cross_cats_sorted":["cs.HC","cs.MA"],"title_canon_sha256":"a2e63611bf1fac3b2f809ad328f43b39aa0206dd58abced3cb8bbdcc69d223ab","abstract_canon_sha256":"9dd4019ea8d2135de9b27d523e5c8e62cada2c4e6f38b55b488e547374974973"},"schema_version":"1.0"},"canonical_sha256":"0340806ddff9e994d832d80b508f16860f4797f36ad2d28caccf1d9c47e6681a","source":{"kind":"arxiv","id":"2211.11602","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2211.11602","created_at":"2026-07-05T05:17:55Z"},{"alias_kind":"arxiv_version","alias_value":"2211.11602v1","created_at":"2026-07-05T05:17:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2211.11602","created_at":"2026-07-05T05:17:55Z"},{"alias_kind":"pith_short_12","alias_value":"ANAIA3O77HUZ","created_at":"2026-07-05T05:17:55Z"},{"alias_kind":"pith_short_16","alias_value":"ANAIA3O77HUZJWBS","created_at":"2026-07-05T05:17:55Z"},{"alias_kind":"pith_short_8","alias_value":"ANAIA3O7","created_at":"2026-07-05T05:17:55Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:ANAIA3O77HUZJWBS3AFVBDYWQY","target":"record","payload":{"canonical_record":{"source":{"id":"2211.11602","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2022-11-21T16:00:31Z","cross_cats_sorted":["cs.HC","cs.MA"],"title_canon_sha256":"a2e63611bf1fac3b2f809ad328f43b39aa0206dd58abced3cb8bbdcc69d223ab","abstract_canon_sha256":"9dd4019ea8d2135de9b27d523e5c8e62cada2c4e6f38b55b488e547374974973"},"schema_version":"1.0"},"canonical_sha256":"0340806ddff9e994d832d80b508f16860f4797f36ad2d28caccf1d9c47e6681a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:17:55.851250Z","signature_b64":"0ItWNXDbE5KI9V2Ant/h3lCIGSzQ+ZQZuW4sbPs08wVpm+K0/gZsS5cegXJ7qHhB+AhBNv20JdYtYD5aX9qVCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0340806ddff9e994d832d80b508f16860f4797f36ad2d28caccf1d9c47e6681a","last_reissued_at":"2026-07-05T05:17:55.850695Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:17:55.850695Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2211.11602","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:17:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"j7wfYRc5CdM7eOurLoK8d8wBdQ4VnF/ODJmB2bZTV7M4SSq+SltOYNUJ/0TxylClHXGHhyLsZCRPKooU6c2vAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T06:59:48.178663Z"},"content_sha256":"b1fae9c1b016742598e3679ca1ab6d4f312017e7b73c9b17e088c2c31e238389","schema_version":"1.0","event_id":"sha256:b1fae9c1b016742598e3679ca1ab6d4f312017e7b73c9b17e088c2c31e238389"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:ANAIA3O77HUZJWBS3AFVBDYWQY","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Improving Multimodal Interactive Agents with Reinforcement Learning from Human Feedback","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.HC","cs.MA"],"primary_cat":"cs.LG","authors_text":"Adam Santoro, Alden Hung, Alex Goldin, Alistair Muldal, Arun Ahuja, Chen Yan, Federico Carnevale, George Powell, Greg Wayne, Guy Scully, Jessica Landon, Jirka Lhotka, Josh Abramson, Nathaniel Wong, Petko Georgiev, Rui Zhu, Sanjana Srivastava, Tamara von Glehn, Timothy Lillicrap","submitted_at":"2022-11-21T16:00:31Z","abstract_excerpt":"An important goal in artificial intelligence is to create agents that can both interact naturally with humans and learn from their feedback. Here we demonstrate how to use reinforcement learning from human feedback (RLHF) to improve upon simulated, embodied agents trained to a base level of competency with imitation learning. First, we collected data of humans interacting with agents in a simulated 3D world. We then asked annotators to record moments where they believed that agents either progressed toward or regressed from their human-instructed goal. Using this annotation data we leveraged a"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2211.11602","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2211.11602/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:17:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"/kZ6SN6ZBZMDP8zyOZFzqqIrPCqCTJU239Ejyj+FW2UeF1gH/ZQsCwDcxqDSYFZOGE/zm3+Fw3h+F+sKTQ3SBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T06:59:48.179208Z"},"content_sha256":"1d91174ec8cf0e6affe240ae57abb4d4e7b16aecc28e3362ab859027948ec0b4","schema_version":"1.0","event_id":"sha256:1d91174ec8cf0e6affe240ae57abb4d4e7b16aecc28e3362ab859027948ec0b4"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/ANAIA3O77HUZJWBS3AFVBDYWQY/bundle.json","state_url":"https://pith.science/pith/ANAIA3O77HUZJWBS3AFVBDYWQY/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/ANAIA3O77HUZJWBS3AFVBDYWQY/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T06:59:48Z","links":{"resolver":"https://pith.science/pith/ANAIA3O77HUZJWBS3AFVBDYWQY","bundle":"https://pith.science/pith/ANAIA3O77HUZJWBS3AFVBDYWQY/bundle.json","state":"https://pith.science/pith/ANAIA3O77HUZJWBS3AFVBDYWQY/state.json","well_known_bundle":"https://pith.science/.well-known/pith/ANAIA3O77HUZJWBS3AFVBDYWQY/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:ANAIA3O77HUZJWBS3AFVBDYWQY","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"9dd4019ea8d2135de9b27d523e5c8e62cada2c4e6f38b55b488e547374974973","cross_cats_sorted":["cs.HC","cs.MA"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2022-11-21T16:00:31Z","title_canon_sha256":"a2e63611bf1fac3b2f809ad328f43b39aa0206dd58abced3cb8bbdcc69d223ab"},"schema_version":"1.0","source":{"id":"2211.11602","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2211.11602","created_at":"2026-07-05T05:17:55Z"},{"alias_kind":"arxiv_version","alias_value":"2211.11602v1","created_at":"2026-07-05T05:17:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2211.11602","created_at":"2026-07-05T05:17:55Z"},{"alias_kind":"pith_short_12","alias_value":"ANAIA3O77HUZ","created_at":"2026-07-05T05:17:55Z"},{"alias_kind":"pith_short_16","alias_value":"ANAIA3O77HUZJWBS","created_at":"2026-07-05T05:17:55Z"},{"alias_kind":"pith_short_8","alias_value":"ANAIA3O7","created_at":"2026-07-05T05:17:55Z"}],"graph_snapshots":[{"event_id":"sha256:1d91174ec8cf0e6affe240ae57abb4d4e7b16aecc28e3362ab859027948ec0b4","target":"graph","created_at":"2026-07-05T05:17:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2211.11602/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"An important goal in artificial intelligence is to create agents that can both interact naturally with humans and learn from their feedback. Here we demonstrate how to use reinforcement learning from human feedback (RLHF) to improve upon simulated, embodied agents trained to a base level of competency with imitation learning. First, we collected data of humans interacting with agents in a simulated 3D world. We then asked annotators to record moments where they believed that agents either progressed toward or regressed from their human-instructed goal. Using this annotation data we leveraged a","authors_text":"Adam Santoro, Alden Hung, Alex Goldin, Alistair Muldal, Arun Ahuja, Chen Yan, Federico Carnevale, George Powell, Greg Wayne, Guy Scully, Jessica Landon, Jirka Lhotka, Josh Abramson, Nathaniel Wong, Petko Georgiev, Rui Zhu, Sanjana Srivastava, Tamara von Glehn, Timothy Lillicrap","cross_cats":["cs.HC","cs.MA"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2022-11-21T16:00:31Z","title":"Improving Multimodal Interactive Agents with Reinforcement Learning from Human Feedback"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2211.11602","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b1fae9c1b016742598e3679ca1ab6d4f312017e7b73c9b17e088c2c31e238389","target":"record","created_at":"2026-07-05T05:17:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"9dd4019ea8d2135de9b27d523e5c8e62cada2c4e6f38b55b488e547374974973","cross_cats_sorted":["cs.HC","cs.MA"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2022-11-21T16:00:31Z","title_canon_sha256":"a2e63611bf1fac3b2f809ad328f43b39aa0206dd58abced3cb8bbdcc69d223ab"},"schema_version":"1.0","source":{"id":"2211.11602","kind":"arxiv","version":1}},"canonical_sha256":"0340806ddff9e994d832d80b508f16860f4797f36ad2d28caccf1d9c47e6681a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0340806ddff9e994d832d80b508f16860f4797f36ad2d28caccf1d9c47e6681a","first_computed_at":"2026-07-05T05:17:55.850695Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:17:55.850695Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"0ItWNXDbE5KI9V2Ant/h3lCIGSzQ+ZQZuW4sbPs08wVpm+K0/gZsS5cegXJ7qHhB+AhBNv20JdYtYD5aX9qVCw==","signature_status":"signed_v1","signed_at":"2026-07-05T05:17:55.851250Z","signed_message":"canonical_sha256_bytes"},"source_id":"2211.11602","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b1fae9c1b016742598e3679ca1ab6d4f312017e7b73c9b17e088c2c31e238389","sha256:1d91174ec8cf0e6affe240ae57abb4d4e7b16aecc28e3362ab859027948ec0b4"],"state_sha256":"7a63d91155bbf588983854bf5d85ba6bf831c6e8480e2ed782550cdcdabb2096"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"rbb32ky485ArfJnw6baj7MMtk5vhEFyjArSSix1r/l8AIGIi1lgfU2O0ninTNFNNbZPUiNVlmFW+CjQR0lQkCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T06:59:48.185500Z","bundle_sha256":"74fed194b2337cb597c23f6ca2e9052077184d4046fc01dc982e95ecffc3798d"}}