{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:3KSMZDLUEIBXOJCKD4TYISRGEZ","short_pith_number":"pith:3KSMZDLU","canonical_record":{"source":{"id":"2109.07054","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2021-09-15T02:29:18Z","cross_cats_sorted":["cs.AI","cs.DS","cs.HC"],"title_canon_sha256":"95da650ecc9742a090e56aa89010ef7868753cf389b171c0dd0ecf7834659762","abstract_canon_sha256":"11b9fef7ad24c7d6e9d514b58cb5676318388bfc7a2cc4273dd14eca538fd960"},"schema_version":"1.0"},"canonical_sha256":"daa4cc8d74220377244a1f27844a262651fb275f3a2e353b3fab90d889023685","source":{"kind":"arxiv","id":"2109.07054","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2109.07054","created_at":"2026-07-05T03:14:38Z"},{"alias_kind":"arxiv_version","alias_value":"2109.07054v1","created_at":"2026-07-05T03:14:38Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2109.07054","created_at":"2026-07-05T03:14:38Z"},{"alias_kind":"pith_short_12","alias_value":"3KSMZDLUEIBX","created_at":"2026-07-05T03:14:38Z"},{"alias_kind":"pith_short_16","alias_value":"3KSMZDLUEIBXOJCK","created_at":"2026-07-05T03:14:38Z"},{"alias_kind":"pith_short_8","alias_value":"3KSMZDLU","created_at":"2026-07-05T03:14:38Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:3KSMZDLUEIBXOJCKD4TYISRGEZ","target":"record","payload":{"canonical_record":{"source":{"id":"2109.07054","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2021-09-15T02:29:18Z","cross_cats_sorted":["cs.AI","cs.DS","cs.HC"],"title_canon_sha256":"95da650ecc9742a090e56aa89010ef7868753cf389b171c0dd0ecf7834659762","abstract_canon_sha256":"11b9fef7ad24c7d6e9d514b58cb5676318388bfc7a2cc4273dd14eca538fd960"},"schema_version":"1.0"},"canonical_sha256":"daa4cc8d74220377244a1f27844a262651fb275f3a2e353b3fab90d889023685","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T03:14:38.417384Z","signature_b64":"86KZ74h8lKGq0HFpC/K0dFtyQfZDg+qKSVTYKIMDr8rWHNTAkSVMoozIKw93FHzZ6GRMSjenC7nB0Qn6U1ByAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"daa4cc8d74220377244a1f27844a262651fb275f3a2e353b3fab90d889023685","last_reissued_at":"2026-07-05T03:14:38.416944Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T03:14:38.416944Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2109.07054","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T03:14:38Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"oFLOcMCS5MA7uFf/O6EpAg9+o+gDjQISKnxyb7Oi5SbCCheiQZY9KGKGqdeQoFlzvCj/od79GjXnwa5jyubmBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T03:56:38.573486Z"},"content_sha256":"53ab2fc4dbbdb5c5d8dc90b09329c5704f8d6b305f4a22360c87c0291b23a941","schema_version":"1.0","event_id":"sha256:53ab2fc4dbbdb5c5d8dc90b09329c5704f8d6b305f4a22360c87c0291b23a941"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:3KSMZDLUEIBXOJCKD4TYISRGEZ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Convergence of a Human-in-the-Loop Policy-Gradient Algorithm With Eligibility Trace Under Reward, Policy, and Advantage Feedback","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.DS","cs.HC"],"primary_cat":"cs.LG","authors_text":"David Halpern, Ishaan Shah, Kavosh Asadi, Michael L. Littman","submitted_at":"2021-09-15T02:29:18Z","abstract_excerpt":"Fluid human-agent communication is essential for the future of human-in-the-loop reinforcement learning. An agent must respond appropriately to feedback from its human trainer even before they have significant experience working together. Therefore, it is important that learning agents respond well to various feedback schemes human trainers are likely to provide. This work analyzes the COnvergent Actor-Critic by Humans (COACH) algorithm under three different types of feedback-policy feedback, reward feedback, and advantage feedback. For these three feedback types, we find that COACH can behave"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2109.07054","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2109.07054/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T03:14:38Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"qck8KRgFNUfUzrgXvEpApTyQtN/PwDg+PVupOGOGTiTjAKs23xgijMR7sS4sUMIDuHdYZdSYdrS9MyWKU7kiAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T03:56:38.573858Z"},"content_sha256":"4999f597cfa2d1f88892443ef278447d73dc3c5df2796d4eee532fde05d56b9b","schema_version":"1.0","event_id":"sha256:4999f597cfa2d1f88892443ef278447d73dc3c5df2796d4eee532fde05d56b9b"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/3KSMZDLUEIBXOJCKD4TYISRGEZ/bundle.json","state_url":"https://pith.science/pith/3KSMZDLUEIBXOJCKD4TYISRGEZ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/3KSMZDLUEIBXOJCKD4TYISRGEZ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-09T03:56:38Z","links":{"resolver":"https://pith.science/pith/3KSMZDLUEIBXOJCKD4TYISRGEZ","bundle":"https://pith.science/pith/3KSMZDLUEIBXOJCKD4TYISRGEZ/bundle.json","state":"https://pith.science/pith/3KSMZDLUEIBXOJCKD4TYISRGEZ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/3KSMZDLUEIBXOJCKD4TYISRGEZ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:3KSMZDLUEIBXOJCKD4TYISRGEZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"11b9fef7ad24c7d6e9d514b58cb5676318388bfc7a2cc4273dd14eca538fd960","cross_cats_sorted":["cs.AI","cs.DS","cs.HC"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2021-09-15T02:29:18Z","title_canon_sha256":"95da650ecc9742a090e56aa89010ef7868753cf389b171c0dd0ecf7834659762"},"schema_version":"1.0","source":{"id":"2109.07054","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2109.07054","created_at":"2026-07-05T03:14:38Z"},{"alias_kind":"arxiv_version","alias_value":"2109.07054v1","created_at":"2026-07-05T03:14:38Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2109.07054","created_at":"2026-07-05T03:14:38Z"},{"alias_kind":"pith_short_12","alias_value":"3KSMZDLUEIBX","created_at":"2026-07-05T03:14:38Z"},{"alias_kind":"pith_short_16","alias_value":"3KSMZDLUEIBXOJCK","created_at":"2026-07-05T03:14:38Z"},{"alias_kind":"pith_short_8","alias_value":"3KSMZDLU","created_at":"2026-07-05T03:14:38Z"}],"graph_snapshots":[{"event_id":"sha256:4999f597cfa2d1f88892443ef278447d73dc3c5df2796d4eee532fde05d56b9b","target":"graph","created_at":"2026-07-05T03:14:38Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2109.07054/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Fluid human-agent communication is essential for the future of human-in-the-loop reinforcement learning. An agent must respond appropriately to feedback from its human trainer even before they have significant experience working together. Therefore, it is important that learning agents respond well to various feedback schemes human trainers are likely to provide. This work analyzes the COnvergent Actor-Critic by Humans (COACH) algorithm under three different types of feedback-policy feedback, reward feedback, and advantage feedback. For these three feedback types, we find that COACH can behave","authors_text":"David Halpern, Ishaan Shah, Kavosh Asadi, Michael L. Littman","cross_cats":["cs.AI","cs.DS","cs.HC"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2021-09-15T02:29:18Z","title":"Convergence of a Human-in-the-Loop Policy-Gradient Algorithm With Eligibility Trace Under Reward, Policy, and Advantage Feedback"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2109.07054","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:53ab2fc4dbbdb5c5d8dc90b09329c5704f8d6b305f4a22360c87c0291b23a941","target":"record","created_at":"2026-07-05T03:14:38Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"11b9fef7ad24c7d6e9d514b58cb5676318388bfc7a2cc4273dd14eca538fd960","cross_cats_sorted":["cs.AI","cs.DS","cs.HC"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2021-09-15T02:29:18Z","title_canon_sha256":"95da650ecc9742a090e56aa89010ef7868753cf389b171c0dd0ecf7834659762"},"schema_version":"1.0","source":{"id":"2109.07054","kind":"arxiv","version":1}},"canonical_sha256":"daa4cc8d74220377244a1f27844a262651fb275f3a2e353b3fab90d889023685","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"daa4cc8d74220377244a1f27844a262651fb275f3a2e353b3fab90d889023685","first_computed_at":"2026-07-05T03:14:38.416944Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T03:14:38.416944Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"86KZ74h8lKGq0HFpC/K0dFtyQfZDg+qKSVTYKIMDr8rWHNTAkSVMoozIKw93FHzZ6GRMSjenC7nB0Qn6U1ByAw==","signature_status":"signed_v1","signed_at":"2026-07-05T03:14:38.417384Z","signed_message":"canonical_sha256_bytes"},"source_id":"2109.07054","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:53ab2fc4dbbdb5c5d8dc90b09329c5704f8d6b305f4a22360c87c0291b23a941","sha256:4999f597cfa2d1f88892443ef278447d73dc3c5df2796d4eee532fde05d56b9b"],"state_sha256":"b8e956d9bc179c8ae92f382a2121cae637a8e9374794603ee7dbf5912bc8fbeb"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"EvZFxzKqKfCthoJNArSwW6KzgcsyCuqd0+m+VHDPYuaGjyMHU5K3QUHQcspPlOM4IuGvzon8nwZG+shkPfq9Dg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-09T03:56:38.575756Z","bundle_sha256":"d22bd19fd7afce1953a199ac283e74a7ffed34a8e084104e921f14652b4de0d4"}}