{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:TRSEBRYMIYMQHYZX7JJPOQVYNJ","short_pith_number":"pith:TRSEBRYM","canonical_record":{"source":{"id":"2403.08309","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-03-13T07:38:20Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"156507ac95dec82982431e16fe98ecb64e916610bcd117d7ca6951bdaa7502b6","abstract_canon_sha256":"ba710d0a4f1ce25650bbccf93aca4ea2af982a90ad8fac161561153d09db3099"},"schema_version":"1.0"},"canonical_sha256":"9c6440c70c461903e337fa52f742b86a4fe1e38575708668c88c0513ace13447","source":{"kind":"arxiv","id":"2403.08309","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2403.08309","created_at":"2026-07-05T07:56:04Z"},{"alias_kind":"arxiv_version","alias_value":"2403.08309v2","created_at":"2026-07-05T07:56:04Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2403.08309","created_at":"2026-07-05T07:56:04Z"},{"alias_kind":"pith_short_12","alias_value":"TRSEBRYMIYMQ","created_at":"2026-07-05T07:56:04Z"},{"alias_kind":"pith_short_16","alias_value":"TRSEBRYMIYMQHYZX","created_at":"2026-07-05T07:56:04Z"},{"alias_kind":"pith_short_8","alias_value":"TRSEBRYM","created_at":"2026-07-05T07:56:04Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:TRSEBRYMIYMQHYZX7JJPOQVYNJ","target":"record","payload":{"canonical_record":{"source":{"id":"2403.08309","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-03-13T07:38:20Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"156507ac95dec82982431e16fe98ecb64e916610bcd117d7ca6951bdaa7502b6","abstract_canon_sha256":"ba710d0a4f1ce25650bbccf93aca4ea2af982a90ad8fac161561153d09db3099"},"schema_version":"1.0"},"canonical_sha256":"9c6440c70c461903e337fa52f742b86a4fe1e38575708668c88c0513ace13447","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:56:04.483428Z","signature_b64":"WUbflEI2nYWEb39Equ0amC1x52YaHUJ/eiNp1xnEonTGCTwMLa/sNaQnknZcXAB7hxscdP7NJKsrDIFVeHOsBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"9c6440c70c461903e337fa52f742b86a4fe1e38575708668c88c0513ace13447","last_reissued_at":"2026-07-05T07:56:04.482964Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:56:04.482964Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2403.08309","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:56:04Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"dNgvp1hqpfLCThBdGhyFK2oWykF7SIWh0989FUjKVPqgabYn7xB2XmE4eL9lF7D8B/n9h54L7JBRz6kBR1afBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T09:30:08.672475Z"},"content_sha256":"b92e70788ca8cde7f9a3dbf77f46feaed2cb12a5c97ea9d7b9a5b7853e882a69","schema_version":"1.0","event_id":"sha256:b92e70788ca8cde7f9a3dbf77f46feaed2cb12a5c97ea9d7b9a5b7853e882a69"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:TRSEBRYMIYMQHYZX7JJPOQVYNJ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"HRLAIF: Improvements in Helpfulness and Harmlessness in Open-domain Reinforcement Learning From AI Feedback","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Ang Li, Daniell Wang, Jian Tang, Kaitong Yang, Liang Zhang, Peng Cao, Qiugen Xiao, Weidong Guo, Xiangyang Li, Xiaoyuan Chen, Xu Yu, Ying Shan, Yi Yuan, Yukang Gan, Zijie Zhao","submitted_at":"2024-03-13T07:38:20Z","abstract_excerpt":"Reinforcement Learning from AI Feedback (RLAIF) has the advantages of shorter annotation cycles and lower costs over Reinforcement Learning from Human Feedback (RLHF), making it highly efficient during the rapid strategy iteration periods of large language model (LLM) training. Using ChatGPT as a labeler to provide feedback on open-domain prompts in RLAIF training, we observe an increase in human evaluators' preference win ratio for model responses, but a decrease in evaluators' satisfaction rate. Analysis suggests that the decrease in satisfaction rate is mainly due to some responses becoming"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2403.08309","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2403.08309/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:56:04Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"GJZylT/waZGvfJbph30OBZEaoVtQxeHAwWGJcuSwAGar0b1nHY0/SzoTQ7j5GNaSkBUHZkRQMm+pyRc2CYcYAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T09:30:08.673418Z"},"content_sha256":"87289e3ce672bf6a38fc67093e9add6ae9af8accf66d578e9a48a10740055536","schema_version":"1.0","event_id":"sha256:87289e3ce672bf6a38fc67093e9add6ae9af8accf66d578e9a48a10740055536"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/TRSEBRYMIYMQHYZX7JJPOQVYNJ/bundle.json","state_url":"https://pith.science/pith/TRSEBRYMIYMQHYZX7JJPOQVYNJ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/TRSEBRYMIYMQHYZX7JJPOQVYNJ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-18T09:30:08Z","links":{"resolver":"https://pith.science/pith/TRSEBRYMIYMQHYZX7JJPOQVYNJ","bundle":"https://pith.science/pith/TRSEBRYMIYMQHYZX7JJPOQVYNJ/bundle.json","state":"https://pith.science/pith/TRSEBRYMIYMQHYZX7JJPOQVYNJ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/TRSEBRYMIYMQHYZX7JJPOQVYNJ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:TRSEBRYMIYMQHYZX7JJPOQVYNJ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"ba710d0a4f1ce25650bbccf93aca4ea2af982a90ad8fac161561153d09db3099","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-03-13T07:38:20Z","title_canon_sha256":"156507ac95dec82982431e16fe98ecb64e916610bcd117d7ca6951bdaa7502b6"},"schema_version":"1.0","source":{"id":"2403.08309","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2403.08309","created_at":"2026-07-05T07:56:04Z"},{"alias_kind":"arxiv_version","alias_value":"2403.08309v2","created_at":"2026-07-05T07:56:04Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2403.08309","created_at":"2026-07-05T07:56:04Z"},{"alias_kind":"pith_short_12","alias_value":"TRSEBRYMIYMQ","created_at":"2026-07-05T07:56:04Z"},{"alias_kind":"pith_short_16","alias_value":"TRSEBRYMIYMQHYZX","created_at":"2026-07-05T07:56:04Z"},{"alias_kind":"pith_short_8","alias_value":"TRSEBRYM","created_at":"2026-07-05T07:56:04Z"}],"graph_snapshots":[{"event_id":"sha256:87289e3ce672bf6a38fc67093e9add6ae9af8accf66d578e9a48a10740055536","target":"graph","created_at":"2026-07-05T07:56:04Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2403.08309/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reinforcement Learning from AI Feedback (RLAIF) has the advantages of shorter annotation cycles and lower costs over Reinforcement Learning from Human Feedback (RLHF), making it highly efficient during the rapid strategy iteration periods of large language model (LLM) training. Using ChatGPT as a labeler to provide feedback on open-domain prompts in RLAIF training, we observe an increase in human evaluators' preference win ratio for model responses, but a decrease in evaluators' satisfaction rate. Analysis suggests that the decrease in satisfaction rate is mainly due to some responses becoming","authors_text":"Ang Li, Daniell Wang, Jian Tang, Kaitong Yang, Liang Zhang, Peng Cao, Qiugen Xiao, Weidong Guo, Xiangyang Li, Xiaoyuan Chen, Xu Yu, Ying Shan, Yi Yuan, Yukang Gan, Zijie Zhao","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-03-13T07:38:20Z","title":"HRLAIF: Improvements in Helpfulness and Harmlessness in Open-domain Reinforcement Learning From AI Feedback"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2403.08309","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b92e70788ca8cde7f9a3dbf77f46feaed2cb12a5c97ea9d7b9a5b7853e882a69","target":"record","created_at":"2026-07-05T07:56:04Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"ba710d0a4f1ce25650bbccf93aca4ea2af982a90ad8fac161561153d09db3099","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-03-13T07:38:20Z","title_canon_sha256":"156507ac95dec82982431e16fe98ecb64e916610bcd117d7ca6951bdaa7502b6"},"schema_version":"1.0","source":{"id":"2403.08309","kind":"arxiv","version":2}},"canonical_sha256":"9c6440c70c461903e337fa52f742b86a4fe1e38575708668c88c0513ace13447","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"9c6440c70c461903e337fa52f742b86a4fe1e38575708668c88c0513ace13447","first_computed_at":"2026-07-05T07:56:04.482964Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:56:04.482964Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"WUbflEI2nYWEb39Equ0amC1x52YaHUJ/eiNp1xnEonTGCTwMLa/sNaQnknZcXAB7hxscdP7NJKsrDIFVeHOsBg==","signature_status":"signed_v1","signed_at":"2026-07-05T07:56:04.483428Z","signed_message":"canonical_sha256_bytes"},"source_id":"2403.08309","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b92e70788ca8cde7f9a3dbf77f46feaed2cb12a5c97ea9d7b9a5b7853e882a69","sha256:87289e3ce672bf6a38fc67093e9add6ae9af8accf66d578e9a48a10740055536"],"state_sha256":"530f2833573b21d4b2ad7e4fe288a44d541b596f7b2a923ce5f2da4b3ee5f7f0"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"aepUKJInK96LKqq0QbmpQZSI4Cwy54PYvJkNq7G+EECuVczCjlvJz+8lVw6kO6qfepjRGgsymm7UVaUSaRTbDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-18T09:30:08.684446Z","bundle_sha256":"8876efc230492d3e34d8dcb49300de6874dd3ac8c8e245e8fdb015353b42f508"}}