{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:7WXFBWRLDWOTJNB4DNI5NMHC2B","short_pith_number":"pith:7WXFBWRL","canonical_record":{"source":{"id":"2405.17618","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-05-27T19:28:33Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"610e45d2f405f7af9118997736b70725d7063f04088cda3e1e63b9f811457c2e","abstract_canon_sha256":"c844fd83ab4970458a428c7f7819ac5965411473e5d872de679fd49577de3e41"},"schema_version":"1.0"},"canonical_sha256":"fdae50da2b1d9d34b43c1b51d6b0e2d04adf37a804453a535329fd2be8ccff49","source":{"kind":"arxiv","id":"2405.17618","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.17618","created_at":"2026-07-05T11:25:01Z"},{"alias_kind":"arxiv_version","alias_value":"2405.17618v3","created_at":"2026-07-05T11:25:01Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.17618","created_at":"2026-07-05T11:25:01Z"},{"alias_kind":"pith_short_12","alias_value":"7WXFBWRLDWOT","created_at":"2026-07-05T11:25:01Z"},{"alias_kind":"pith_short_16","alias_value":"7WXFBWRLDWOTJNB4","created_at":"2026-07-05T11:25:01Z"},{"alias_kind":"pith_short_8","alias_value":"7WXFBWRL","created_at":"2026-07-05T11:25:01Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:7WXFBWRLDWOTJNB4DNI5NMHC2B","target":"record","payload":{"canonical_record":{"source":{"id":"2405.17618","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-05-27T19:28:33Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"610e45d2f405f7af9118997736b70725d7063f04088cda3e1e63b9f811457c2e","abstract_canon_sha256":"c844fd83ab4970458a428c7f7819ac5965411473e5d872de679fd49577de3e41"},"schema_version":"1.0"},"canonical_sha256":"fdae50da2b1d9d34b43c1b51d6b0e2d04adf37a804453a535329fd2be8ccff49","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:25:01.666809Z","signature_b64":"drtNQmdPl0UkxwV2jEe4FUf3I+TF8JxLeOj5bVw31hiBTqMRs7F6LBAyMxsDNrI2Qx0bmpvUo2ra3IZKlpleCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"fdae50da2b1d9d34b43c1b51d6b0e2d04adf37a804453a535329fd2be8ccff49","last_reissued_at":"2026-07-05T11:25:01.666335Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:25:01.666335Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2405.17618","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:25:01Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"l7WxLpspK+TSUvNOj2H9/k088/JwvtFbkRkSbX7qEqRggqnG9ZnxWaR/mlJeXe/u830XpU+mg2ZUHkUVnf4GDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T18:30:26.638211Z"},"content_sha256":"799693d21a803c581a64ee4cc90162835767a152f783ff0f714c1810f1de7c24","schema_version":"1.0","event_id":"sha256:799693d21a803c581a64ee4cc90162835767a152f783ff0f714c1810f1de7c24"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:7WXFBWRLDWOTJNB4DNI5NMHC2B","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Symmetric Reinforcement Learning Loss for Robust Learning on Diverse Tasks and Model Scales","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Andrew Perrault, Ju-Seung Byun","submitted_at":"2024-05-27T19:28:33Z","abstract_excerpt":"Reinforcement learning (RL) training is inherently unstable due to factors such as moving targets and high gradient variance. Reinforcement Learning from Human Feedback (RLHF) and Reinforcement Learning from AI Feedback (RLAIF) can introduce additional difficulty. Differing preferences can complicate the alignment process, and prediction errors in a trained reward model can become more severe as the LLM generates unseen outputs. To enhance training robustness, RL has adopted techniques from supervised learning, such as ensembles and layer normalization. In this work, we improve the stability o"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.17618","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.17618/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:25:01Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"E6r/XQXQAny7HwyyTCh3SECh4OY0lALwV1fvmY6qS4uZyh4mnrucRmkujryZnDdR+7nGHKsA0aeJMbbaJThAAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T18:30:26.639315Z"},"content_sha256":"ae57f805544d784c35bdcbee96db22f21a0ab5874408063fa14deea29ae4951f","schema_version":"1.0","event_id":"sha256:ae57f805544d784c35bdcbee96db22f21a0ab5874408063fa14deea29ae4951f"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/7WXFBWRLDWOTJNB4DNI5NMHC2B/bundle.json","state_url":"https://pith.science/pith/7WXFBWRLDWOTJNB4DNI5NMHC2B/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/7WXFBWRLDWOTJNB4DNI5NMHC2B/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-04T18:30:26Z","links":{"resolver":"https://pith.science/pith/7WXFBWRLDWOTJNB4DNI5NMHC2B","bundle":"https://pith.science/pith/7WXFBWRLDWOTJNB4DNI5NMHC2B/bundle.json","state":"https://pith.science/pith/7WXFBWRLDWOTJNB4DNI5NMHC2B/state.json","well_known_bundle":"https://pith.science/.well-known/pith/7WXFBWRLDWOTJNB4DNI5NMHC2B/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:7WXFBWRLDWOTJNB4DNI5NMHC2B","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c844fd83ab4970458a428c7f7819ac5965411473e5d872de679fd49577de3e41","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-05-27T19:28:33Z","title_canon_sha256":"610e45d2f405f7af9118997736b70725d7063f04088cda3e1e63b9f811457c2e"},"schema_version":"1.0","source":{"id":"2405.17618","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.17618","created_at":"2026-07-05T11:25:01Z"},{"alias_kind":"arxiv_version","alias_value":"2405.17618v3","created_at":"2026-07-05T11:25:01Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.17618","created_at":"2026-07-05T11:25:01Z"},{"alias_kind":"pith_short_12","alias_value":"7WXFBWRLDWOT","created_at":"2026-07-05T11:25:01Z"},{"alias_kind":"pith_short_16","alias_value":"7WXFBWRLDWOTJNB4","created_at":"2026-07-05T11:25:01Z"},{"alias_kind":"pith_short_8","alias_value":"7WXFBWRL","created_at":"2026-07-05T11:25:01Z"}],"graph_snapshots":[{"event_id":"sha256:ae57f805544d784c35bdcbee96db22f21a0ab5874408063fa14deea29ae4951f","target":"graph","created_at":"2026-07-05T11:25:01Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.17618/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reinforcement learning (RL) training is inherently unstable due to factors such as moving targets and high gradient variance. Reinforcement Learning from Human Feedback (RLHF) and Reinforcement Learning from AI Feedback (RLAIF) can introduce additional difficulty. Differing preferences can complicate the alignment process, and prediction errors in a trained reward model can become more severe as the LLM generates unseen outputs. To enhance training robustness, RL has adopted techniques from supervised learning, such as ensembles and layer normalization. In this work, we improve the stability o","authors_text":"Andrew Perrault, Ju-Seung Byun","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-05-27T19:28:33Z","title":"Symmetric Reinforcement Learning Loss for Robust Learning on Diverse Tasks and Model Scales"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.17618","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:799693d21a803c581a64ee4cc90162835767a152f783ff0f714c1810f1de7c24","target":"record","created_at":"2026-07-05T11:25:01Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c844fd83ab4970458a428c7f7819ac5965411473e5d872de679fd49577de3e41","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-05-27T19:28:33Z","title_canon_sha256":"610e45d2f405f7af9118997736b70725d7063f04088cda3e1e63b9f811457c2e"},"schema_version":"1.0","source":{"id":"2405.17618","kind":"arxiv","version":3}},"canonical_sha256":"fdae50da2b1d9d34b43c1b51d6b0e2d04adf37a804453a535329fd2be8ccff49","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"fdae50da2b1d9d34b43c1b51d6b0e2d04adf37a804453a535329fd2be8ccff49","first_computed_at":"2026-07-05T11:25:01.666335Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:25:01.666335Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"drtNQmdPl0UkxwV2jEe4FUf3I+TF8JxLeOj5bVw31hiBTqMRs7F6LBAyMxsDNrI2Qx0bmpvUo2ra3IZKlpleCQ==","signature_status":"signed_v1","signed_at":"2026-07-05T11:25:01.666809Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.17618","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:799693d21a803c581a64ee4cc90162835767a152f783ff0f714c1810f1de7c24","sha256:ae57f805544d784c35bdcbee96db22f21a0ab5874408063fa14deea29ae4951f"],"state_sha256":"a76570dce95520deeb0ef78029aff1ecf2108462156f71a754681d5cd51b191e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"lY4pRzJWW5Fl3wE7LEgTN/2ZicJZzys37rtHASWoBF59R7jrlIGSRuXYc3nJgwQOExoPcYup93zKQaytzzxhCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-04T18:30:26.647454Z","bundle_sha256":"e3673b0ceaf393ccd5cc225f31af234753fe3db0e99b4cd10786b24350221dfd"}}