{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:G6A2ILVNEO2EHETMQVUCGFRXLC","short_pith_number":"pith:G6A2ILVN","canonical_record":{"source":{"id":"2310.20703","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-10-31T17:59:05Z","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"title_canon_sha256":"602a206918c663961fafa2b9ba2310fe59a967753eb653e37fe3f162a6e444bb","abstract_canon_sha256":"8dea738fed39fb343fed14c37fa39560e1c53a32e829769af4821c08d655f6cd"},"schema_version":"1.0"},"canonical_sha256":"3781a42ead23b443926c856823163758ba35e542f5606327de057e72f3dd51ab","source":{"kind":"arxiv","id":"2310.20703","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.20703","created_at":"2026-07-05T07:56:00Z"},{"alias_kind":"arxiv_version","alias_value":"2310.20703v3","created_at":"2026-07-05T07:56:00Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.20703","created_at":"2026-07-05T07:56:00Z"},{"alias_kind":"pith_short_12","alias_value":"G6A2ILVNEO2E","created_at":"2026-07-05T07:56:00Z"},{"alias_kind":"pith_short_16","alias_value":"G6A2ILVNEO2EHETM","created_at":"2026-07-05T07:56:00Z"},{"alias_kind":"pith_short_8","alias_value":"G6A2ILVN","created_at":"2026-07-05T07:56:00Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:G6A2ILVNEO2EHETMQVUCGFRXLC","target":"record","payload":{"canonical_record":{"source":{"id":"2310.20703","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-10-31T17:59:05Z","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"title_canon_sha256":"602a206918c663961fafa2b9ba2310fe59a967753eb653e37fe3f162a6e444bb","abstract_canon_sha256":"8dea738fed39fb343fed14c37fa39560e1c53a32e829769af4821c08d655f6cd"},"schema_version":"1.0"},"canonical_sha256":"3781a42ead23b443926c856823163758ba35e542f5606327de057e72f3dd51ab","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:56:00.950263Z","signature_b64":"8Gm/e/bmVVnAdhKAdqbKS3zNibURHbTwDV/yXm0ZSFpf4obqMe63ozvhVXxGbOS0pEmLUFHRwim4bqrs4He/CQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"3781a42ead23b443926c856823163758ba35e542f5606327de057e72f3dd51ab","last_reissued_at":"2026-07-05T07:56:00.949735Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:56:00.949735Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2310.20703","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:56:00Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"mfO8nEah0HnvEjxQ6dXvglO0meXvt9bsuEtw20SPFfUiDcb8bziK7szVSRSa0tTuA7aW9ApRAh8ULEm9RUwkCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-17T01:42:27.691104Z"},"content_sha256":"e72bb88070bc3884073fe7d29a8274cc7044c5bda9ff4210d23d8c31d37068f8","schema_version":"1.0","event_id":"sha256:e72bb88070bc3884073fe7d29a8274cc7044c5bda9ff4210d23d8c31d37068f8"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:G6A2ILVNEO2EHETMQVUCGFRXLC","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Vanishing Gradients in Reinforcement Finetuning of Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL","stat.ML"],"primary_cat":"cs.LG","authors_text":"Arwen Bradley, Etai Littwin, Hattie Zhou, Joshua Susskind, Noam Razin, Omid Saremi, Preetum Nakkiran, Vimal Thilak","submitted_at":"2023-10-31T17:59:05Z","abstract_excerpt":"Pretrained language models are commonly aligned with human preferences and downstream tasks via reinforcement finetuning (RFT), which refers to maximizing a (possibly learned) reward function using policy gradient algorithms. This work identifies a fundamental optimization obstacle in RFT: we prove that the expected gradient for an input vanishes when its reward standard deviation under the model is small, even if the expected reward is far from optimal. Through experiments on an RFT benchmark and controlled environments, as well as a theoretical analysis, we then demonstrate that vanishing gr"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.20703","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2310.20703/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:56:00Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"D2O4uiXL36lOpSd5q/XeryCVZYVKs/uVQDw51t/UCEhVdDPEXRI3va1fvQaDLgde8BAmc5YSBcdfZDErRCEaBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-17T01:42:27.691616Z"},"content_sha256":"627578e23d241eaaea50a9cf6db5ccc80955014279a8aa0d711489f0445864e6","schema_version":"1.0","event_id":"sha256:627578e23d241eaaea50a9cf6db5ccc80955014279a8aa0d711489f0445864e6"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/G6A2ILVNEO2EHETMQVUCGFRXLC/bundle.json","state_url":"https://pith.science/pith/G6A2ILVNEO2EHETMQVUCGFRXLC/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/G6A2ILVNEO2EHETMQVUCGFRXLC/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-17T01:42:27Z","links":{"resolver":"https://pith.science/pith/G6A2ILVNEO2EHETMQVUCGFRXLC","bundle":"https://pith.science/pith/G6A2ILVNEO2EHETMQVUCGFRXLC/bundle.json","state":"https://pith.science/pith/G6A2ILVNEO2EHETMQVUCGFRXLC/state.json","well_known_bundle":"https://pith.science/.well-known/pith/G6A2ILVNEO2EHETMQVUCGFRXLC/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:G6A2ILVNEO2EHETMQVUCGFRXLC","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"8dea738fed39fb343fed14c37fa39560e1c53a32e829769af4821c08d655f6cd","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-10-31T17:59:05Z","title_canon_sha256":"602a206918c663961fafa2b9ba2310fe59a967753eb653e37fe3f162a6e444bb"},"schema_version":"1.0","source":{"id":"2310.20703","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.20703","created_at":"2026-07-05T07:56:00Z"},{"alias_kind":"arxiv_version","alias_value":"2310.20703v3","created_at":"2026-07-05T07:56:00Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.20703","created_at":"2026-07-05T07:56:00Z"},{"alias_kind":"pith_short_12","alias_value":"G6A2ILVNEO2E","created_at":"2026-07-05T07:56:00Z"},{"alias_kind":"pith_short_16","alias_value":"G6A2ILVNEO2EHETM","created_at":"2026-07-05T07:56:00Z"},{"alias_kind":"pith_short_8","alias_value":"G6A2ILVN","created_at":"2026-07-05T07:56:00Z"}],"graph_snapshots":[{"event_id":"sha256:627578e23d241eaaea50a9cf6db5ccc80955014279a8aa0d711489f0445864e6","target":"graph","created_at":"2026-07-05T07:56:00Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2310.20703/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Pretrained language models are commonly aligned with human preferences and downstream tasks via reinforcement finetuning (RFT), which refers to maximizing a (possibly learned) reward function using policy gradient algorithms. This work identifies a fundamental optimization obstacle in RFT: we prove that the expected gradient for an input vanishes when its reward standard deviation under the model is small, even if the expected reward is far from optimal. Through experiments on an RFT benchmark and controlled environments, as well as a theoretical analysis, we then demonstrate that vanishing gr","authors_text":"Arwen Bradley, Etai Littwin, Hattie Zhou, Joshua Susskind, Noam Razin, Omid Saremi, Preetum Nakkiran, Vimal Thilak","cross_cats":["cs.AI","cs.CL","stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-10-31T17:59:05Z","title":"Vanishing Gradients in Reinforcement Finetuning of Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.20703","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:e72bb88070bc3884073fe7d29a8274cc7044c5bda9ff4210d23d8c31d37068f8","target":"record","created_at":"2026-07-05T07:56:00Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"8dea738fed39fb343fed14c37fa39560e1c53a32e829769af4821c08d655f6cd","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2023-10-31T17:59:05Z","title_canon_sha256":"602a206918c663961fafa2b9ba2310fe59a967753eb653e37fe3f162a6e444bb"},"schema_version":"1.0","source":{"id":"2310.20703","kind":"arxiv","version":3}},"canonical_sha256":"3781a42ead23b443926c856823163758ba35e542f5606327de057e72f3dd51ab","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"3781a42ead23b443926c856823163758ba35e542f5606327de057e72f3dd51ab","first_computed_at":"2026-07-05T07:56:00.949735Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:56:00.949735Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"8Gm/e/bmVVnAdhKAdqbKS3zNibURHbTwDV/yXm0ZSFpf4obqMe63ozvhVXxGbOS0pEmLUFHRwim4bqrs4He/CQ==","signature_status":"signed_v1","signed_at":"2026-07-05T07:56:00.950263Z","signed_message":"canonical_sha256_bytes"},"source_id":"2310.20703","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:e72bb88070bc3884073fe7d29a8274cc7044c5bda9ff4210d23d8c31d37068f8","sha256:627578e23d241eaaea50a9cf6db5ccc80955014279a8aa0d711489f0445864e6"],"state_sha256":"a555877ae7b426d08038acf1793a80584eb27ee3cb283f6fbe93fe6d6c44a785"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"NyZy5R3xaSgn70UY/wzuhoAarkbtaOSP0KZ2qN5wPhAqSnQND+XG2l604f4s7aVmTY4LFoErNFRRPwQeYN3vAQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-17T01:42:27.695397Z","bundle_sha256":"f7a96be8a751e02a3cbb08cffc6dba42c9fcc48f22911b51ca85af8ebcf3e014"}}