{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:YBUXGDWKW3V76TCMPKIMLD6CTC","short_pith_number":"pith:YBUXGDWK","canonical_record":{"source":{"id":"2503.19612","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2025-03-25T12:52:38Z","cross_cats_sorted":[],"title_canon_sha256":"cc423010901ad6b183549e11eefd940067f2b011f0aab17ce82e4a124694c6b0","abstract_canon_sha256":"14705feb80a7659726d6c3940f3bbf2ddd5d38d98ca6a405222e89903ac42fcf"},"schema_version":"1.0"},"canonical_sha256":"c069730ecab6ebff4c4c7a90c58fc2988290847bf4f207d64a5a0677da070fba","source":{"kind":"arxiv","id":"2503.19612","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2503.19612","created_at":"2026-07-05T10:41:22Z"},{"alias_kind":"arxiv_version","alias_value":"2503.19612v2","created_at":"2026-07-05T10:41:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.19612","created_at":"2026-07-05T10:41:22Z"},{"alias_kind":"pith_short_12","alias_value":"YBUXGDWKW3V7","created_at":"2026-07-05T10:41:22Z"},{"alias_kind":"pith_short_16","alias_value":"YBUXGDWKW3V76TCM","created_at":"2026-07-05T10:41:22Z"},{"alias_kind":"pith_short_8","alias_value":"YBUXGDWK","created_at":"2026-07-05T10:41:22Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:YBUXGDWKW3V76TCMPKIMLD6CTC","target":"record","payload":{"canonical_record":{"source":{"id":"2503.19612","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2025-03-25T12:52:38Z","cross_cats_sorted":[],"title_canon_sha256":"cc423010901ad6b183549e11eefd940067f2b011f0aab17ce82e4a124694c6b0","abstract_canon_sha256":"14705feb80a7659726d6c3940f3bbf2ddd5d38d98ca6a405222e89903ac42fcf"},"schema_version":"1.0"},"canonical_sha256":"c069730ecab6ebff4c4c7a90c58fc2988290847bf4f207d64a5a0677da070fba","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:41:22.875058Z","signature_b64":"JeHBUYjSc0C0cfq7/awELhT23l6kuewYb6QcJ5BWm+Q/QR5pZmaf118kSSlZuY49GS8Y3hfSjsL01Ywu6XkDDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c069730ecab6ebff4c4c7a90c58fc2988290847bf4f207d64a5a0677da070fba","last_reissued_at":"2026-07-05T10:41:22.874555Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:41:22.874555Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2503.19612","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:41:22Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"D0p7f0wRJLPFe2CaUuRub3Qb9dMYY74hu9a50Sl/YS38firVHSbpnLCHFIoppGCxogIZScF4BPOov1X5RzoDAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T11:42:42.221084Z"},"content_sha256":"eaa663ccadbb162c9bf2f803c2ded30015c4bf15101c93d943729b13d294f06a","schema_version":"1.0","event_id":"sha256:eaa663ccadbb162c9bf2f803c2ded30015c4bf15101c93d943729b13d294f06a"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:YBUXGDWKW3V76TCMPKIMLD6CTC","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"RL-finetuning LLMs from on- and off-policy data with a single algorithm","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"David W. Zhang, Michal Valko, R\\'emi Munos, Taco Cohen, Yunhao Tang","submitted_at":"2025-03-25T12:52:38Z","abstract_excerpt":"We introduce a novel reinforcement learning algorithm (AGRO, for Any-Generation Reward Optimization) for fine-tuning large-language models. AGRO leverages the concept of generation consistency, which states that the optimal policy satisfies the notion of consistency across any possible generation of the model. We derive algorithms that find optimal solutions via the sample-based policy gradient and provide theoretical guarantees on their convergence. Our experiments demonstrate the effectiveness of AGRO in both on-policy and off-policy settings, showing improved performance on the mathematical"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.19612","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2503.19612/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:41:22Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"EBzcfQswmHd4AFvlxzHFdI2FSO2AmxGWD9Qw4RJDaugk/huisAYcjmFB05tRdEsP/79G0ZpY4tvT7W0vsl5tBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T11:42:42.221458Z"},"content_sha256":"6c754b0bbe8ec3bd36aca131a757a13df15edfc8d96acbcea4c9587ad9485bc5","schema_version":"1.0","event_id":"sha256:6c754b0bbe8ec3bd36aca131a757a13df15edfc8d96acbcea4c9587ad9485bc5"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/YBUXGDWKW3V76TCMPKIMLD6CTC/bundle.json","state_url":"https://pith.science/pith/YBUXGDWKW3V76TCMPKIMLD6CTC/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/YBUXGDWKW3V76TCMPKIMLD6CTC/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T11:42:42Z","links":{"resolver":"https://pith.science/pith/YBUXGDWKW3V76TCMPKIMLD6CTC","bundle":"https://pith.science/pith/YBUXGDWKW3V76TCMPKIMLD6CTC/bundle.json","state":"https://pith.science/pith/YBUXGDWKW3V76TCMPKIMLD6CTC/state.json","well_known_bundle":"https://pith.science/.well-known/pith/YBUXGDWKW3V76TCMPKIMLD6CTC/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:YBUXGDWKW3V76TCMPKIMLD6CTC","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"14705feb80a7659726d6c3940f3bbf2ddd5d38d98ca6a405222e89903ac42fcf","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2025-03-25T12:52:38Z","title_canon_sha256":"cc423010901ad6b183549e11eefd940067f2b011f0aab17ce82e4a124694c6b0"},"schema_version":"1.0","source":{"id":"2503.19612","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2503.19612","created_at":"2026-07-05T10:41:22Z"},{"alias_kind":"arxiv_version","alias_value":"2503.19612v2","created_at":"2026-07-05T10:41:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.19612","created_at":"2026-07-05T10:41:22Z"},{"alias_kind":"pith_short_12","alias_value":"YBUXGDWKW3V7","created_at":"2026-07-05T10:41:22Z"},{"alias_kind":"pith_short_16","alias_value":"YBUXGDWKW3V76TCM","created_at":"2026-07-05T10:41:22Z"},{"alias_kind":"pith_short_8","alias_value":"YBUXGDWK","created_at":"2026-07-05T10:41:22Z"}],"graph_snapshots":[{"event_id":"sha256:6c754b0bbe8ec3bd36aca131a757a13df15edfc8d96acbcea4c9587ad9485bc5","target":"graph","created_at":"2026-07-05T10:41:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2503.19612/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We introduce a novel reinforcement learning algorithm (AGRO, for Any-Generation Reward Optimization) for fine-tuning large-language models. AGRO leverages the concept of generation consistency, which states that the optimal policy satisfies the notion of consistency across any possible generation of the model. We derive algorithms that find optimal solutions via the sample-based policy gradient and provide theoretical guarantees on their convergence. Our experiments demonstrate the effectiveness of AGRO in both on-policy and off-policy settings, showing improved performance on the mathematical","authors_text":"David W. Zhang, Michal Valko, R\\'emi Munos, Taco Cohen, Yunhao Tang","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2025-03-25T12:52:38Z","title":"RL-finetuning LLMs from on- and off-policy data with a single algorithm"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.19612","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:eaa663ccadbb162c9bf2f803c2ded30015c4bf15101c93d943729b13d294f06a","target":"record","created_at":"2026-07-05T10:41:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"14705feb80a7659726d6c3940f3bbf2ddd5d38d98ca6a405222e89903ac42fcf","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2025-03-25T12:52:38Z","title_canon_sha256":"cc423010901ad6b183549e11eefd940067f2b011f0aab17ce82e4a124694c6b0"},"schema_version":"1.0","source":{"id":"2503.19612","kind":"arxiv","version":2}},"canonical_sha256":"c069730ecab6ebff4c4c7a90c58fc2988290847bf4f207d64a5a0677da070fba","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"c069730ecab6ebff4c4c7a90c58fc2988290847bf4f207d64a5a0677da070fba","first_computed_at":"2026-07-05T10:41:22.874555Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:41:22.874555Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"JeHBUYjSc0C0cfq7/awELhT23l6kuewYb6QcJ5BWm+Q/QR5pZmaf118kSSlZuY49GS8Y3hfSjsL01Ywu6XkDDQ==","signature_status":"signed_v1","signed_at":"2026-07-05T10:41:22.875058Z","signed_message":"canonical_sha256_bytes"},"source_id":"2503.19612","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:eaa663ccadbb162c9bf2f803c2ded30015c4bf15101c93d943729b13d294f06a","sha256:6c754b0bbe8ec3bd36aca131a757a13df15edfc8d96acbcea4c9587ad9485bc5"],"state_sha256":"be6f8f3c2fad43347e2e7327df4c3c82a565e59104428aef0da29e502c5612a4"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"+sBGfbTpeOtxlpv4T4iIdFcMsjxUeauM9LJ0WAyCs5eFfi5rDpSLFe44PeH7+Z/neoTcEi5JcdvEY7Qz2kRQDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T11:42:42.224380Z","bundle_sha256":"ddf276005a44dbddd197ad0fd95bb662226c68244a2d9de8e88a210a3f7a5900"}}