{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:QHH4A43R3BZH4OSN4D5JLBVTOF","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"18d452a73c0b4b8f0bee30c58e0db35a847eed5f5393af5809e83a26d1890dcb","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2025-10-12T10:49:57Z","title_canon_sha256":"8ee32390d7adeea83e068a59b7d871dea70ddae4febcdf1bbf6b0a3e6b39022c"},"schema_version":"1.0","source":{"id":"2510.10541","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2510.10541","created_at":"2026-06-02T01:03:35Z"},{"alias_kind":"arxiv_version","alias_value":"2510.10541v2","created_at":"2026-06-02T01:03:35Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2510.10541","created_at":"2026-06-02T01:03:35Z"},{"alias_kind":"pith_short_12","alias_value":"QHH4A43R3BZH","created_at":"2026-06-02T01:03:35Z"},{"alias_kind":"pith_short_16","alias_value":"QHH4A43R3BZH4OSN","created_at":"2026-06-02T01:03:35Z"},{"alias_kind":"pith_short_8","alias_value":"QHH4A43R","created_at":"2026-06-02T01:03:35Z"}],"graph_snapshots":[{"event_id":"sha256:5f8946148865e93d615b9d8197953936a109ed6da58b465fb40f6118448a7af3","target":"graph","created_at":"2026-06-02T01:03:35Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2510.10541/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Current benchmarks are inadequate for evaluating progress in reinforcement learning (RL) for large language models (LLMs).Despite recent benchmark gains reported for RL, we find that training on these benchmarks' training sets achieves nearly the same performance as training directly on the test sets, suggesting that the benchmarks cannot reliably separate further progress.To study this phenomenon, we introduce a diagnostic suite and the Oracle Performance Gap (OPG) metric that quantifies the performance difference between training on the train split versus the test split of a benchmark. We fu","authors_text":"Cho-Jui Hsieh, Hengguang Zhou, Yiming Zhang, Yining Sun, Zenghui Ding, Zihan Chen","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2025-10-12T10:49:57Z","title":"Rethinking RL Evaluation: Can Benchmarks Truly Reveal Failures of RL Methods?"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2510.10541","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:8cd0defb01d9c9028dbf438e6c0fd5f3636f86fe66b48b38a1e12edee2b6d428","target":"record","created_at":"2026-06-02T01:03:35Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"18d452a73c0b4b8f0bee30c58e0db35a847eed5f5393af5809e83a26d1890dcb","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2025-10-12T10:49:57Z","title_canon_sha256":"8ee32390d7adeea83e068a59b7d871dea70ddae4febcdf1bbf6b0a3e6b39022c"},"schema_version":"1.0","source":{"id":"2510.10541","kind":"arxiv","version":2}},"canonical_sha256":"81cfc07371d8727e3a4de0fa9586b37154f59100cc1d98057bf93f6c046ce343","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"81cfc07371d8727e3a4de0fa9586b37154f59100cc1d98057bf93f6c046ce343","first_computed_at":"2026-06-02T01:03:35.240589Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-06-02T01:03:35.240589Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"E/lAU/9EGHLVLorNrJWcmDi8rtUME+SUKnoaUBLI4PTHfS/HIl5FHm+tG2ihwkKEt71zOo0YVTnvHnluPL9QDg==","signature_status":"signed_v1","signed_at":"2026-06-02T01:03:35.241422Z","signed_message":"canonical_sha256_bytes"},"source_id":"2510.10541","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:8cd0defb01d9c9028dbf438e6c0fd5f3636f86fe66b48b38a1e12edee2b6d428","sha256:5f8946148865e93d615b9d8197953936a109ed6da58b465fb40f6118448a7af3"],"state_sha256":"2776c439aa5791017e302543f158f56c914185c33c4a94d0c7f2d726fa747cca"}