{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:2CPG3DYX2LIRE2ZPSLLKXURCEJ","short_pith_number":"pith:2CPG3DYX","schema_version":"1.0","canonical_sha256":"d09e6d8f17d2d1126b2f92d6abd22222507128976fcb9de472936e56ced16536","source":{"kind":"arxiv","id":"2508.04409","version":3},"attestation_state":"computed","paper":{"title":"The Relative Instability of Model Comparison with Cross-validation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"stat.ML","authors_text":"Alexandre Bayle, Lester Mackey, Lucas Janson","submitted_at":"2025-08-06T12:54:56Z","abstract_excerpt":"Cross-validation (CV) is known to provide asymptotically exact tests and confidence intervals for model improvement but only when the model comparison is relatively stable. Surprisingly, we prove that even simple, individually stable models can generate relatively unstable comparisons, calling into question the validity of CV inference. Specifically, we show that the Lasso and its close cousin, soft-thresholding, generate relatively unstable comparisons and invalid CV inferences, even in the most favorable of learning settings and when both models are individually stable. These findings highli"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2508.04409","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.ML","submitted_at":"2025-08-06T12:54:56Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"1180bc8a88e9f36ca6d19b8828ca800a6dccf3fa258d813b06225fbf2bf330f2","abstract_canon_sha256":"bb8bec7da0eebc1bde65458a24b41a38b9b41392b311f869c84b8c06c1ff8b63"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-06-05T01:15:15.166964Z","signature_b64":"IDjz7V+s45gjL0Jp1pdD5V2i7xezprbBkp/rFTbANlydAP2l7etNUk29eZ8yKePBe2CrzlvioUEv09WKO4hdDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d09e6d8f17d2d1126b2f92d6abd22222507128976fcb9de472936e56ced16536","last_reissued_at":"2026-06-05T01:15:15.166178Z","signature_status":"signed_v1","first_computed_at":"2026-06-05T01:15:15.166178Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"The Relative Instability of Model Comparison with Cross-validation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"stat.ML","authors_text":"Alexandre Bayle, Lester Mackey, Lucas Janson","submitted_at":"2025-08-06T12:54:56Z","abstract_excerpt":"Cross-validation (CV) is known to provide asymptotically exact tests and confidence intervals for model improvement but only when the model comparison is relatively stable. Surprisingly, we prove that even simple, individually stable models can generate relatively unstable comparisons, calling into question the validity of CV inference. Specifically, we show that the Lasso and its close cousin, soft-thresholding, generate relatively unstable comparisons and invalid CV inferences, even in the most favorable of learning settings and when both models are individually stable. These findings highli"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2508.04409","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2508.04409/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2508.04409","created_at":"2026-06-05T01:15:15.166279+00:00"},{"alias_kind":"arxiv_version","alias_value":"2508.04409v3","created_at":"2026-06-05T01:15:15.166279+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2508.04409","created_at":"2026-06-05T01:15:15.166279+00:00"},{"alias_kind":"pith_short_12","alias_value":"2CPG3DYX2LIR","created_at":"2026-06-05T01:15:15.166279+00:00"},{"alias_kind":"pith_short_16","alias_value":"2CPG3DYX2LIRE2ZP","created_at":"2026-06-05T01:15:15.166279+00:00"},{"alias_kind":"pith_short_8","alias_value":"2CPG3DYX","created_at":"2026-06-05T01:15:15.166279+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/2CPG3DYX2LIRE2ZPSLLKXURCEJ","json":"https://pith.science/pith/2CPG3DYX2LIRE2ZPSLLKXURCEJ.json","graph_json":"https://pith.science/api/pith-number/2CPG3DYX2LIRE2ZPSLLKXURCEJ/graph.json","events_json":"https://pith.science/api/pith-number/2CPG3DYX2LIRE2ZPSLLKXURCEJ/events.json","paper":"https://pith.science/paper/2CPG3DYX"},"agent_actions":{"view_html":"https://pith.science/pith/2CPG3DYX2LIRE2ZPSLLKXURCEJ","download_json":"https://pith.science/pith/2CPG3DYX2LIRE2ZPSLLKXURCEJ.json","view_paper":"https://pith.science/paper/2CPG3DYX","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2508.04409&json=true","fetch_graph":"https://pith.science/api/pith-number/2CPG3DYX2LIRE2ZPSLLKXURCEJ/graph.json","fetch_events":"https://pith.science/api/pith-number/2CPG3DYX2LIRE2ZPSLLKXURCEJ/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/2CPG3DYX2LIRE2ZPSLLKXURCEJ/action/timestamp_anchor","attest_storage":"https://pith.science/pith/2CPG3DYX2LIRE2ZPSLLKXURCEJ/action/storage_attestation","attest_author":"https://pith.science/pith/2CPG3DYX2LIRE2ZPSLLKXURCEJ/action/author_attestation","sign_citation":"https://pith.science/pith/2CPG3DYX2LIRE2ZPSLLKXURCEJ/action/citation_signature","submit_replication":"https://pith.science/pith/2CPG3DYX2LIRE2ZPSLLKXURCEJ/action/replication_record"}},"created_at":"2026-06-05T01:15:15.166279+00:00","updated_at":"2026-06-05T01:15:15.166279+00:00"}