{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:EDVAROFNGNN2XTAR2B632QQTXM","short_pith_number":"pith:EDVAROFN","schema_version":"1.0","canonical_sha256":"20ea08b8ad335babcc11d07dbd4213bb0b7e4d1006fb4e10296d48c39a07bb0b","source":{"kind":"arxiv","id":"2502.05085","version":1},"attestation_state":"computed","paper":{"title":"Causality can systematically address the monsters under the bench(marks)","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Bernhard Sch\\\"olkopf, Felix Leeb, Zhijing Jin","submitted_at":"2025-02-07T17:01:37Z","abstract_excerpt":"Effective and reliable evaluation is essential for advancing empirical machine learning. However, the increasing accessibility of generalist models and the progress towards ever more complex, high-level tasks make systematic evaluation more challenging. Benchmarks are plagued by various biases, artifacts, or leakage, while models may behave unreliably due to poorly explored failure modes. Haphazard treatments and inconsistent formulations of such \"monsters\" can contribute to a duplication of efforts, a lack of trust in results, and unsupported inferences. In this position paper, we argue causa"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2502.05085","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2025-02-07T17:01:37Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"8f37992a8dbee52a5848a85daaf8d29f770257a610dacb7699f54e74bbbdba39","abstract_canon_sha256":"0251e8dbfdf1406d936b969aa2f7dc4808dc4d5374524e2de9feb09f53186557"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:11:03.013555Z","signature_b64":"dIWr7BEBhA+ZSCVABBh9xKbrDh8p5kcq64k8YV/B6Bb76S9OoyFnrldG8rffVklISlePxKd78OK4milHSOPhDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"20ea08b8ad335babcc11d07dbd4213bb0b7e4d1006fb4e10296d48c39a07bb0b","last_reissued_at":"2026-07-05T10:11:03.012999Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:11:03.012999Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Causality can systematically address the monsters under the bench(marks)","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Bernhard Sch\\\"olkopf, Felix Leeb, Zhijing Jin","submitted_at":"2025-02-07T17:01:37Z","abstract_excerpt":"Effective and reliable evaluation is essential for advancing empirical machine learning. However, the increasing accessibility of generalist models and the progress towards ever more complex, high-level tasks make systematic evaluation more challenging. Benchmarks are plagued by various biases, artifacts, or leakage, while models may behave unreliably due to poorly explored failure modes. Haphazard treatments and inconsistent formulations of such \"monsters\" can contribute to a duplication of efforts, a lack of trust in results, and unsupported inferences. In this position paper, we argue causa"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.05085","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2502.05085/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2502.05085","created_at":"2026-07-05T10:11:03.013065+00:00"},{"alias_kind":"arxiv_version","alias_value":"2502.05085v1","created_at":"2026-07-05T10:11:03.013065+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.05085","created_at":"2026-07-05T10:11:03.013065+00:00"},{"alias_kind":"pith_short_12","alias_value":"EDVAROFNGNN2","created_at":"2026-07-05T10:11:03.013065+00:00"},{"alias_kind":"pith_short_16","alias_value":"EDVAROFNGNN2XTAR","created_at":"2026-07-05T10:11:03.013065+00:00"},{"alias_kind":"pith_short_8","alias_value":"EDVAROFN","created_at":"2026-07-05T10:11:03.013065+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2607.21090","citing_title":"Training Large Language Models for Self-Explanation Faithfulness","ref_index":21,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/EDVAROFNGNN2XTAR2B632QQTXM","json":"https://pith.science/pith/EDVAROFNGNN2XTAR2B632QQTXM.json","graph_json":"https://pith.science/api/pith-number/EDVAROFNGNN2XTAR2B632QQTXM/graph.json","events_json":"https://pith.science/api/pith-number/EDVAROFNGNN2XTAR2B632QQTXM/events.json","paper":"https://pith.science/paper/EDVAROFN"},"agent_actions":{"view_html":"https://pith.science/pith/EDVAROFNGNN2XTAR2B632QQTXM","download_json":"https://pith.science/pith/EDVAROFNGNN2XTAR2B632QQTXM.json","view_paper":"https://pith.science/paper/EDVAROFN","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2502.05085&json=true","fetch_graph":"https://pith.science/api/pith-number/EDVAROFNGNN2XTAR2B632QQTXM/graph.json","fetch_events":"https://pith.science/api/pith-number/EDVAROFNGNN2XTAR2B632QQTXM/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/EDVAROFNGNN2XTAR2B632QQTXM/action/timestamp_anchor","attest_storage":"https://pith.science/pith/EDVAROFNGNN2XTAR2B632QQTXM/action/storage_attestation","attest_author":"https://pith.science/pith/EDVAROFNGNN2XTAR2B632QQTXM/action/author_attestation","sign_citation":"https://pith.science/pith/EDVAROFNGNN2XTAR2B632QQTXM/action/citation_signature","submit_replication":"https://pith.science/pith/EDVAROFNGNN2XTAR2B632QQTXM/action/replication_record"}},"created_at":"2026-07-05T10:11:03.013065+00:00","updated_at":"2026-07-05T10:11:03.013065+00:00"}