{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:PJFAWVROMHXZUI4OR2CFVWREXS","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"f0015c7ca0528a9c03e8dd96e73acf5477d45ba9d5e22ce038178777b5291327","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-09-30T14:00:34Z","title_canon_sha256":"fce32ad044acbad9500b0500e8e1f8d036ccc1c12f19f92671bfc39911422bc8"},"schema_version":"1.0","source":{"id":"2409.20303","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.20303","created_at":"2026-07-05T09:13:41Z"},{"alias_kind":"arxiv_version","alias_value":"2409.20303v1","created_at":"2026-07-05T09:13:41Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.20303","created_at":"2026-07-05T09:13:41Z"},{"alias_kind":"pith_short_12","alias_value":"PJFAWVROMHXZ","created_at":"2026-07-05T09:13:41Z"},{"alias_kind":"pith_short_16","alias_value":"PJFAWVROMHXZUI4O","created_at":"2026-07-05T09:13:41Z"},{"alias_kind":"pith_short_8","alias_value":"PJFAWVRO","created_at":"2026-07-05T09:13:41Z"}],"graph_snapshots":[{"event_id":"sha256:513bb39781b955e6ab8cacf260355938b149a5a94eb4b848ca00df3bbd9c18eb","target":"graph","created_at":"2026-07-05T09:13:41Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2409.20303/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In an era where large language models (LLMs) are increasingly integrated into a wide range of everyday applications, research into these models' behavior has surged. However, due to the novelty of the field, clear methodological guidelines are lacking. This raises concerns about the replicability and generalizability of insights gained from research on LLM behavior. In this study, we discuss the potential risk of a replication crisis and support our concerns with a series of replication experiments focused on prompt engineering techniques purported to influence reasoning abilities in LLMs. We ","authors_text":"Laur\\`ene Vaugrante, Mathias Niepert, Thilo Hagendorff","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-09-30T14:00:34Z","title":"A Looming Replication Crisis in Evaluating Behavior in Language Models? Evidence and Solutions"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.20303","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:fdc0920a6137ebc303302f8d3fe4f424adc5aa3c869edd30f131a6046a2c8394","target":"record","created_at":"2026-07-05T09:13:41Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"f0015c7ca0528a9c03e8dd96e73acf5477d45ba9d5e22ce038178777b5291327","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-09-30T14:00:34Z","title_canon_sha256":"fce32ad044acbad9500b0500e8e1f8d036ccc1c12f19f92671bfc39911422bc8"},"schema_version":"1.0","source":{"id":"2409.20303","kind":"arxiv","version":1}},"canonical_sha256":"7a4a0b562e61ef9a238e8e845ada24bcbd1cbbe7ee28b79cdbf0ae9b28b01a43","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"7a4a0b562e61ef9a238e8e845ada24bcbd1cbbe7ee28b79cdbf0ae9b28b01a43","first_computed_at":"2026-07-05T09:13:41.514869Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:13:41.514869Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"r5DPFQGIw3DWXtSQdh+jMZ3JLXmsWRWe7GslMUOWiYIPNnbofDXedMB7I6f7XTh4jSXdB/TDM1x5+Oj4hyanDQ==","signature_status":"signed_v1","signed_at":"2026-07-05T09:13:41.515235Z","signed_message":"canonical_sha256_bytes"},"source_id":"2409.20303","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:fdc0920a6137ebc303302f8d3fe4f424adc5aa3c869edd30f131a6046a2c8394","sha256:513bb39781b955e6ab8cacf260355938b149a5a94eb4b848ca00df3bbd9c18eb"],"state_sha256":"fda03c4ec02eaeb994d7f565142ecd0307873ddffcec14a1d899812d1d93ec95"}