{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:A2RVDFE3NVMYT5ELXR5OUOCOHV","short_pith_number":"pith:A2RVDFE3","canonical_record":{"source":{"id":"2410.13343","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-17T08:52:52Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"775ebfb7f74e5103c7586e82266279a1f250c6b259e1573b568a89721b51104f","abstract_canon_sha256":"ccc30940d762f0467a4a91c66f62042a83d4a29010b408a24f0c6d168e53cb27"},"schema_version":"1.0"},"canonical_sha256":"06a351949b6d5989f48bbc7aea384e3d6e43459a3c86adf3b5b2bfc0512acb30","source":{"kind":"arxiv","id":"2410.13343","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.13343","created_at":"2026-07-05T09:21:58Z"},{"alias_kind":"arxiv_version","alias_value":"2410.13343v1","created_at":"2026-07-05T09:21:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.13343","created_at":"2026-07-05T09:21:58Z"},{"alias_kind":"pith_short_12","alias_value":"A2RVDFE3NVMY","created_at":"2026-07-05T09:21:58Z"},{"alias_kind":"pith_short_16","alias_value":"A2RVDFE3NVMYT5EL","created_at":"2026-07-05T09:21:58Z"},{"alias_kind":"pith_short_8","alias_value":"A2RVDFE3","created_at":"2026-07-05T09:21:58Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:A2RVDFE3NVMYT5ELXR5OUOCOHV","target":"record","payload":{"canonical_record":{"source":{"id":"2410.13343","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-17T08:52:52Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"775ebfb7f74e5103c7586e82266279a1f250c6b259e1573b568a89721b51104f","abstract_canon_sha256":"ccc30940d762f0467a4a91c66f62042a83d4a29010b408a24f0c6d168e53cb27"},"schema_version":"1.0"},"canonical_sha256":"06a351949b6d5989f48bbc7aea384e3d6e43459a3c86adf3b5b2bfc0512acb30","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:21:58.929729Z","signature_b64":"4tOUEhHXYma5mej/BxNb/gFZM6zoFEftNMKTUF9HLI4vyVkanzIClracrVFK6XMOWAM+PhuiQZyvEw3wLucAAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"06a351949b6d5989f48bbc7aea384e3d6e43459a3c86adf3b5b2bfc0512acb30","last_reissued_at":"2026-07-05T09:21:58.929235Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:21:58.929235Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.13343","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:21:58Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"q0WPjQk0hkbrPWZx/iDNJrPHq9AzFdJWi9Q/PWKpD6R+w9DoLiVRDleV0E0SDXycebJeE1ITmdQenkSFyJxRCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T07:11:27.168265Z"},"content_sha256":"20676233df3a1a6a3c84f3df8631d9bbde043b2709d8b2ca9d7e955ffdded7de","schema_version":"1.0","event_id":"sha256:20676233df3a1a6a3c84f3df8631d9bbde043b2709d8b2ca9d7e955ffdded7de"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:A2RVDFE3NVMYT5ELXR5OUOCOHV","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Do LLMs Overcome Shortcut Learning? An Evaluation of Shortcut Challenges in Large Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Guangting Zheng, Kai Zhang, Lili Zhao, Qi Liu, Yu Yuan","submitted_at":"2024-10-17T08:52:52Z","abstract_excerpt":"Large Language Models (LLMs) have shown remarkable capabilities in various natural language processing tasks. However, LLMs may rely on dataset biases as shortcuts for prediction, which can significantly impair their robustness and generalization capabilities. This paper presents Shortcut Suite, a comprehensive test suite designed to evaluate the impact of shortcuts on LLMs' performance, incorporating six shortcut types, five evaluation metrics, and four prompting strategies. Our extensive experiments yield several key findings: 1) LLMs demonstrate varying reliance on shortcuts for downstream "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.13343","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.13343/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:21:58Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"irlnq+fnw8DM5kVsupo+f7vyUYEVunQnwI0Z3vtVl1t5KKt86yKauOXZ3+bG4u2SmZiiqUdqwc7WOCV5LJYyDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T07:11:27.168816Z"},"content_sha256":"7942eff1ba0f3bb55a19aefc0e28d4f6a382d014e692970fdf17fe4af8a5d9fe","schema_version":"1.0","event_id":"sha256:7942eff1ba0f3bb55a19aefc0e28d4f6a382d014e692970fdf17fe4af8a5d9fe"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/A2RVDFE3NVMYT5ELXR5OUOCOHV/bundle.json","state_url":"https://pith.science/pith/A2RVDFE3NVMYT5ELXR5OUOCOHV/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/A2RVDFE3NVMYT5ELXR5OUOCOHV/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T07:11:27Z","links":{"resolver":"https://pith.science/pith/A2RVDFE3NVMYT5ELXR5OUOCOHV","bundle":"https://pith.science/pith/A2RVDFE3NVMYT5ELXR5OUOCOHV/bundle.json","state":"https://pith.science/pith/A2RVDFE3NVMYT5ELXR5OUOCOHV/state.json","well_known_bundle":"https://pith.science/.well-known/pith/A2RVDFE3NVMYT5ELXR5OUOCOHV/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:A2RVDFE3NVMYT5ELXR5OUOCOHV","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"ccc30940d762f0467a4a91c66f62042a83d4a29010b408a24f0c6d168e53cb27","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-17T08:52:52Z","title_canon_sha256":"775ebfb7f74e5103c7586e82266279a1f250c6b259e1573b568a89721b51104f"},"schema_version":"1.0","source":{"id":"2410.13343","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.13343","created_at":"2026-07-05T09:21:58Z"},{"alias_kind":"arxiv_version","alias_value":"2410.13343v1","created_at":"2026-07-05T09:21:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.13343","created_at":"2026-07-05T09:21:58Z"},{"alias_kind":"pith_short_12","alias_value":"A2RVDFE3NVMY","created_at":"2026-07-05T09:21:58Z"},{"alias_kind":"pith_short_16","alias_value":"A2RVDFE3NVMYT5EL","created_at":"2026-07-05T09:21:58Z"},{"alias_kind":"pith_short_8","alias_value":"A2RVDFE3","created_at":"2026-07-05T09:21:58Z"}],"graph_snapshots":[{"event_id":"sha256:7942eff1ba0f3bb55a19aefc0e28d4f6a382d014e692970fdf17fe4af8a5d9fe","target":"graph","created_at":"2026-07-05T09:21:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.13343/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large Language Models (LLMs) have shown remarkable capabilities in various natural language processing tasks. However, LLMs may rely on dataset biases as shortcuts for prediction, which can significantly impair their robustness and generalization capabilities. This paper presents Shortcut Suite, a comprehensive test suite designed to evaluate the impact of shortcuts on LLMs' performance, incorporating six shortcut types, five evaluation metrics, and four prompting strategies. Our extensive experiments yield several key findings: 1) LLMs demonstrate varying reliance on shortcuts for downstream ","authors_text":"Guangting Zheng, Kai Zhang, Lili Zhao, Qi Liu, Yu Yuan","cross_cats":["cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-17T08:52:52Z","title":"Do LLMs Overcome Shortcut Learning? An Evaluation of Shortcut Challenges in Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.13343","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:20676233df3a1a6a3c84f3df8631d9bbde043b2709d8b2ca9d7e955ffdded7de","target":"record","created_at":"2026-07-05T09:21:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"ccc30940d762f0467a4a91c66f62042a83d4a29010b408a24f0c6d168e53cb27","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-17T08:52:52Z","title_canon_sha256":"775ebfb7f74e5103c7586e82266279a1f250c6b259e1573b568a89721b51104f"},"schema_version":"1.0","source":{"id":"2410.13343","kind":"arxiv","version":1}},"canonical_sha256":"06a351949b6d5989f48bbc7aea384e3d6e43459a3c86adf3b5b2bfc0512acb30","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"06a351949b6d5989f48bbc7aea384e3d6e43459a3c86adf3b5b2bfc0512acb30","first_computed_at":"2026-07-05T09:21:58.929235Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:21:58.929235Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"4tOUEhHXYma5mej/BxNb/gFZM6zoFEftNMKTUF9HLI4vyVkanzIClracrVFK6XMOWAM+PhuiQZyvEw3wLucAAA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:21:58.929729Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.13343","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:20676233df3a1a6a3c84f3df8631d9bbde043b2709d8b2ca9d7e955ffdded7de","sha256:7942eff1ba0f3bb55a19aefc0e28d4f6a382d014e692970fdf17fe4af8a5d9fe"],"state_sha256":"d27fe5f98a3bdc2b35e03dd9c354db30cec810c86daa7d0c01c3927816050a63"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"RYMgK3xJ8tOeW+e02r8jJMOA9YAFQkhpRd9sef1LDREkbuZCINSsel7pwwkCH5IbNpa3Mo9G/aZiPKrdrYVmBQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T07:11:27.174534Z","bundle_sha256":"0e08452ca6a1a66ae298b78dd70989202bb09f366aa1345faba69dba3e851b97"}}