{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:CHOHPGD2OUAWYQCNLB2JELFP7I","short_pith_number":"pith:CHOHPGD2","canonical_record":{"source":{"id":"2502.05675","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-02-08T19:49:32Z","cross_cats_sorted":[],"title_canon_sha256":"d613a4d1c7994137754583f8fed960aae9a067978fe8711332460b7484f668a9","abstract_canon_sha256":"1355f7eb417092eb5e5fc8362fa7d040c6a3def637a79bbb8b031d4326bf2437"},"schema_version":"1.0"},"canonical_sha256":"11dc77987a75016c404d5874922caffa31162527ca42c07f2f4d1e8b5d777f9f","source":{"kind":"arxiv","id":"2502.05675","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2502.05675","created_at":"2026-07-05T10:11:41Z"},{"alias_kind":"arxiv_version","alias_value":"2502.05675v1","created_at":"2026-07-05T10:11:41Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.05675","created_at":"2026-07-05T10:11:41Z"},{"alias_kind":"pith_short_12","alias_value":"CHOHPGD2OUAW","created_at":"2026-07-05T10:11:41Z"},{"alias_kind":"pith_short_16","alias_value":"CHOHPGD2OUAWYQCN","created_at":"2026-07-05T10:11:41Z"},{"alias_kind":"pith_short_8","alias_value":"CHOHPGD2","created_at":"2026-07-05T10:11:41Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:CHOHPGD2OUAWYQCNLB2JELFP7I","target":"record","payload":{"canonical_record":{"source":{"id":"2502.05675","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-02-08T19:49:32Z","cross_cats_sorted":[],"title_canon_sha256":"d613a4d1c7994137754583f8fed960aae9a067978fe8711332460b7484f668a9","abstract_canon_sha256":"1355f7eb417092eb5e5fc8362fa7d040c6a3def637a79bbb8b031d4326bf2437"},"schema_version":"1.0"},"canonical_sha256":"11dc77987a75016c404d5874922caffa31162527ca42c07f2f4d1e8b5d777f9f","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:11:41.894396Z","signature_b64":"KvOyTt4x00yrHuIfZjubGAxydoupJITntiVJycYl8JHoAKU61bFKTtA15Q81gglF9sZmRatZj9YXRgZU4fqYBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"11dc77987a75016c404d5874922caffa31162527ca42c07f2f4d1e8b5d777f9f","last_reissued_at":"2026-07-05T10:11:41.893953Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:11:41.893953Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2502.05675","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:11:41Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"n1+xTPfzImi3+1cIa3b8j64tLA5zN9owvX8A8980m3v5qSOZ4KnOuNgILPU2kS+MULlhYwzoCR2EIHCGqOVvAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-13T14:52:16.798261Z"},"content_sha256":"24bebe1ba53b6c3fd7b84c6d680ec6f273261d8a3c5582fc2474c954e8e028b2","schema_version":"1.0","event_id":"sha256:24bebe1ba53b6c3fd7b84c6d680ec6f273261d8a3c5582fc2474c954e8e028b2"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:CHOHPGD2OUAWYQCNLB2JELFP7I","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Investigating the Shortcomings of LLMs in Step-by-Step Legal Reasoning","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Ali Payani, Bimsara Pathiraja, Chitta Baral, Gaowen Liu, Jayanth Srinivasa, Mihir Parmar, Sat Chidananda, Venkatesh Mishra","submitted_at":"2025-02-08T19:49:32Z","abstract_excerpt":"Reasoning abilities of LLMs have been a key focus in recent years. One challenging reasoning domain with interesting nuances is legal reasoning, which requires careful application of rules, and precedents while balancing deductive and analogical reasoning, and conflicts between rules. Although there have been a few works on using LLMs for legal reasoning, their focus has been on overall accuracy. In this paper, we dig deeper to do a step-by-step analysis and figure out where they commit errors. We use the college-level Multiple Choice Question-Answering (MCQA) task from the \\textit{Civil Proce"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.05675","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2502.05675/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:11:41Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"BcakEi67CNQJ1N+ohdFZ2umFqS6DGpb5fBSf5VF8De3YdrpvyqMRNFTn/dPj4S3QG9U2HJ8VI0uPHjSQbt9ZBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-13T14:52:16.799155Z"},"content_sha256":"43af1fb82a1dc239c26ecb7e9a41e5c29427cd8bdf41f5ba87c87e583bf482be","schema_version":"1.0","event_id":"sha256:43af1fb82a1dc239c26ecb7e9a41e5c29427cd8bdf41f5ba87c87e583bf482be"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/CHOHPGD2OUAWYQCNLB2JELFP7I/bundle.json","state_url":"https://pith.science/pith/CHOHPGD2OUAWYQCNLB2JELFP7I/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/CHOHPGD2OUAWYQCNLB2JELFP7I/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-13T14:52:16Z","links":{"resolver":"https://pith.science/pith/CHOHPGD2OUAWYQCNLB2JELFP7I","bundle":"https://pith.science/pith/CHOHPGD2OUAWYQCNLB2JELFP7I/bundle.json","state":"https://pith.science/pith/CHOHPGD2OUAWYQCNLB2JELFP7I/state.json","well_known_bundle":"https://pith.science/.well-known/pith/CHOHPGD2OUAWYQCNLB2JELFP7I/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:CHOHPGD2OUAWYQCNLB2JELFP7I","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"1355f7eb417092eb5e5fc8362fa7d040c6a3def637a79bbb8b031d4326bf2437","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-02-08T19:49:32Z","title_canon_sha256":"d613a4d1c7994137754583f8fed960aae9a067978fe8711332460b7484f668a9"},"schema_version":"1.0","source":{"id":"2502.05675","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2502.05675","created_at":"2026-07-05T10:11:41Z"},{"alias_kind":"arxiv_version","alias_value":"2502.05675v1","created_at":"2026-07-05T10:11:41Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.05675","created_at":"2026-07-05T10:11:41Z"},{"alias_kind":"pith_short_12","alias_value":"CHOHPGD2OUAW","created_at":"2026-07-05T10:11:41Z"},{"alias_kind":"pith_short_16","alias_value":"CHOHPGD2OUAWYQCN","created_at":"2026-07-05T10:11:41Z"},{"alias_kind":"pith_short_8","alias_value":"CHOHPGD2","created_at":"2026-07-05T10:11:41Z"}],"graph_snapshots":[{"event_id":"sha256:43af1fb82a1dc239c26ecb7e9a41e5c29427cd8bdf41f5ba87c87e583bf482be","target":"graph","created_at":"2026-07-05T10:11:41Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2502.05675/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reasoning abilities of LLMs have been a key focus in recent years. One challenging reasoning domain with interesting nuances is legal reasoning, which requires careful application of rules, and precedents while balancing deductive and analogical reasoning, and conflicts between rules. Although there have been a few works on using LLMs for legal reasoning, their focus has been on overall accuracy. In this paper, we dig deeper to do a step-by-step analysis and figure out where they commit errors. We use the college-level Multiple Choice Question-Answering (MCQA) task from the \\textit{Civil Proce","authors_text":"Ali Payani, Bimsara Pathiraja, Chitta Baral, Gaowen Liu, Jayanth Srinivasa, Mihir Parmar, Sat Chidananda, Venkatesh Mishra","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-02-08T19:49:32Z","title":"Investigating the Shortcomings of LLMs in Step-by-Step Legal Reasoning"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.05675","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:24bebe1ba53b6c3fd7b84c6d680ec6f273261d8a3c5582fc2474c954e8e028b2","target":"record","created_at":"2026-07-05T10:11:41Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"1355f7eb417092eb5e5fc8362fa7d040c6a3def637a79bbb8b031d4326bf2437","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-02-08T19:49:32Z","title_canon_sha256":"d613a4d1c7994137754583f8fed960aae9a067978fe8711332460b7484f668a9"},"schema_version":"1.0","source":{"id":"2502.05675","kind":"arxiv","version":1}},"canonical_sha256":"11dc77987a75016c404d5874922caffa31162527ca42c07f2f4d1e8b5d777f9f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"11dc77987a75016c404d5874922caffa31162527ca42c07f2f4d1e8b5d777f9f","first_computed_at":"2026-07-05T10:11:41.893953Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:11:41.893953Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"KvOyTt4x00yrHuIfZjubGAxydoupJITntiVJycYl8JHoAKU61bFKTtA15Q81gglF9sZmRatZj9YXRgZU4fqYBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T10:11:41.894396Z","signed_message":"canonical_sha256_bytes"},"source_id":"2502.05675","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:24bebe1ba53b6c3fd7b84c6d680ec6f273261d8a3c5582fc2474c954e8e028b2","sha256:43af1fb82a1dc239c26ecb7e9a41e5c29427cd8bdf41f5ba87c87e583bf482be"],"state_sha256":"80a6fbca6fae5499abd78fe725e3437ca06ddf0f6798602cf117db0ec6139d15"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4juVvDcw0o0/b+mwkk7XMqEj8mxeQ2GYUvWhLa3bGwh7CnaRMYYme5cC61QSCJZhEo43SBsim+0dqkUt8AWcBQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-13T14:52:16.806482Z","bundle_sha256":"f9404f8f0785c42f3407644c54ed957af67a849b3edf668a4f422cb54abbe57b"}}