{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:S4NNPJAX3BIJLE6TSLOICMVKYM","short_pith_number":"pith:S4NNPJAX","canonical_record":{"source":{"id":"2402.11924","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-19T08:12:30Z","cross_cats_sorted":[],"title_canon_sha256":"32fad87ec3217e76c62c4d24c2eae71f512e34c30d6f9d2ad8335c302bdc93ce","abstract_canon_sha256":"ffadffb528a8addca8be2bb329b363dd979ea641a5b573c7c8a406d1d080946c"},"schema_version":"1.0"},"canonical_sha256":"971ad7a417d8509593d392dc8132aac3048e7ee8a64b6f2e3b6b0bfbd3ab2115","source":{"kind":"arxiv","id":"2402.11924","version":5},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.11924","created_at":"2026-07-05T09:20:40Z"},{"alias_kind":"arxiv_version","alias_value":"2402.11924v5","created_at":"2026-07-05T09:20:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.11924","created_at":"2026-07-05T09:20:40Z"},{"alias_kind":"pith_short_12","alias_value":"S4NNPJAX3BIJ","created_at":"2026-07-05T09:20:40Z"},{"alias_kind":"pith_short_16","alias_value":"S4NNPJAX3BIJLE6T","created_at":"2026-07-05T09:20:40Z"},{"alias_kind":"pith_short_8","alias_value":"S4NNPJAX","created_at":"2026-07-05T09:20:40Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:S4NNPJAX3BIJLE6TSLOICMVKYM","target":"record","payload":{"canonical_record":{"source":{"id":"2402.11924","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-19T08:12:30Z","cross_cats_sorted":[],"title_canon_sha256":"32fad87ec3217e76c62c4d24c2eae71f512e34c30d6f9d2ad8335c302bdc93ce","abstract_canon_sha256":"ffadffb528a8addca8be2bb329b363dd979ea641a5b573c7c8a406d1d080946c"},"schema_version":"1.0"},"canonical_sha256":"971ad7a417d8509593d392dc8132aac3048e7ee8a64b6f2e3b6b0bfbd3ab2115","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:20:40.342203Z","signature_b64":"MEVqewX8gOvQsH5dwibKGb352BUzUNPuH2bBqNnZ1mvCQ2uVUG/umeev26hejoUazJsBJIfmxqSA0+06Ja3gDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"971ad7a417d8509593d392dc8132aac3048e7ee8a64b6f2e3b6b0bfbd3ab2115","last_reissued_at":"2026-07-05T09:20:40.341713Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:20:40.341713Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2402.11924","source_version":5,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:20:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Y5QQhxTWlo5jSU/jdsKrHIuaGYl5b+iipkn62zRzVQqpNyCp/H1ikiPD+RZHcRxEyYKopslQZq5d9PwE05GmAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T13:52:06.404605Z"},"content_sha256":"a2914653e6e63349ec4c6b89e493a640fb0fd2a7aa7b693439e63786968a4aa3","schema_version":"1.0","event_id":"sha256:a2914653e6e63349ec4c6b89e493a640fb0fd2a7aa7b693439e63786968a4aa3"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:S4NNPJAX3BIJLE6TSLOICMVKYM","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Cofca: A Step-Wise Counterfactual Multi-hop QA benchmark","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Jian Wu, Linyi Yang, Manabu Okumura, Yue Zhang, Zhen Wang","submitted_at":"2024-02-19T08:12:30Z","abstract_excerpt":"While Large Language Models (LLMs) excel in question-answering (QA) tasks, their real reasoning abilities on multiple evidence retrieval and integration on Multi-hop QA tasks remain less explored. Firstly, LLMs sometimes generate answers that rely on internal memory rather than retrieving evidence and reasoning in the given context, which brings concerns about the evaluation quality of real reasoning abilities. Although previous counterfactual QA benchmarks can separate the internal memory of LLMs, they focus solely on final QA performance, which is insufficient for reporting LLMs' real reason"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.11924","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.11924/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:20:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"oXs/wlOeF/DJV9EMNmyzLRdCwOfDvbI2u0YdSAHJPp46MmoreRYHVhh3t1NsRrObB+PS7dpWRIRVJccNEB0VCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T13:52:06.404904Z"},"content_sha256":"a83c15415abc8e553903a10947f964a89965f5fab06807adc01388be95876c63","schema_version":"1.0","event_id":"sha256:a83c15415abc8e553903a10947f964a89965f5fab06807adc01388be95876c63"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/S4NNPJAX3BIJLE6TSLOICMVKYM/bundle.json","state_url":"https://pith.science/pith/S4NNPJAX3BIJLE6TSLOICMVKYM/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/S4NNPJAX3BIJLE6TSLOICMVKYM/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-06T13:52:06Z","links":{"resolver":"https://pith.science/pith/S4NNPJAX3BIJLE6TSLOICMVKYM","bundle":"https://pith.science/pith/S4NNPJAX3BIJLE6TSLOICMVKYM/bundle.json","state":"https://pith.science/pith/S4NNPJAX3BIJLE6TSLOICMVKYM/state.json","well_known_bundle":"https://pith.science/.well-known/pith/S4NNPJAX3BIJLE6TSLOICMVKYM/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:S4NNPJAX3BIJLE6TSLOICMVKYM","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"ffadffb528a8addca8be2bb329b363dd979ea641a5b573c7c8a406d1d080946c","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-19T08:12:30Z","title_canon_sha256":"32fad87ec3217e76c62c4d24c2eae71f512e34c30d6f9d2ad8335c302bdc93ce"},"schema_version":"1.0","source":{"id":"2402.11924","kind":"arxiv","version":5}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.11924","created_at":"2026-07-05T09:20:40Z"},{"alias_kind":"arxiv_version","alias_value":"2402.11924v5","created_at":"2026-07-05T09:20:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.11924","created_at":"2026-07-05T09:20:40Z"},{"alias_kind":"pith_short_12","alias_value":"S4NNPJAX3BIJ","created_at":"2026-07-05T09:20:40Z"},{"alias_kind":"pith_short_16","alias_value":"S4NNPJAX3BIJLE6T","created_at":"2026-07-05T09:20:40Z"},{"alias_kind":"pith_short_8","alias_value":"S4NNPJAX","created_at":"2026-07-05T09:20:40Z"}],"graph_snapshots":[{"event_id":"sha256:a83c15415abc8e553903a10947f964a89965f5fab06807adc01388be95876c63","target":"graph","created_at":"2026-07-05T09:20:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.11924/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"While Large Language Models (LLMs) excel in question-answering (QA) tasks, their real reasoning abilities on multiple evidence retrieval and integration on Multi-hop QA tasks remain less explored. Firstly, LLMs sometimes generate answers that rely on internal memory rather than retrieving evidence and reasoning in the given context, which brings concerns about the evaluation quality of real reasoning abilities. Although previous counterfactual QA benchmarks can separate the internal memory of LLMs, they focus solely on final QA performance, which is insufficient for reporting LLMs' real reason","authors_text":"Jian Wu, Linyi Yang, Manabu Okumura, Yue Zhang, Zhen Wang","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-19T08:12:30Z","title":"Cofca: A Step-Wise Counterfactual Multi-hop QA benchmark"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.11924","kind":"arxiv","version":5},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:a2914653e6e63349ec4c6b89e493a640fb0fd2a7aa7b693439e63786968a4aa3","target":"record","created_at":"2026-07-05T09:20:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"ffadffb528a8addca8be2bb329b363dd979ea641a5b573c7c8a406d1d080946c","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-19T08:12:30Z","title_canon_sha256":"32fad87ec3217e76c62c4d24c2eae71f512e34c30d6f9d2ad8335c302bdc93ce"},"schema_version":"1.0","source":{"id":"2402.11924","kind":"arxiv","version":5}},"canonical_sha256":"971ad7a417d8509593d392dc8132aac3048e7ee8a64b6f2e3b6b0bfbd3ab2115","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"971ad7a417d8509593d392dc8132aac3048e7ee8a64b6f2e3b6b0bfbd3ab2115","first_computed_at":"2026-07-05T09:20:40.341713Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:20:40.341713Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"MEVqewX8gOvQsH5dwibKGb352BUzUNPuH2bBqNnZ1mvCQ2uVUG/umeev26hejoUazJsBJIfmxqSA0+06Ja3gDg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:20:40.342203Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.11924","source_kind":"arxiv","source_version":5}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:a2914653e6e63349ec4c6b89e493a640fb0fd2a7aa7b693439e63786968a4aa3","sha256:a83c15415abc8e553903a10947f964a89965f5fab06807adc01388be95876c63"],"state_sha256":"50d8cf58ff99b9b960bc95cb99dec678bccab83def0dddf68ec80f5464c2506a"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"kuQhkgq4fhG4hN/fDxAsVg2/kRgikQKyvYTDgKYK8gZSy/4Ch6DECHSMMp9tNSYsTwMV7DpXa3ARE3mLGvMxAg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-06T13:52:06.408073Z","bundle_sha256":"5583aa5c3da2ba8869ea0345293d386e802ef2caf2a5ed64e6f1cae61ff0ddd4"}}