{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:G2XA7TKEYV53AO5KYTG6OUGBP5","short_pith_number":"pith:G2XA7TKE","schema_version":"1.0","canonical_sha256":"36ae0fcd44c57bb03baac4cde750c17f5fc633d6e2b8c874bcd85ed980ec3b75","source":{"kind":"arxiv","id":"2412.15204","version":2},"attestation_state":"computed","paper":{"title":"LongBench v2: Towards Deeper Understanding and Reasoning on Realistic Long-context Multitasks","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"LongBench v2 shows current LLMs score 50% on long-context reasoning tasks while reasoning models exceed the 54% human baseline.","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Hao Peng, Jiajie Zhang, Jiazheng Xu, Jie Tang, Juanzi Li, Lei Hou, Shangqing Tu, Shulin Cao, Xiaozhi Wang, Xin Lv, Yushi Bai, Yuxiao Dong","submitted_at":"2024-12-19T18:59:17Z","abstract_excerpt":"This paper introduces LongBench v2, a benchmark designed to assess the ability of LLMs to handle long-context problems requiring deep understanding and reasoning across real-world multitasks. LongBench v2 consists of 503 challenging multiple-choice questions, with contexts ranging from 8k to 2M words, across six major task categories: single-document QA, multi-document QA, long in-context learning, long-dialogue history understanding, code repository understanding, and long structured data understanding. To ensure the breadth and the practicality, we collect data from nearly 100 highly educate"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":true,"formal_links_present":true},"canonical_record":{"source":{"id":"2412.15204","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-12-19T18:59:17Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"4998e049c23af4c78fd2e5f612dad7ae2284185f686b6fa03754a436ae679944","abstract_canon_sha256":"7765192ced9a40be15cb5d5ecd09e4647b36f808cf665312205b0b87976cb5f6"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:38:46.654671Z","signature_b64":"e0BHOsTvvmmN0O6bFH0Yk/Pn4msMlvUY2+9o6k2XXjbfEB4SK+LKevyo+Z9/fqWG+fmWXU1upyAAtg4OVB7MDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"36ae0fcd44c57bb03baac4cde750c17f5fc633d6e2b8c874bcd85ed980ec3b75","last_reissued_at":"2026-05-17T23:38:46.654233Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:38:46.654233Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"LongBench v2: Towards Deeper Understanding and Reasoning on Realistic Long-context Multitasks","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"LongBench v2 shows current LLMs score 50% on long-context reasoning tasks while reasoning models exceed the 54% human baseline.","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Hao Peng, Jiajie Zhang, Jiazheng Xu, Jie Tang, Juanzi Li, Lei Hou, Shangqing Tu, Shulin Cao, Xiaozhi Wang, Xin Lv, Yushi Bai, Yuxiao Dong","submitted_at":"2024-12-19T18:59:17Z","abstract_excerpt":"This paper introduces LongBench v2, a benchmark designed to assess the ability of LLMs to handle long-context problems requiring deep understanding and reasoning across real-world multitasks. LongBench v2 consists of 503 challenging multiple-choice questions, with contexts ranging from 8k to 2M words, across six major task categories: single-document QA, multi-document QA, long in-context learning, long-dialogue history understanding, code repository understanding, and long structured data understanding. To ensure the breadth and the practicality, we collect data from nearly 100 highly educate"},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"The best-performing model, when directly answers the questions, achieves only 50.1% accuracy. In contrast, the o1-preview model, which includes longer reasoning, achieves 57.7%, surpassing the human baseline by 4%.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"That the 503 questions genuinely require deep understanding and multi-step reasoning rather than being solvable through surface cues or training-data leakage, and that the 15-minute human time limit produces a fair comparison to model performance.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"LongBench v2 benchmark shows current LLMs underperform humans on deep long-context reasoning tasks, but extended inference-time reasoning enables surpassing the human baseline.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"LongBench v2 shows current LLMs score 50% on long-context reasoning tasks while reasoning models exceed the 54% human baseline.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"54b1e97657495c00fa6902432bc3adeed38c363a8c001933653dda0d1a4c2117"},"source":{"id":"2412.15204","kind":"arxiv","version":2},"verdict":{"id":"143973a6-ec2d-413c-b051-ac0698797f19","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-16T20:34:54.088333Z","strongest_claim":"The best-performing model, when directly answers the questions, achieves only 50.1% accuracy. In contrast, the o1-preview model, which includes longer reasoning, achieves 57.7%, surpassing the human baseline by 4%.","one_line_summary":"LongBench v2 benchmark shows current LLMs underperform humans on deep long-context reasoning tasks, but extended inference-time reasoning enables surpassing the human baseline.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"That the 503 questions genuinely require deep understanding and multi-step reasoning rather than being solvable through surface cues or training-data leakage, and that the 15-minute human time limit produces a fair comparison to model performance.","pith_extraction_headline":"LongBench v2 shows current LLMs score 50% on long-context reasoning tasks while reasoning models exceed the 54% human baseline."},"references":{"count":23,"sample":[{"doi":"","year":2024,"title":"Agrawal, P., Craig, N., Madden, A., and Lombera, I","work_id":"1ff25d89-9966-4623-9f61-876eb43549b4","ref_index":1,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2021,"title":"The Llama 3 Herd of Models","work_id":"1549a635-88af-4ac1-acfe-51ae7bb53345","ref_index":2,"cited_arxiv_id":"2407.21783","is_internal_anchor":true},{"doi":"","year":2024,"title":"ChatGLM: A Family of Large Language Models from GLM-130B to GLM-4 All Tools","work_id":"de9ce5af-0d8d-4b94-9793-64968d9bc06d","ref_index":3,"cited_arxiv_id":"2406.12793","is_internal_anchor":true},{"doi":"","year":2024,"title":"RepoBench: Benchmarking Repository-Level Code Auto-Completion Systems","work_id":"24b24ba5-b21c-43c6-866f-4b874305372e","ref_index":4,"cited_arxiv_id":"2306.03091","is_internal_anchor":true},{"doi":"","year":2024,"title":"In Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (V olume 1: Long Papers), pages 11621–11640, Bangkok, Thailand","work_id":"4ad83798-4e7c-471d-97e7-97ca3a0a1127","ref_index":5,"cited_arxiv_id":"","is_internal_anchor":false}],"resolved_work":23,"snapshot_sha256":"8e30c0b650334039f4a8687c289f063ab987dc14efa5fddade629e7581467b79","internal_anchors":3},"formal_canon":{"evidence_count":1,"snapshot_sha256":"cd08eaf3270df4b78a2cfb664e5934400260c13312d1d9b889577aae2b8d8e40"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2412.15204","created_at":"2026-05-17T23:38:46.654304+00:00"},{"alias_kind":"arxiv_version","alias_value":"2412.15204v2","created_at":"2026-05-17T23:38:46.654304+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.15204","created_at":"2026-05-17T23:38:46.654304+00:00"},{"alias_kind":"pith_short_12","alias_value":"G2XA7TKEYV53","created_at":"2026-05-18T12:33:37.589309+00:00"},{"alias_kind":"pith_short_16","alias_value":"G2XA7TKEYV53AO5K","created_at":"2026-05-18T12:33:37.589309+00:00"},{"alias_kind":"pith_short_8","alias_value":"G2XA7TKE","created_at":"2026-05-18T12:33:37.589309+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":26,"internal_anchor_count":26,"sample":[{"citing_arxiv_id":"2411.15594","citing_title":"A Survey on LLM-as-a-Judge","ref_index":8,"is_internal_anchor":true},{"citing_arxiv_id":"2502.01941","citing_title":"Semantic Integrity Matters: Benchmarking and Preserving High-Density Reasoning in KV Cache Compression","ref_index":35,"is_internal_anchor":true},{"citing_arxiv_id":"2605.20315","citing_title":"Mix-Quant: Quantized Prefilling, Precise Decoding for Agentic LLMs","ref_index":2,"is_internal_anchor":true},{"citing_arxiv_id":"2605.20104","citing_title":"Draft Less, Retrieve More: Hybrid Tree Construction for Speculative Decoding","ref_index":1,"is_internal_anchor":true},{"citing_arxiv_id":"2605.16839","citing_title":"CompactAttention: Accelerating Chunked Prefill with Block-Union KV Selection","ref_index":23,"is_internal_anchor":true},{"citing_arxiv_id":"2509.08016","citing_title":"Video Parallel Scaling: Aggregating Diverse Frame Subsets for VideoLLMs","ref_index":2,"is_internal_anchor":true},{"citing_arxiv_id":"2512.12087","citing_title":"BLASST: Dynamic BLocked Attention Sparsity via Softmax Thresholding","ref_index":2,"is_internal_anchor":true},{"citing_arxiv_id":"2512.16056","citing_title":"MultiPath Memory Access: Breaking Host-GPU Bandwidth Bottlenecks in LLM Services","ref_index":6,"is_internal_anchor":true},{"citing_arxiv_id":"2507.05257","citing_title":"Evaluating Memory in LLM Agents via Incremental Multi-Turn Interactions","ref_index":2,"is_internal_anchor":true},{"citing_arxiv_id":"2601.20251","citing_title":"Efficient Evaluation of LLM Performance with Statistical Guarantees","ref_index":3,"is_internal_anchor":true},{"citing_arxiv_id":"2602.22575","citing_title":"S2O: Early Stopping for Sparse Attention via Online Permutation","ref_index":1,"is_internal_anchor":true},{"citing_arxiv_id":"2604.02371","citing_title":"Internalized Reasoning for Long-Context Visual Document Understanding","ref_index":3,"is_internal_anchor":true},{"citing_arxiv_id":"2510.26692","citing_title":"Kimi Linear: An Expressive, Efficient Attention Architecture","ref_index":6,"is_internal_anchor":true},{"citing_arxiv_id":"2506.13585","citing_title":"MiniMax-M1: Scaling Test-Time Compute Efficiently with Lightning Attention","ref_index":2,"is_internal_anchor":true},{"citing_arxiv_id":"2604.26837","citing_title":"Unifying Sparse Attention with Hierarchical Memory for Scalable Long-Context LLM Serving","ref_index":7,"is_internal_anchor":true},{"citing_arxiv_id":"2605.09649","citing_title":"Make Each Token Count: Towards Improving Long-Context Performance with KV Cache Eviction","ref_index":1,"is_internal_anchor":true},{"citing_arxiv_id":"2604.21231","citing_title":"SparKV: Overhead-Aware KV Cache Loading for Efficient On-Device LLM Inference","ref_index":48,"is_internal_anchor":true},{"citing_arxiv_id":"2605.04426","citing_title":"Telegraph English: Semantic Prompt Compression via Structured Symbolic Rewriting","ref_index":1,"is_internal_anchor":true},{"citing_arxiv_id":"2604.07809","citing_title":"PolicyLong: Towards On-Policy Context Extension","ref_index":1,"is_internal_anchor":true},{"citing_arxiv_id":"2604.07593","citing_title":"Too long; didn't solve","ref_index":1,"is_internal_anchor":true},{"citing_arxiv_id":"2605.07719","citing_title":"An Efficient Hybrid Sparse Attention with CPU-GPU Parallelism for Long-Context Inference","ref_index":3,"is_internal_anchor":true},{"citing_arxiv_id":"2507.20534","citing_title":"Kimi K2: Open Agentic Intelligence","ref_index":2,"is_internal_anchor":true},{"citing_arxiv_id":"2602.02276","citing_title":"Kimi K2.5: Visual Agentic Intelligence","ref_index":9,"is_internal_anchor":true},{"citing_arxiv_id":"2604.16007","citing_title":"MemExplorer: Navigating the Heterogeneous Memory Design Space for Agentic Inference NPUs","ref_index":6,"is_internal_anchor":true},{"citing_arxiv_id":"2604.18234","citing_title":"Evaluating Multi-Hop Reasoning in RAG Systems: A Comparison of LLM-Based Retriever Evaluation Strategies","ref_index":4,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":1,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/G2XA7TKEYV53AO5KYTG6OUGBP5","json":"https://pith.science/pith/G2XA7TKEYV53AO5KYTG6OUGBP5.json","graph_json":"https://pith.science/api/pith-number/G2XA7TKEYV53AO5KYTG6OUGBP5/graph.json","events_json":"https://pith.science/api/pith-number/G2XA7TKEYV53AO5KYTG6OUGBP5/events.json","paper":"https://pith.science/paper/G2XA7TKE"},"agent_actions":{"view_html":"https://pith.science/pith/G2XA7TKEYV53AO5KYTG6OUGBP5","download_json":"https://pith.science/pith/G2XA7TKEYV53AO5KYTG6OUGBP5.json","view_paper":"https://pith.science/paper/G2XA7TKE","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2412.15204&json=true","fetch_graph":"https://pith.science/api/pith-number/G2XA7TKEYV53AO5KYTG6OUGBP5/graph.json","fetch_events":"https://pith.science/api/pith-number/G2XA7TKEYV53AO5KYTG6OUGBP5/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/G2XA7TKEYV53AO5KYTG6OUGBP5/action/timestamp_anchor","attest_storage":"https://pith.science/pith/G2XA7TKEYV53AO5KYTG6OUGBP5/action/storage_attestation","attest_author":"https://pith.science/pith/G2XA7TKEYV53AO5KYTG6OUGBP5/action/author_attestation","sign_citation":"https://pith.science/pith/G2XA7TKEYV53AO5KYTG6OUGBP5/action/citation_signature","submit_replication":"https://pith.science/pith/G2XA7TKEYV53AO5KYTG6OUGBP5/action/replication_record"}},"created_at":"2026-05-17T23:38:46.654304+00:00","updated_at":"2026-05-17T23:38:46.654304+00:00"}