{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:7QMVCB33PDQ5ZYPIXLOSF6LAJ7","short_pith_number":"pith:7QMVCB33","schema_version":"1.0","canonical_sha256":"fc1951077b78e1dce1e8badd22f9604ff01ee271ce4ccec24b7cf684b71b980c","source":{"kind":"arxiv","id":"2409.04774","version":1},"attestation_state":"computed","paper":{"title":"Untie the Knots: An Efficient Data Augmentation Strategy for Long-Context Pre-Training in Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Colin Zhang, Da Zheng, Debing Zhang, Junfeng Tian, Rui Wang, Yang Cheng","submitted_at":"2024-09-07T09:28:55Z","abstract_excerpt":"Large language models (LLM) have prioritized expanding the context window from which models can incorporate more information. However, training models to handle long contexts presents significant challenges. These include the scarcity of high-quality natural long-context data, the potential for performance degradation on short-context tasks, and the reduced training efficiency associated with attention mechanisms. In this paper, we introduce Untie the Knots (\\textbf{UtK}), a novel data augmentation strategy employed during the continue pre-training phase, designed to efficiently enable LLMs to"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2409.04774","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-07T09:28:55Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"d404cadb94f8aceabc9e87077d8cc0900d723f9336931e20e034ee04adb55d74","abstract_canon_sha256":"fac3e681a93c2d99b76179492f8578f3c21ec992be438494f3a779e74c33cf75"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:04:17.795069Z","signature_b64":"GOkZLL0xUK69yAGyro/2MW8usqPLY1TMM32FQtp9SHipA37FUFF/cGUPfoSG15CMDPLpXLGWe3ZT+MUdiM7XDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"fc1951077b78e1dce1e8badd22f9604ff01ee271ce4ccec24b7cf684b71b980c","last_reissued_at":"2026-07-05T09:04:17.794625Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:04:17.794625Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Untie the Knots: An Efficient Data Augmentation Strategy for Long-Context Pre-Training in Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Colin Zhang, Da Zheng, Debing Zhang, Junfeng Tian, Rui Wang, Yang Cheng","submitted_at":"2024-09-07T09:28:55Z","abstract_excerpt":"Large language models (LLM) have prioritized expanding the context window from which models can incorporate more information. However, training models to handle long contexts presents significant challenges. These include the scarcity of high-quality natural long-context data, the potential for performance degradation on short-context tasks, and the reduced training efficiency associated with attention mechanisms. In this paper, we introduce Untie the Knots (\\textbf{UtK}), a novel data augmentation strategy employed during the continue pre-training phase, designed to efficiently enable LLMs to"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.04774","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2409.04774/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2409.04774","created_at":"2026-07-05T09:04:17.794685+00:00"},{"alias_kind":"arxiv_version","alias_value":"2409.04774v1","created_at":"2026-07-05T09:04:17.794685+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.04774","created_at":"2026-07-05T09:04:17.794685+00:00"},{"alias_kind":"pith_short_12","alias_value":"7QMVCB33PDQ5","created_at":"2026-07-05T09:04:17.794685+00:00"},{"alias_kind":"pith_short_16","alias_value":"7QMVCB33PDQ5ZYPI","created_at":"2026-07-05T09:04:17.794685+00:00"},{"alias_kind":"pith_short_8","alias_value":"7QMVCB33","created_at":"2026-07-05T09:04:17.794685+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2501.12766","citing_title":"NExtLong: Toward Effective Long-Context Training without Long Documents","ref_index":91,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/7QMVCB33PDQ5ZYPIXLOSF6LAJ7","json":"https://pith.science/pith/7QMVCB33PDQ5ZYPIXLOSF6LAJ7.json","graph_json":"https://pith.science/api/pith-number/7QMVCB33PDQ5ZYPIXLOSF6LAJ7/graph.json","events_json":"https://pith.science/api/pith-number/7QMVCB33PDQ5ZYPIXLOSF6LAJ7/events.json","paper":"https://pith.science/paper/7QMVCB33"},"agent_actions":{"view_html":"https://pith.science/pith/7QMVCB33PDQ5ZYPIXLOSF6LAJ7","download_json":"https://pith.science/pith/7QMVCB33PDQ5ZYPIXLOSF6LAJ7.json","view_paper":"https://pith.science/paper/7QMVCB33","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2409.04774&json=true","fetch_graph":"https://pith.science/api/pith-number/7QMVCB33PDQ5ZYPIXLOSF6LAJ7/graph.json","fetch_events":"https://pith.science/api/pith-number/7QMVCB33PDQ5ZYPIXLOSF6LAJ7/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/7QMVCB33PDQ5ZYPIXLOSF6LAJ7/action/timestamp_anchor","attest_storage":"https://pith.science/pith/7QMVCB33PDQ5ZYPIXLOSF6LAJ7/action/storage_attestation","attest_author":"https://pith.science/pith/7QMVCB33PDQ5ZYPIXLOSF6LAJ7/action/author_attestation","sign_citation":"https://pith.science/pith/7QMVCB33PDQ5ZYPIXLOSF6LAJ7/action/citation_signature","submit_replication":"https://pith.science/pith/7QMVCB33PDQ5ZYPIXLOSF6LAJ7/action/replication_record"}},"created_at":"2026-07-05T09:04:17.794685+00:00","updated_at":"2026-07-05T09:04:17.794685+00:00"}