{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:LRIOB6JHGFKNMWPKKKZ3TF4CVL","short_pith_number":"pith:LRIOB6JH","schema_version":"1.0","canonical_sha256":"5c50e0f9273154d659ea52b3b99782aaed76773ff060ba7416e15d4fe4818d24","source":{"kind":"arxiv","id":"2405.21068","version":1},"attestation_state":"computed","paper":{"title":"Code Pretraining Improves Entity Tracking Abilities of Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Najoung Kim, Sebastian Schuster, Shubham Toshniwal","submitted_at":"2024-05-31T17:56:33Z","abstract_excerpt":"Recent work has provided indirect evidence that pretraining language models on code improves the ability of models to track state changes of discourse entities expressed in natural language. In this work, we systematically test this claim by comparing pairs of language models on their entity tracking performance. Critically, the pairs consist of base models and models trained on top of these base models with additional code data. We extend this analysis to additionally examine the effect of math training, another highly structured data type, and alignment tuning, an important step for enhancin"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2405.21068","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-05-31T17:56:33Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"4808a3040cb1cac9da0b96aae78b50d5d0bd5230b3fff8fc677ae08643f890b5","abstract_canon_sha256":"71ba20301c0b67ebbd39ab8893fcb36267b6cdd57b4477ddce2b5b4730b371db"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:25:45.807166Z","signature_b64":"evkLvBcjAV0dYBjEUUb3ShUkq6x8KS3dnsxq8r/A2cfwTLhjujjh6+7paKnod3PbJO27i4sHxVkDNgPRoIrFCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"5c50e0f9273154d659ea52b3b99782aaed76773ff060ba7416e15d4fe4818d24","last_reissued_at":"2026-07-05T08:25:45.806677Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:25:45.806677Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Code Pretraining Improves Entity Tracking Abilities of Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Najoung Kim, Sebastian Schuster, Shubham Toshniwal","submitted_at":"2024-05-31T17:56:33Z","abstract_excerpt":"Recent work has provided indirect evidence that pretraining language models on code improves the ability of models to track state changes of discourse entities expressed in natural language. In this work, we systematically test this claim by comparing pairs of language models on their entity tracking performance. Critically, the pairs consist of base models and models trained on top of these base models with additional code data. We extend this analysis to additionally examine the effect of math training, another highly structured data type, and alignment tuning, an important step for enhancin"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.21068","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.21068/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2405.21068","created_at":"2026-07-05T08:25:45.806741+00:00"},{"alias_kind":"arxiv_version","alias_value":"2405.21068v1","created_at":"2026-07-05T08:25:45.806741+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.21068","created_at":"2026-07-05T08:25:45.806741+00:00"},{"alias_kind":"pith_short_12","alias_value":"LRIOB6JHGFKN","created_at":"2026-07-05T08:25:45.806741+00:00"},{"alias_kind":"pith_short_16","alias_value":"LRIOB6JHGFKNMWPK","created_at":"2026-07-05T08:25:45.806741+00:00"},{"alias_kind":"pith_short_8","alias_value":"LRIOB6JH","created_at":"2026-07-05T08:25:45.806741+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":5,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.24408","citing_title":"Natural Identifiers for Privacy and Data Audits in Large Language Models","ref_index":12,"is_internal_anchor":false},{"citing_arxiv_id":"2606.08644","citing_title":"A retrieval conditioned rebinding circuit for dynamic entity tracking in large language models","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2506.22598","citing_title":"RExBench: Can coding agents autonomously implement AI research extensions?","ref_index":24,"is_internal_anchor":false},{"citing_arxiv_id":"2605.10061","citing_title":"Not-So-Strange Love: Language Models and Generative Linguistic Theories are More Compatible than They Appear","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2604.15503","citing_title":"Brain Score Tracks Shared Properties of Languages: Evidence from Many Natural Languages and Structured Sequences","ref_index":14,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/LRIOB6JHGFKNMWPKKKZ3TF4CVL","json":"https://pith.science/pith/LRIOB6JHGFKNMWPKKKZ3TF4CVL.json","graph_json":"https://pith.science/api/pith-number/LRIOB6JHGFKNMWPKKKZ3TF4CVL/graph.json","events_json":"https://pith.science/api/pith-number/LRIOB6JHGFKNMWPKKKZ3TF4CVL/events.json","paper":"https://pith.science/paper/LRIOB6JH"},"agent_actions":{"view_html":"https://pith.science/pith/LRIOB6JHGFKNMWPKKKZ3TF4CVL","download_json":"https://pith.science/pith/LRIOB6JHGFKNMWPKKKZ3TF4CVL.json","view_paper":"https://pith.science/paper/LRIOB6JH","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2405.21068&json=true","fetch_graph":"https://pith.science/api/pith-number/LRIOB6JHGFKNMWPKKKZ3TF4CVL/graph.json","fetch_events":"https://pith.science/api/pith-number/LRIOB6JHGFKNMWPKKKZ3TF4CVL/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/LRIOB6JHGFKNMWPKKKZ3TF4CVL/action/timestamp_anchor","attest_storage":"https://pith.science/pith/LRIOB6JHGFKNMWPKKKZ3TF4CVL/action/storage_attestation","attest_author":"https://pith.science/pith/LRIOB6JHGFKNMWPKKKZ3TF4CVL/action/author_attestation","sign_citation":"https://pith.science/pith/LRIOB6JHGFKNMWPKKKZ3TF4CVL/action/citation_signature","submit_replication":"https://pith.science/pith/LRIOB6JHGFKNMWPKKKZ3TF4CVL/action/replication_record"}},"created_at":"2026-07-05T08:25:45.806741+00:00","updated_at":"2026-07-05T08:25:45.806741+00:00"}