{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:NR3H3PGQIR7FSO64VRPLGQQCDL","short_pith_number":"pith:NR3H3PGQ","canonical_record":{"source":{"id":"2407.17686","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-07-25T01:07:09Z","cross_cats_sorted":["cs.CL","cs.IT","math.IT","stat.ML"],"title_canon_sha256":"37aaf12b2de44b7f0824c41fc929ee94531e17d9f356a2c7cd453e7bc01df874","abstract_canon_sha256":"edafe713c6eb934da58f3efedb5d4fa49dfb8eaf31bd36ecb3403e384bafb791"},"schema_version":"1.0"},"canonical_sha256":"6c767dbcd0447e593bdcac5eb342021ae076910b60abd09c8dace6d1ca23aa60","source":{"kind":"arxiv","id":"2407.17686","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.17686","created_at":"2026-07-05T08:48:19Z"},{"alias_kind":"arxiv_version","alias_value":"2407.17686v1","created_at":"2026-07-05T08:48:19Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.17686","created_at":"2026-07-05T08:48:19Z"},{"alias_kind":"pith_short_12","alias_value":"NR3H3PGQIR7F","created_at":"2026-07-05T08:48:19Z"},{"alias_kind":"pith_short_16","alias_value":"NR3H3PGQIR7FSO64","created_at":"2026-07-05T08:48:19Z"},{"alias_kind":"pith_short_8","alias_value":"NR3H3PGQ","created_at":"2026-07-05T08:48:19Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:NR3H3PGQIR7FSO64VRPLGQQCDL","target":"record","payload":{"canonical_record":{"source":{"id":"2407.17686","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-07-25T01:07:09Z","cross_cats_sorted":["cs.CL","cs.IT","math.IT","stat.ML"],"title_canon_sha256":"37aaf12b2de44b7f0824c41fc929ee94531e17d9f356a2c7cd453e7bc01df874","abstract_canon_sha256":"edafe713c6eb934da58f3efedb5d4fa49dfb8eaf31bd36ecb3403e384bafb791"},"schema_version":"1.0"},"canonical_sha256":"6c767dbcd0447e593bdcac5eb342021ae076910b60abd09c8dace6d1ca23aa60","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:48:19.773549Z","signature_b64":"p20iAiLoFOFLgdtCPvC3/ltZlHlGldU/m+xwXl7LxIZP+7DGqDVHiFByeqT7TjUPvZVpdF15Bz1PFGSSL/emCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6c767dbcd0447e593bdcac5eb342021ae076910b60abd09c8dace6d1ca23aa60","last_reissued_at":"2026-07-05T08:48:19.772994Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:48:19.772994Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2407.17686","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:48:19Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QK97j3emL5bMNpNaPAqTwhYuycxM110plcGdGlgFFzDpBInW04O5cp0UjOd4iu1trP1ZsVjYecc4oF6OIaxiBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T10:00:00.825912Z"},"content_sha256":"87e5414a54bb940dc52862ce690fb7236f461dfd215694a721b4dc3908334c21","schema_version":"1.0","event_id":"sha256:87e5414a54bb940dc52862ce690fb7236f461dfd215694a721b4dc3908334c21"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:NR3H3PGQIR7FSO64VRPLGQQCDL","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Transformers on Markov Data: Constant Depth Suffices","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL","cs.IT","math.IT","stat.ML"],"primary_cat":"cs.LG","authors_text":"Ashok Vardhan Makkuva, Kannan Ramchandran, Marco Bondaschi, Michael Gastpar, Nived Rajaraman","submitted_at":"2024-07-25T01:07:09Z","abstract_excerpt":"Attention-based transformers have been remarkably successful at modeling generative processes across various domains and modalities. In this paper, we study the behavior of transformers on data drawn from \\kth Markov processes, where the conditional distribution of the next symbol in a sequence depends on the previous $k$ symbols observed. We observe a surprising phenomenon empirically which contradicts previous findings: when trained for sufficiently long, a transformer with a fixed depth and $1$ head per layer is able to achieve low test loss on sequences drawn from \\kth Markov sources, even"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.17686","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2407.17686/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:48:19Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"cfHH6Ge/+PDrISbkRxV5/4VKNU6XqfmDMmvZSOdrr5A/6oG4Rg9N2x7mx+wcp/rhWsQ+XeSZxwZ3i4IY8D8kDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T10:00:00.826288Z"},"content_sha256":"2116033f026f8fc6e476753f63731807a9360a7d15bf277c8c6b4cb9b4776347","schema_version":"1.0","event_id":"sha256:2116033f026f8fc6e476753f63731807a9360a7d15bf277c8c6b4cb9b4776347"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/NR3H3PGQIR7FSO64VRPLGQQCDL/bundle.json","state_url":"https://pith.science/pith/NR3H3PGQIR7FSO64VRPLGQQCDL/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/NR3H3PGQIR7FSO64VRPLGQQCDL/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-04T10:00:00Z","links":{"resolver":"https://pith.science/pith/NR3H3PGQIR7FSO64VRPLGQQCDL","bundle":"https://pith.science/pith/NR3H3PGQIR7FSO64VRPLGQQCDL/bundle.json","state":"https://pith.science/pith/NR3H3PGQIR7FSO64VRPLGQQCDL/state.json","well_known_bundle":"https://pith.science/.well-known/pith/NR3H3PGQIR7FSO64VRPLGQQCDL/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:NR3H3PGQIR7FSO64VRPLGQQCDL","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"edafe713c6eb934da58f3efedb5d4fa49dfb8eaf31bd36ecb3403e384bafb791","cross_cats_sorted":["cs.CL","cs.IT","math.IT","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-07-25T01:07:09Z","title_canon_sha256":"37aaf12b2de44b7f0824c41fc929ee94531e17d9f356a2c7cd453e7bc01df874"},"schema_version":"1.0","source":{"id":"2407.17686","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.17686","created_at":"2026-07-05T08:48:19Z"},{"alias_kind":"arxiv_version","alias_value":"2407.17686v1","created_at":"2026-07-05T08:48:19Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.17686","created_at":"2026-07-05T08:48:19Z"},{"alias_kind":"pith_short_12","alias_value":"NR3H3PGQIR7F","created_at":"2026-07-05T08:48:19Z"},{"alias_kind":"pith_short_16","alias_value":"NR3H3PGQIR7FSO64","created_at":"2026-07-05T08:48:19Z"},{"alias_kind":"pith_short_8","alias_value":"NR3H3PGQ","created_at":"2026-07-05T08:48:19Z"}],"graph_snapshots":[{"event_id":"sha256:2116033f026f8fc6e476753f63731807a9360a7d15bf277c8c6b4cb9b4776347","target":"graph","created_at":"2026-07-05T08:48:19Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2407.17686/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Attention-based transformers have been remarkably successful at modeling generative processes across various domains and modalities. In this paper, we study the behavior of transformers on data drawn from \\kth Markov processes, where the conditional distribution of the next symbol in a sequence depends on the previous $k$ symbols observed. We observe a surprising phenomenon empirically which contradicts previous findings: when trained for sufficiently long, a transformer with a fixed depth and $1$ head per layer is able to achieve low test loss on sequences drawn from \\kth Markov sources, even","authors_text":"Ashok Vardhan Makkuva, Kannan Ramchandran, Marco Bondaschi, Michael Gastpar, Nived Rajaraman","cross_cats":["cs.CL","cs.IT","math.IT","stat.ML"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-07-25T01:07:09Z","title":"Transformers on Markov Data: Constant Depth Suffices"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.17686","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:87e5414a54bb940dc52862ce690fb7236f461dfd215694a721b4dc3908334c21","target":"record","created_at":"2026-07-05T08:48:19Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"edafe713c6eb934da58f3efedb5d4fa49dfb8eaf31bd36ecb3403e384bafb791","cross_cats_sorted":["cs.CL","cs.IT","math.IT","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-07-25T01:07:09Z","title_canon_sha256":"37aaf12b2de44b7f0824c41fc929ee94531e17d9f356a2c7cd453e7bc01df874"},"schema_version":"1.0","source":{"id":"2407.17686","kind":"arxiv","version":1}},"canonical_sha256":"6c767dbcd0447e593bdcac5eb342021ae076910b60abd09c8dace6d1ca23aa60","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"6c767dbcd0447e593bdcac5eb342021ae076910b60abd09c8dace6d1ca23aa60","first_computed_at":"2026-07-05T08:48:19.772994Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:48:19.772994Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"p20iAiLoFOFLgdtCPvC3/ltZlHlGldU/m+xwXl7LxIZP+7DGqDVHiFByeqT7TjUPvZVpdF15Bz1PFGSSL/emCQ==","signature_status":"signed_v1","signed_at":"2026-07-05T08:48:19.773549Z","signed_message":"canonical_sha256_bytes"},"source_id":"2407.17686","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:87e5414a54bb940dc52862ce690fb7236f461dfd215694a721b4dc3908334c21","sha256:2116033f026f8fc6e476753f63731807a9360a7d15bf277c8c6b4cb9b4776347"],"state_sha256":"1a642c65b6a8a5f7ed40adbbd6aee6bd9f9cacb7c92c656f3aa207a851d1a1a9"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"T+XtA/LHwqJ5ISHMQ6zWFPBZ5u5d5DvkzZ3jf8i4BZz/LOrmCNhEup+lhNFx07+EywrdXnq6f6UM9TzJK85aAw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-04T10:00:00.828955Z","bundle_sha256":"f38b52b8976c310248b210bd02fad8bddb2f910326544d81c364f696b1d157ad"}}