{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:FRY23P3V2NZE5GNCF3WV3OUKA2","short_pith_number":"pith:FRY23P3V","schema_version":"1.0","canonical_sha256":"2c71adbf75d3724e99a22eed5dba8a06aeca9b620803f1ee6707a9df0fa184f2","source":{"kind":"arxiv","id":"2404.01139","version":1},"attestation_state":"computed","paper":{"title":"Structured Initialization for Attention in Vision Transformers","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CV","authors_text":"Jianqiao Zheng, Simon Lucey, Xueqian Li","submitted_at":"2024-04-01T14:34:47Z","abstract_excerpt":"The training of vision transformer (ViT) networks on small-scale datasets poses a significant challenge. By contrast, convolutional neural networks (CNNs) have an architectural inductive bias enabling them to perform well on such problems. In this paper, we argue that the architectural bias inherent to CNNs can be reinterpreted as an initialization bias within ViT. This insight is significant as it empowers ViTs to perform equally well on small-scale problems while maintaining their flexibility for large-scale applications. Our inspiration for this ``structured'' initialization stems from our "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2404.01139","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2024-04-01T14:34:47Z","cross_cats_sorted":[],"title_canon_sha256":"c34b0f7e6be977acc9374140f919e5eb875aca93ba0fa01fc541f96972dd792f","abstract_canon_sha256":"28ee1977053aeeaa26569cff557a9aabecc1dd6c24e36bc8cdfcf21fea164598"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:03:01.358910Z","signature_b64":"lW7CsTyCapVFmPE8orgN4R9V0k/ikEgP1mqaM4WYa1e9wZ483sBoiNK0+Yj7OHzoNZ7+ECKrgJtXIeiZZR5HDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2c71adbf75d3724e99a22eed5dba8a06aeca9b620803f1ee6707a9df0fa184f2","last_reissued_at":"2026-07-05T08:03:01.358380Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:03:01.358380Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Structured Initialization for Attention in Vision Transformers","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CV","authors_text":"Jianqiao Zheng, Simon Lucey, Xueqian Li","submitted_at":"2024-04-01T14:34:47Z","abstract_excerpt":"The training of vision transformer (ViT) networks on small-scale datasets poses a significant challenge. By contrast, convolutional neural networks (CNNs) have an architectural inductive bias enabling them to perform well on such problems. In this paper, we argue that the architectural bias inherent to CNNs can be reinterpreted as an initialization bias within ViT. This insight is significant as it empowers ViTs to perform equally well on small-scale problems while maintaining their flexibility for large-scale applications. Our inspiration for this ``structured'' initialization stems from our "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.01139","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2404.01139/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2404.01139","created_at":"2026-07-05T08:03:01.358441+00:00"},{"alias_kind":"arxiv_version","alias_value":"2404.01139v1","created_at":"2026-07-05T08:03:01.358441+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.01139","created_at":"2026-07-05T08:03:01.358441+00:00"},{"alias_kind":"pith_short_12","alias_value":"FRY23P3V2NZE","created_at":"2026-07-05T08:03:01.358441+00:00"},{"alias_kind":"pith_short_16","alias_value":"FRY23P3V2NZE5GNC","created_at":"2026-07-05T08:03:01.358441+00:00"},{"alias_kind":"pith_short_8","alias_value":"FRY23P3V","created_at":"2026-07-05T08:03:01.358441+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2405.13901","citing_title":"Discrete Cosine Transform Based Decorrelated Attention for Vision Transformers","ref_index":44,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/FRY23P3V2NZE5GNCF3WV3OUKA2","json":"https://pith.science/pith/FRY23P3V2NZE5GNCF3WV3OUKA2.json","graph_json":"https://pith.science/api/pith-number/FRY23P3V2NZE5GNCF3WV3OUKA2/graph.json","events_json":"https://pith.science/api/pith-number/FRY23P3V2NZE5GNCF3WV3OUKA2/events.json","paper":"https://pith.science/paper/FRY23P3V"},"agent_actions":{"view_html":"https://pith.science/pith/FRY23P3V2NZE5GNCF3WV3OUKA2","download_json":"https://pith.science/pith/FRY23P3V2NZE5GNCF3WV3OUKA2.json","view_paper":"https://pith.science/paper/FRY23P3V","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2404.01139&json=true","fetch_graph":"https://pith.science/api/pith-number/FRY23P3V2NZE5GNCF3WV3OUKA2/graph.json","fetch_events":"https://pith.science/api/pith-number/FRY23P3V2NZE5GNCF3WV3OUKA2/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/FRY23P3V2NZE5GNCF3WV3OUKA2/action/timestamp_anchor","attest_storage":"https://pith.science/pith/FRY23P3V2NZE5GNCF3WV3OUKA2/action/storage_attestation","attest_author":"https://pith.science/pith/FRY23P3V2NZE5GNCF3WV3OUKA2/action/author_attestation","sign_citation":"https://pith.science/pith/FRY23P3V2NZE5GNCF3WV3OUKA2/action/citation_signature","submit_replication":"https://pith.science/pith/FRY23P3V2NZE5GNCF3WV3OUKA2/action/replication_record"}},"created_at":"2026-07-05T08:03:01.358441+00:00","updated_at":"2026-07-05T08:03:01.358441+00:00"}