{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:WBVZ35QRIY7JBSSRTOU4CV7BXR","short_pith_number":"pith:WBVZ35QR","schema_version":"1.0","canonical_sha256":"b06b9df611463e90ca519ba9c157e1bc6376a79ee8cc7364bd40f015528eaa6b","source":{"kind":"arxiv","id":"2410.15578","version":1},"attestation_state":"computed","paper":{"title":"Generalized Probabilistic Attention Mechanism in Transformers","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.LG","authors_text":"DongNyeong Heo, Heeyoul Choi","submitted_at":"2024-10-21T01:55:52Z","abstract_excerpt":"The Transformer architecture has become widely adopted due to its demonstrated success, attributed to the attention mechanism at its core. Despite these successes, the attention mechanism of Transformers is associated with two well-known issues: rank-collapse and gradient vanishing. In this paper, we present a theoretical analysis that it is inherently difficult to address both issues simultaneously in the conventional attention mechanism. To handle these issues, we introduce a novel class of attention mechanism, referred to as generalized probabilistic attention mechanism (GPAM), and its dual"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2410.15578","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-21T01:55:52Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"2fe46d1aa190f2432b4fd61db9f2b0f760d164049e5b92ca10b3f5cfb011e7cf","abstract_canon_sha256":"d2661d5c80d29606de763b71949fafba5fbadc8e3cbdccbbd5d21ab2a9b54314"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:23:21.632809Z","signature_b64":"Hp4c+rG4RTpItb3cEZESGZRiWVP8X8Gh4eokqcLnjxwvDXksmWMUEEsPJl0J7wj8SEmmVpMyrsrCod0n9TdACw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b06b9df611463e90ca519ba9c157e1bc6376a79ee8cc7364bd40f015528eaa6b","last_reissued_at":"2026-07-05T09:23:21.632376Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:23:21.632376Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Generalized Probabilistic Attention Mechanism in Transformers","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.LG","authors_text":"DongNyeong Heo, Heeyoul Choi","submitted_at":"2024-10-21T01:55:52Z","abstract_excerpt":"The Transformer architecture has become widely adopted due to its demonstrated success, attributed to the attention mechanism at its core. Despite these successes, the attention mechanism of Transformers is associated with two well-known issues: rank-collapse and gradient vanishing. In this paper, we present a theoretical analysis that it is inherently difficult to address both issues simultaneously in the conventional attention mechanism. To handle these issues, we introduce a novel class of attention mechanism, referred to as generalized probabilistic attention mechanism (GPAM), and its dual"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.15578","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.15578/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2410.15578","created_at":"2026-07-05T09:23:21.632440+00:00"},{"alias_kind":"arxiv_version","alias_value":"2410.15578v1","created_at":"2026-07-05T09:23:21.632440+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.15578","created_at":"2026-07-05T09:23:21.632440+00:00"},{"alias_kind":"pith_short_12","alias_value":"WBVZ35QRIY7J","created_at":"2026-07-05T09:23:21.632440+00:00"},{"alias_kind":"pith_short_16","alias_value":"WBVZ35QRIY7JBSSR","created_at":"2026-07-05T09:23:21.632440+00:00"},{"alias_kind":"pith_short_8","alias_value":"WBVZ35QR","created_at":"2026-07-05T09:23:21.632440+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2505.16284","citing_title":"Only Large Weights (And Not Skip Connections) Can Prevent the Perils of Rank Collapse","ref_index":32,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/WBVZ35QRIY7JBSSRTOU4CV7BXR","json":"https://pith.science/pith/WBVZ35QRIY7JBSSRTOU4CV7BXR.json","graph_json":"https://pith.science/api/pith-number/WBVZ35QRIY7JBSSRTOU4CV7BXR/graph.json","events_json":"https://pith.science/api/pith-number/WBVZ35QRIY7JBSSRTOU4CV7BXR/events.json","paper":"https://pith.science/paper/WBVZ35QR"},"agent_actions":{"view_html":"https://pith.science/pith/WBVZ35QRIY7JBSSRTOU4CV7BXR","download_json":"https://pith.science/pith/WBVZ35QRIY7JBSSRTOU4CV7BXR.json","view_paper":"https://pith.science/paper/WBVZ35QR","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2410.15578&json=true","fetch_graph":"https://pith.science/api/pith-number/WBVZ35QRIY7JBSSRTOU4CV7BXR/graph.json","fetch_events":"https://pith.science/api/pith-number/WBVZ35QRIY7JBSSRTOU4CV7BXR/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/WBVZ35QRIY7JBSSRTOU4CV7BXR/action/timestamp_anchor","attest_storage":"https://pith.science/pith/WBVZ35QRIY7JBSSRTOU4CV7BXR/action/storage_attestation","attest_author":"https://pith.science/pith/WBVZ35QRIY7JBSSRTOU4CV7BXR/action/author_attestation","sign_citation":"https://pith.science/pith/WBVZ35QRIY7JBSSRTOU4CV7BXR/action/citation_signature","submit_replication":"https://pith.science/pith/WBVZ35QRIY7JBSSRTOU4CV7BXR/action/replication_record"}},"created_at":"2026-07-05T09:23:21.632440+00:00","updated_at":"2026-07-05T09:23:21.632440+00:00"}