{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2021:UI44JPAMHIHVUYHY7GZE7NOCYN","short_pith_number":"pith:UI44JPAM","schema_version":"1.0","canonical_sha256":"a239c4bc0c3a0f5a60f8f9b24fb5c2c37ee80f715d40009443c23206d85c6859","source":{"kind":"arxiv","id":"2104.01572","version":1},"attestation_state":"computed","paper":{"title":"TransfoRNN: Capturing the Sequential Information in Self-Attention Representations for Language Modeling","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Junjie Wang, Lin Yang, Tze Yuang Chong, Xuyang Wang","submitted_at":"2021-04-04T09:31:18Z","abstract_excerpt":"In this paper, we describe the use of recurrent neural networks to capture sequential information from the self-attention representations to improve the Transformers. Although self-attention mechanism provides a means to exploit long context, the sequential information, i.e. the arrangement of tokens, is not explicitly captured. We propose to cascade the recurrent neural networks to the Transformers, which referred to as the TransfoRNN model, to capture the sequential information. We found that the TransfoRNN models which consists of only shallow Transformers stack is suffice to give comparabl"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2104.01572","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-04-04T09:31:18Z","cross_cats_sorted":[],"title_canon_sha256":"b9a861b3dbe903f2df0e65202a662e06eb22ce4156057ae5bc5bab626dce6147","abstract_canon_sha256":"2964394c4770e06a87d3c50361bdd56a0041833b65fed19745f35ef923cb5f2c"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:29:01.461738Z","signature_b64":"XOz6uL9NhrV4S+K5CAFWbHkAHkNZNThFoYQHhyrJmpASJ3Y3rjV0zBl+M/9fzO9yrEMlIiLWD23bqVrrwZrUCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a239c4bc0c3a0f5a60f8f9b24fb5c2c37ee80f715d40009443c23206d85c6859","last_reissued_at":"2026-07-05T02:29:01.461159Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:29:01.461159Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"TransfoRNN: Capturing the Sequential Information in Self-Attention Representations for Language Modeling","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Junjie Wang, Lin Yang, Tze Yuang Chong, Xuyang Wang","submitted_at":"2021-04-04T09:31:18Z","abstract_excerpt":"In this paper, we describe the use of recurrent neural networks to capture sequential information from the self-attention representations to improve the Transformers. Although self-attention mechanism provides a means to exploit long context, the sequential information, i.e. the arrangement of tokens, is not explicitly captured. We propose to cascade the recurrent neural networks to the Transformers, which referred to as the TransfoRNN model, to capture the sequential information. We found that the TransfoRNN models which consists of only shallow Transformers stack is suffice to give comparabl"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2104.01572","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2104.01572/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2104.01572","created_at":"2026-07-05T02:29:01.461221+00:00"},{"alias_kind":"arxiv_version","alias_value":"2104.01572v1","created_at":"2026-07-05T02:29:01.461221+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2104.01572","created_at":"2026-07-05T02:29:01.461221+00:00"},{"alias_kind":"pith_short_12","alias_value":"UI44JPAMHIHV","created_at":"2026-07-05T02:29:01.461221+00:00"},{"alias_kind":"pith_short_16","alias_value":"UI44JPAMHIHVUYHY","created_at":"2026-07-05T02:29:01.461221+00:00"},{"alias_kind":"pith_short_8","alias_value":"UI44JPAM","created_at":"2026-07-05T02:29:01.461221+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/UI44JPAMHIHVUYHY7GZE7NOCYN","json":"https://pith.science/pith/UI44JPAMHIHVUYHY7GZE7NOCYN.json","graph_json":"https://pith.science/api/pith-number/UI44JPAMHIHVUYHY7GZE7NOCYN/graph.json","events_json":"https://pith.science/api/pith-number/UI44JPAMHIHVUYHY7GZE7NOCYN/events.json","paper":"https://pith.science/paper/UI44JPAM"},"agent_actions":{"view_html":"https://pith.science/pith/UI44JPAMHIHVUYHY7GZE7NOCYN","download_json":"https://pith.science/pith/UI44JPAMHIHVUYHY7GZE7NOCYN.json","view_paper":"https://pith.science/paper/UI44JPAM","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2104.01572&json=true","fetch_graph":"https://pith.science/api/pith-number/UI44JPAMHIHVUYHY7GZE7NOCYN/graph.json","fetch_events":"https://pith.science/api/pith-number/UI44JPAMHIHVUYHY7GZE7NOCYN/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/UI44JPAMHIHVUYHY7GZE7NOCYN/action/timestamp_anchor","attest_storage":"https://pith.science/pith/UI44JPAMHIHVUYHY7GZE7NOCYN/action/storage_attestation","attest_author":"https://pith.science/pith/UI44JPAMHIHVUYHY7GZE7NOCYN/action/author_attestation","sign_citation":"https://pith.science/pith/UI44JPAMHIHVUYHY7GZE7NOCYN/action/citation_signature","submit_replication":"https://pith.science/pith/UI44JPAMHIHVUYHY7GZE7NOCYN/action/replication_record"}},"created_at":"2026-07-05T02:29:01.461221+00:00","updated_at":"2026-07-05T02:29:01.461221+00:00"}