{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:UI44JPAMHIHVUYHY7GZE7NOCYN","short_pith_number":"pith:UI44JPAM","canonical_record":{"source":{"id":"2104.01572","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-04-04T09:31:18Z","cross_cats_sorted":[],"title_canon_sha256":"b9a861b3dbe903f2df0e65202a662e06eb22ce4156057ae5bc5bab626dce6147","abstract_canon_sha256":"2964394c4770e06a87d3c50361bdd56a0041833b65fed19745f35ef923cb5f2c"},"schema_version":"1.0"},"canonical_sha256":"a239c4bc0c3a0f5a60f8f9b24fb5c2c37ee80f715d40009443c23206d85c6859","source":{"kind":"arxiv","id":"2104.01572","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2104.01572","created_at":"2026-07-05T02:29:01Z"},{"alias_kind":"arxiv_version","alias_value":"2104.01572v1","created_at":"2026-07-05T02:29:01Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2104.01572","created_at":"2026-07-05T02:29:01Z"},{"alias_kind":"pith_short_12","alias_value":"UI44JPAMHIHV","created_at":"2026-07-05T02:29:01Z"},{"alias_kind":"pith_short_16","alias_value":"UI44JPAMHIHVUYHY","created_at":"2026-07-05T02:29:01Z"},{"alias_kind":"pith_short_8","alias_value":"UI44JPAM","created_at":"2026-07-05T02:29:01Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:UI44JPAMHIHVUYHY7GZE7NOCYN","target":"record","payload":{"canonical_record":{"source":{"id":"2104.01572","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-04-04T09:31:18Z","cross_cats_sorted":[],"title_canon_sha256":"b9a861b3dbe903f2df0e65202a662e06eb22ce4156057ae5bc5bab626dce6147","abstract_canon_sha256":"2964394c4770e06a87d3c50361bdd56a0041833b65fed19745f35ef923cb5f2c"},"schema_version":"1.0"},"canonical_sha256":"a239c4bc0c3a0f5a60f8f9b24fb5c2c37ee80f715d40009443c23206d85c6859","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:29:01.461738Z","signature_b64":"XOz6uL9NhrV4S+K5CAFWbHkAHkNZNThFoYQHhyrJmpASJ3Y3rjV0zBl+M/9fzO9yrEMlIiLWD23bqVrrwZrUCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a239c4bc0c3a0f5a60f8f9b24fb5c2c37ee80f715d40009443c23206d85c6859","last_reissued_at":"2026-07-05T02:29:01.461159Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:29:01.461159Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2104.01572","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:29:01Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Cx1C3x6zpyY+riRfMafagDA1V2PohMV9pV2dA7KhZJAnjESfb2nmmt1tdZ1nUVqIVzKx9UJWHcJChIqUTlyrDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-31T16:23:44.289583Z"},"content_sha256":"0eb3d39fe8c685bc02dad21ef6b325e52be51d715e23ac77c97e6be9f8c36e5e","schema_version":"1.0","event_id":"sha256:0eb3d39fe8c685bc02dad21ef6b325e52be51d715e23ac77c97e6be9f8c36e5e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:UI44JPAMHIHVUYHY7GZE7NOCYN","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"TransfoRNN: Capturing the Sequential Information in Self-Attention Representations for Language Modeling","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Junjie Wang, Lin Yang, Tze Yuang Chong, Xuyang Wang","submitted_at":"2021-04-04T09:31:18Z","abstract_excerpt":"In this paper, we describe the use of recurrent neural networks to capture sequential information from the self-attention representations to improve the Transformers. Although self-attention mechanism provides a means to exploit long context, the sequential information, i.e. the arrangement of tokens, is not explicitly captured. We propose to cascade the recurrent neural networks to the Transformers, which referred to as the TransfoRNN model, to capture the sequential information. We found that the TransfoRNN models which consists of only shallow Transformers stack is suffice to give comparabl"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2104.01572","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2104.01572/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:29:01Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"xe96DGtY871WM++IeyYzelalvE7Fmb6i8Mc4LATi0567kJzQlI5mifHxzmDf7lDtlH/sM8fcNw7TTZwwjzGqDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-31T16:23:44.290076Z"},"content_sha256":"221ec760237f1dbb9e96601153e1b04e41835120fcf7f81de9b93251b398a3ae","schema_version":"1.0","event_id":"sha256:221ec760237f1dbb9e96601153e1b04e41835120fcf7f81de9b93251b398a3ae"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/UI44JPAMHIHVUYHY7GZE7NOCYN/bundle.json","state_url":"https://pith.science/pith/UI44JPAMHIHVUYHY7GZE7NOCYN/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/UI44JPAMHIHVUYHY7GZE7NOCYN/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-31T16:23:44Z","links":{"resolver":"https://pith.science/pith/UI44JPAMHIHVUYHY7GZE7NOCYN","bundle":"https://pith.science/pith/UI44JPAMHIHVUYHY7GZE7NOCYN/bundle.json","state":"https://pith.science/pith/UI44JPAMHIHVUYHY7GZE7NOCYN/state.json","well_known_bundle":"https://pith.science/.well-known/pith/UI44JPAMHIHVUYHY7GZE7NOCYN/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:UI44JPAMHIHVUYHY7GZE7NOCYN","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"2964394c4770e06a87d3c50361bdd56a0041833b65fed19745f35ef923cb5f2c","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-04-04T09:31:18Z","title_canon_sha256":"b9a861b3dbe903f2df0e65202a662e06eb22ce4156057ae5bc5bab626dce6147"},"schema_version":"1.0","source":{"id":"2104.01572","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2104.01572","created_at":"2026-07-05T02:29:01Z"},{"alias_kind":"arxiv_version","alias_value":"2104.01572v1","created_at":"2026-07-05T02:29:01Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2104.01572","created_at":"2026-07-05T02:29:01Z"},{"alias_kind":"pith_short_12","alias_value":"UI44JPAMHIHV","created_at":"2026-07-05T02:29:01Z"},{"alias_kind":"pith_short_16","alias_value":"UI44JPAMHIHVUYHY","created_at":"2026-07-05T02:29:01Z"},{"alias_kind":"pith_short_8","alias_value":"UI44JPAM","created_at":"2026-07-05T02:29:01Z"}],"graph_snapshots":[{"event_id":"sha256:221ec760237f1dbb9e96601153e1b04e41835120fcf7f81de9b93251b398a3ae","target":"graph","created_at":"2026-07-05T02:29:01Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2104.01572/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In this paper, we describe the use of recurrent neural networks to capture sequential information from the self-attention representations to improve the Transformers. Although self-attention mechanism provides a means to exploit long context, the sequential information, i.e. the arrangement of tokens, is not explicitly captured. We propose to cascade the recurrent neural networks to the Transformers, which referred to as the TransfoRNN model, to capture the sequential information. We found that the TransfoRNN models which consists of only shallow Transformers stack is suffice to give comparabl","authors_text":"Junjie Wang, Lin Yang, Tze Yuang Chong, Xuyang Wang","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-04-04T09:31:18Z","title":"TransfoRNN: Capturing the Sequential Information in Self-Attention Representations for Language Modeling"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2104.01572","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:0eb3d39fe8c685bc02dad21ef6b325e52be51d715e23ac77c97e6be9f8c36e5e","target":"record","created_at":"2026-07-05T02:29:01Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"2964394c4770e06a87d3c50361bdd56a0041833b65fed19745f35ef923cb5f2c","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2021-04-04T09:31:18Z","title_canon_sha256":"b9a861b3dbe903f2df0e65202a662e06eb22ce4156057ae5bc5bab626dce6147"},"schema_version":"1.0","source":{"id":"2104.01572","kind":"arxiv","version":1}},"canonical_sha256":"a239c4bc0c3a0f5a60f8f9b24fb5c2c37ee80f715d40009443c23206d85c6859","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a239c4bc0c3a0f5a60f8f9b24fb5c2c37ee80f715d40009443c23206d85c6859","first_computed_at":"2026-07-05T02:29:01.461159Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T02:29:01.461159Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"XOz6uL9NhrV4S+K5CAFWbHkAHkNZNThFoYQHhyrJmpASJ3Y3rjV0zBl+M/9fzO9yrEMlIiLWD23bqVrrwZrUCw==","signature_status":"signed_v1","signed_at":"2026-07-05T02:29:01.461738Z","signed_message":"canonical_sha256_bytes"},"source_id":"2104.01572","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:0eb3d39fe8c685bc02dad21ef6b325e52be51d715e23ac77c97e6be9f8c36e5e","sha256:221ec760237f1dbb9e96601153e1b04e41835120fcf7f81de9b93251b398a3ae"],"state_sha256":"e1314e5b9a16103aba11e90ef45998582b1db9e2bdc119ba44c16c2f37cc9a2c"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"q+xGJSE+xR6u/GFIZ8Vf/KrrdBIckBwZTYX24W8A01F/o7VpRxW3Ns5zmlFBBuOFYbqjEm8scmvGGTT1vaJcBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-31T16:23:44.295408Z","bundle_sha256":"0509f9acc0e65b5bb48804633d8d1cc960ab80294d879609b45a0124348265ba"}}