{"as_of":"2026-08-19T14:58:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:24b4dbdd6f0bd5c88e7d70e090723c50f81e3afea345f1d84033328791202ac0","coverage":[{"denominator":44,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":44,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-16T10:12:43.140463Z","state":"measured"},{"denominator":46,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":46,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-19T06:32:44.657259+00:00","state":"measured"},{"denominator":2,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":2,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-07T11:19:47.277591Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"pith","source_observed_at":"2026-08-07T11:19:50.554573Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"cited_work":{"arxiv_id":"2504.18929","doi":null,"metadata_source":"pith","pith_arxiv_id":"2504.18929","snapshot_observed_at":"2026-08-07T11:19:50.554573Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","venue":"cs.LG","work_id":"d46a40ef-e0c9-4439-8c99-29afd27674a5","year":2025},"citing_paper":{"arxiv_id":"2506.02867","last_updated":"2025-06-04T15:00:58Z","snapshot_observed_at":"2026-08-17T01:41:23.474492Z","submitted_at":"2025-06-03T13:31:10Z","title":"Demystifying Reasoning Dynamics with Mutual Information: Thinking Tokens are Information Peaks in LLM Reasoning","version":2},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-07T11:19:47.277591Z"},"links":{"cited_paper":"/paper/2504.18929","citing_paper":"/paper/2506.02867"},"observation_digest":"sha256:3451d2268b46992c719c6c3769b0094e444804c4a72df6bb9994ab3bd69639bc","observation_id":"e172ca69-4a61-4cb6-96fb-c0624d7eceef","resolution":{"observed_at":"2026-08-07T11:19:50.618498Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2504.18929","snapshot_observed_at":"2026-08-01T18:49:29.554633Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.17188","last_updated":"2026-07-19T11:00:59Z","snapshot_observed_at":"2026-08-13T20:22:15.952538Z","submitted_at":"2026-07-19T11:00:59Z","title":"Is Your Model Thinking or Just Stagnating? PUMA: Diagnosing Reasoning Pathology via Phase-Momentum Alignment","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-01T18:49:29.554633Z"},"links":{"cited_paper":"/paper/2504.18929","citing_paper":"/paper/2607.17188"},"observation_digest":"sha256:eca250bff0e7247551fb69d7b66a69fc699ca52bfee22432f2f86228cbdde63f","observation_id":"6b0e8549-9fc8-445b-9d74-c37554c4e2aa","resolution":{"observed_at":"2026-08-01T18:49:29.554633Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2504.18929/citation-record","integrity":"/paper/2504.18929/integrity","json":"/paper/2504.18929/citation-record.json","paper":"/paper/2504.18929"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2112.10684","last_updated":"2022-10-26T16:14:05Z","snapshot_observed_at":"2026-08-17T06:02:42.650883Z","submitted_at":"2021-12-20T17:05:11Z","title":"Efficient Large Scale Language Modeling with Mixtures of Experts","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2112.10684","snapshot_observed_at":"2026-08-16T10:12:42.955026Z","title":"Efficient large scale language modeling with mixtures of experts","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:42.955026Z"},"links":{"cited_paper":"/paper/2112.10684","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:73f00cf8571cc6d0c25289f6ee685ad1bb51b09fee7280c48d92087ee219de84","observation_id":"3e4e07e6-1c27-4e70-8a9b-03fbd1aa76b5","resolution":{"observed_at":"2026-08-16T10:12:42.955026Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.06204","last_updated":"2025-04-09T13:54:59Z","snapshot_observed_at":"2026-08-18T19:49:32.900527Z","submitted_at":"2024-06-26T16:34:33Z","title":"A Survey on Mixture of Experts in Large Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.06204","snapshot_observed_at":"2026-08-16T10:12:42.959822Z","title":"A survey on mixture of experts","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:42.959822Z"},"links":{"cited_paper":"/paper/2407.06204","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:b78bf9250ddfea0a53c26f4ac7a4750b8601718d6e943877fb08bb7849771585","observation_id":"8151577a-649c-4ac1-8ea6-5920185f6d45","resolution":{"observed_at":"2026-08-16T10:12:42.959822Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1904.10509","last_updated":"2019-04-23T19:29:47Z","snapshot_observed_at":"2026-08-16T10:03:03.268538Z","submitted_at":"2019-04-23T19:29:47Z","title":"Generating Long Sequences with Sparse Transformers","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1904.10509","snapshot_observed_at":"2026-08-16T10:12:42.965445Z","title":"Generating long sequences with sparse transformers","venue":null,"work_id":null,"year":1904},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:42.965445Z"},"links":{"cited_paper":"/paper/1904.10509","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:5c998c753b6bf6efd4139aef9c2b4bbbde2118ed8e5732d6485ace4cd71924b8","observation_id":"aac28bb3-50ce-41d7-a990-089d16b0f629","resolution":{"observed_at":"2026-08-16T10:12:42.965445Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T10:12:42.970245Z","title":"Palm: Scaling language modeling with pathways","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:42.970245Z"},"links":{"citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:95a2c73180352375cd83780542c04beb6f9d664620f7827bce90dd2191b416f3","observation_id":"bfac2720-24e1-4235-8258-312f8fbc46bb","resolution":{"observed_at":"2026-08-16T10:12:42.970245Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1412.3555","last_updated":"2014-12-11T06:46:53Z","snapshot_observed_at":"2026-08-13T10:35:27.214652Z","submitted_at":"2014-12-11T06:46:53Z","title":"Empirical Evaluation of Gated Recurrent Neural Networks on Sequence Modeling","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1412.3555","snapshot_observed_at":"2026-08-16T10:12:42.974669Z","title":"Empirical evaluation of gated recurrent neural networks on sequence modeling","venue":null,"work_id":null,"year":2014},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:42.974669Z"},"links":{"cited_paper":"/paper/1412.3555","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:ba5c765cbe9112576a0f9710b3029131888fc777ac7135878027bb6093c80f4c","observation_id":"c91351ba-6685-40e7-9581-28798aa59ad1","resolution":{"observed_at":"2026-08-16T10:12:42.974669Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1909.00015","last_updated":"2019-09-06T16:55:20Z","snapshot_observed_at":"2026-08-19T06:53:00.897451Z","submitted_at":"2019-08-30T18:06:14Z","title":"Adaptively Sparse Transformers","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1909.00015","snapshot_observed_at":"2026-08-16T10:12:42.978885Z","title":"Adaptively sparse transformers","venue":null,"work_id":null,"year":1909},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:42.978885Z"},"links":{"cited_paper":"/paper/1909.00015","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:41d3eefa3db7f475a021f5e1e08d22e35fb2e9d5af94d65cc742227252ba4283","observation_id":"02e99347-3cb4-4bac-b116-a008d9ca76f3","resolution":{"observed_at":"2026-08-16T10:12:42.978885Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T10:12:43.851605Z","title":"Batch normalization biases residual blocks towards the identity function in deep networks","venue":null,"work_id":"759fdf11-2248-4906-8385-63365424c30f","year":2020},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:42.984231Z"},"links":{"citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:5d864781dae511041f690cb9d705cd449cc944c71e5e74f40d90d97d86071676","observation_id":"ae1e7f62-15b2-473e-b207-bf9e1f265f57","resolution":{"observed_at":"2026-08-16T10:12:43.855896Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2309.10668","last_updated":"2024-03-18T23:15:47Z","snapshot_observed_at":"2026-07-06T16:20:36.866744Z","submitted_at":"2023-09-19T14:50:38Z","title":"Language Modeling Is Compression","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.10668","snapshot_observed_at":"2026-08-16T10:12:42.988390Z","title":"Language modeling is compression","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:42.988390Z"},"links":{"cited_paper":"/paper/2309.10668","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:50bbb8937384bded11789aa3010e3deb38f282191854d359884ad40a2e001175","observation_id":"2f5afe94-15bf-4069-b2db-444becbd5e8e","resolution":{"observed_at":"2026-08-16T10:12:42.988390Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T10:12:42.992746Z","title":"Attention is not all you need: Pure attention loses rank doubly exponentially with depth","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:42.992746Z"},"links":{"citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:2c83bb5592a7663820705cb7cf73c039f8d499eb062b3bac7c6e15d18a12a4af","observation_id":"8c413e36-3469-40c7-9a41-70d9dadf229e","resolution":{"observed_at":"2026-08-16T10:12:42.992746Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.15796","last_updated":"2025-01-15T02:48:59Z","snapshot_observed_at":"2026-08-16T14:07:07.590212Z","submitted_at":"2024-03-23T11:03:31Z","title":"Understanding Emergent Abilities of Language Models from the Loss Perspective","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.15796","snapshot_observed_at":"2026-08-16T10:12:42.996527Z","title":"Understanding emergent abilities of language models from the loss perspective","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:42.996527Z"},"links":{"cited_paper":"/paper/2403.15796","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:4ebf27e7336ccfb3b745c8b9c79e9d13f5dda0b55591d6fe01c1c9319cf495f4","observation_id":"278ccce7-6d1e-411c-aaca-6cef347f9ce6","resolution":{"observed_at":"2026-08-16T10:12:42.996527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1910.10073","last_updated":"2020-02-14T20:49:40Z","snapshot_observed_at":"2026-08-18T14:23:50.228061Z","submitted_at":"2019-10-22T16:15:58Z","title":"Depth-Adaptive Transformer","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1910.10073","snapshot_observed_at":"2026-08-16T10:12:43.001124Z","title":"Depth-adaptive transformer","venue":null,"work_id":null,"year":1910},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.001124Z"},"links":{"cited_paper":"/paper/1910.10073","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:5f3c75db98cf1581d62c3d188b01de3892acc329090a35dd16266296df243dd7","observation_id":"146d0c43-1861-472a-a09e-e834ebd4048a","resolution":{"observed_at":"2026-08-16T10:12:43.001124Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T10:12:43.005357Z","title":"Switch transformers: Scaling to trillion parameter models with simple and efficient sparsity","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.005357Z"},"links":{"citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:4deeb0da19893dd1cbb8812534372cf04b1ca46c22172b7aae91b8c745729dcd","observation_id":"996f35e5-2ff3-471f-821b-bf72719f033a","resolution":{"observed_at":"2026-08-16T10:12:43.005357Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2012.14913","last_updated":"2021-09-05T17:32:27Z","snapshot_observed_at":"2026-08-13T01:11:33.500647Z","submitted_at":"2020-12-29T19:12:05Z","title":"Transformer Feed-Forward Layers Are Key-Value Memories","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2012.14913","snapshot_observed_at":"2026-08-16T10:12:43.009409Z","title":"Transformer feed-forward layers are key-value memories","venue":null,"work_id":null,"year":2012},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.009409Z"},"links":{"cited_paper":"/paper/2012.14913","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:e051da053406b78f312e838d92579a6ee3a81ed4e1311f04a8e6b45cc1e22ed8","observation_id":"965d0336-88d0-4f19-96f1-08b0109e038d","resolution":{"observed_at":"2026-08-16T10:12:43.009409Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2203.14680","last_updated":"2022-10-12T18:01:23Z","snapshot_observed_at":"2026-08-16T17:11:19.673268Z","submitted_at":"2022-03-28T12:26:00Z","title":"Transformer Feed-Forward Layers Build Predictions by Promoting Concepts in the Vocabulary Space","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2203.14680","snapshot_observed_at":"2026-08-16T10:12:43.013569Z","title":"Transformer feed-forward layers build predictions by promoting concepts in the vocabulary space","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.013569Z"},"links":{"cited_paper":"/paper/2203.14680","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:84926907dac04d81d070abb6cb885ae6b959092f04aa1b67b6253f654b0c1131","observation_id":"c7f42339-194a-443d-bd7e-2a6c803e4ed3","resolution":{"observed_at":"2026-08-16T10:12:43.013569Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.01906","last_updated":"2024-05-31T11:14:16Z","snapshot_observed_at":"2026-08-16T14:46:13.359223Z","submitted_at":"2023-11-03T13:30:52Z","title":"Simplifying Transformer Blocks","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.01906","snapshot_observed_at":"2026-08-16T10:12:43.018009Z","title":"Simplifying transformer blocks","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.018009Z"},"links":{"cited_paper":"/paper/2311.01906","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:95263265320d586d06038ab564d3a4ea1ba57c43c43a44a7e555949f00c80b09","observation_id":"c9afd19a-1657-4f88-afb0-c2677c2cf50c","resolution":{"observed_at":"2026-08-16T10:12:43.018009Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2302.10322","last_updated":"2023-02-20T21:26:25Z","snapshot_observed_at":"2026-08-16T15:54:00.141178Z","submitted_at":"2023-02-20T21:26:25Z","title":"Deep Transformers without Shortcuts: Modifying Self-attention for Faithful Signal Propagation","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2302.10322","snapshot_observed_at":"2026-08-16T10:12:43.022142Z","title":"Deep transformers without shortcuts: Modifying self-attention for faithful signal propagation","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.022142Z"},"links":{"cited_paper":"/paper/2302.10322","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:a150078a4561fe390f3e26c49c0cc5a23a020798df4e7839b7acef8ee53d7585","observation_id":"a6d013a4-0881-4d74-99a9-611cd3eec20a","resolution":{"observed_at":"2026-08-16T10:12:43.022142Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T10:12:43.026268Z","title":"Long short-term memory","venue":null,"work_id":null,"year":1997},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.026268Z"},"links":{"citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:7f6b42f9cfd2b9de028813fe65f47a051adb14748556d5242334b5c016954fec","observation_id":"c4ad010c-e7a5-423e-9f36-972e10a35510","resolution":{"observed_at":"2026-08-16T10:12:43.026268Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.09937","last_updated":"2024-08-19T13:55:42Z","snapshot_observed_at":"2026-08-18T03:59:04.762248Z","submitted_at":"2024-04-15T17:03:41Z","title":"Compression Represents Intelligence Linearly","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.09937","snapshot_observed_at":"2026-08-16T10:12:43.030615Z","title":"Compression represents intelligence linearly","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.030615Z"},"links":{"cited_paper":"/paper/2404.09937","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:5d30aa14bfd84de8bad01aec9e9f2469ffae1b0552e12e0c6f44086f6c060037","observation_id":"e2f41cbf-f7d6-4c1c-9f37-123a4a019e72","resolution":{"observed_at":"2026-08-16T10:12:43.030615Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T10:12:43.034975Z","title":"Universal artificial intelligence: Sequential decisions based on algorithmic probability","venue":null,"work_id":null,"year":2005},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.034975Z"},"links":{"citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:c82dc629f580afdba3c9dd9ac81c9a844df262740e567e4bbf8335e75e1ce272","observation_id":"509501dc-d39a-4268-8ffa-8739f46404e1","resolution":{"observed_at":"2026-08-16T10:12:43.034975Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T10:12:43.799982Z","title":"The hutter prize","venue":null,"work_id":"7479550f-c6c5-406d-bcae-78a645249359","year":2006},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.038860Z"},"links":{"citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:70eb6f88e803d5dce8f92efc9008d629fab47bd1d09fab245557860e5263dc81","observation_id":"9af05e54-f284-43b8-a6ec-32eb61785791","resolution":{"observed_at":"2026-08-16T10:12:43.804115Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1412.6980","last_updated":"2017-01-30T01:27:54Z","snapshot_observed_at":"2026-08-17T19:26:44.032537Z","submitted_at":"2014-12-22T13:54:29Z","title":"Adam: A Method for Stochastic Optimization","version":9},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1412.6980","snapshot_observed_at":"2026-08-16T10:12:43.043050Z","title":"Adam: A method for stochastic optimization","venue":null,"work_id":null,"year":2014},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.043050Z"},"links":{"cited_paper":"/paper/1412.6980","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:088f39471ff40f258797afd6aa68125949b1d61507d8a5c19b9bd887a81d3dd8","observation_id":"acf530e3-acea-410b-817d-27ce2a1347f6","resolution":{"observed_at":"2026-08-16T10:12:43.043050Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.07871","last_updated":"2024-02-12T18:33:47Z","snapshot_observed_at":"2026-08-18T12:11:18.333774Z","submitted_at":"2024-02-12T18:33:47Z","title":"Scaling Laws for Fine-Grained Mixture of Experts","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.07871","snapshot_observed_at":"2026-08-16T10:12:43.047153Z","title":"Scaling laws for fine-grained mixture of experts","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.047153Z"},"links":{"cited_paper":"/paper/2402.07871","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:28f9a5fa7c9919a24619f5d238b02e63820ee81cef093f3913e9124a8e5a5b68","observation_id":"dfb99475-540a-47e6-87b4-efed5b4f22c7","resolution":{"observed_at":"2026-08-16T10:12:43.047153Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T10:12:43.786092Z","title":"Same pre-training loss, better downstream: Implicit bias matters for language models","venue":null,"work_id":"5bbfc460-fcb8-4589-837f-7cd003d3e910","year":2023},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.051596Z"},"links":{"citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:e8cd414c5c1a7d8443fe94873c47045d1e28878c03a8e4898ab7a11b4ab48f34","observation_id":"70399b3a-090f-4eec-9f1e-ea9e9c46a01a","resolution":{"observed_at":"2026-08-16T10:12:43.790415Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1711.05101","last_updated":"2019-01-04T21:01:49Z","snapshot_observed_at":"2026-08-14T20:13:52.872565Z","submitted_at":"2017-11-14T14:24:06Z","title":"Decoupled Weight Decay Regularization","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1711.05101","snapshot_observed_at":"2026-08-16T10:12:43.055761Z","title":"Decoupled weight decay regularization","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.055761Z"},"links":{"cited_paper":"/paper/1711.05101","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:edad800f3dee5df160c8bdc9091b629c1e489ca2fd4e16527242a5331d2de012","observation_id":"8fae963d-9ba5-4c63-88ed-f568b5a9271a","resolution":{"observed_at":"2026-08-16T10:12:43.055761Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T10:12:43.771539Z","title":"Dynamic sparsity in the brain and machines routing information through neural pathways","venue":null,"work_id":"4795abb9-c075-49a9-8bc5-ffaba8d8d8ea","year":2024},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.060725Z"},"links":{"citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:798ebf7722e7482cd4c321fa617208d383532875838ae0f720ecfc7a12809f18","observation_id":"224c659a-70fe-45ee-8499-43f6d989ba0e","resolution":{"observed_at":"2026-08-16T10:12:43.776896Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2304.09871","last_updated":"2023-04-25T04:48:47Z","snapshot_observed_at":"2026-08-16T22:51:53.749107Z","submitted_at":"2023-04-19T06:15:11Z","title":"A Theory on Adam Instability in Large-Scale Machine Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2304.09871","snapshot_observed_at":"2026-08-16T10:12:43.064636Z","title":"A theory on adam instability in large-scale machine learning","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.064636Z"},"links":{"cited_paper":"/paper/2304.09871","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:3306e2f7bff6ebc5f7bfea50954ff4ba1e0531519a86393aff2339254ef75430","observation_id":"8bd2369d-48ba-4fa1-9542-e15011faa771","resolution":{"observed_at":"2026-08-16T10:12:43.064636Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T10:12:43.069184Z","title":"Signal propagation in transformers: Theoretical perspectives and the role of rank collapse","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":27,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.069184Z"},"links":{"citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:e6ec99f8ebfe57b3f1f501758415a1c739fe4793aa0660b990a872291265d213","observation_id":"97b8ce08-5392-443a-a9e3-f4098ef4ed00","resolution":{"observed_at":"2026-08-16T10:12:43.069184Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06104","last_updated":"2024-06-18T09:16:14Z","snapshot_observed_at":"2026-08-16T14:28:08.030037Z","submitted_at":"2024-01-11T18:35:26Z","title":"Transformers are Multi-State RNNs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.06104","snapshot_observed_at":"2026-08-16T10:12:43.073149Z","title":"Transformers are multi-state rnns","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.073149Z"},"links":{"cited_paper":"/paper/2401.06104","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:b14385dc1a6cb59127062c7d72f956e2665dc00fd5a8abd4ca44277a814f9ea6","observation_id":"f1169752-61fe-4f7a-bfe0-7c4fb84ef24c","resolution":{"observed_at":"2026-08-16T10:12:43.073149Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T10:12:43.077273Z","title":"Understanding llm behaviors via compression: Data generation, knowledge acquisition and scaling laws","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":29,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.077273Z"},"links":{"citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:12221a53fe8bc1c1fced4792026f3efd0a1132076ffc0a8d6cafc8c4cd27d997","observation_id":"80a37227-342c-4165-b839-2c68f5e8426b","resolution":{"observed_at":"2026-08-16T10:12:43.077273Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1905.05702","last_updated":"2019-06-12T18:20:25Z","snapshot_observed_at":"2026-08-14T16:32:25.824753Z","submitted_at":"2019-05-14T16:21:34Z","title":"Sparse Sequence-to-Sequence Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1905.05702","snapshot_observed_at":"2026-08-16T10:12:43.081065Z","title":"Sparse sequence-to-sequence models","venue":null,"work_id":null,"year":1905},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":30,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.081065Z"},"links":{"cited_paper":"/paper/1905.05702","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:61751d51789cdb328244bcc2bbcfe463d28e5641385bc44d4baad8e86413823d","observation_id":"f06b45c6-03d4-4870-a638-be76a1a489e6","resolution":{"observed_at":"2026-08-16T10:12:43.081065Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.02258","last_updated":"2024-04-02T19:28:11Z","snapshot_observed_at":"2026-08-17T04:55:07.787141Z","submitted_at":"2024-04-02T19:28:11Z","title":"Mixture-of-Depths: Dynamically allocating compute in transformer-based language models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.02258","snapshot_observed_at":"2026-08-16T10:12:43.085459Z","title":"Mixture-of-depths: Dynamically allocating compute in transformer-based language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":31,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.085459Z"},"links":{"cited_paper":"/paper/2404.02258","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:4785bd99fed8331acee0032d96a289ed3e2e76d936f7913d17b7f8844d0c3e59","observation_id":"85a88a37-cd63-41ad-98d6-0234081ca429","resolution":{"observed_at":"2026-08-16T10:12:43.085459Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T10:12:43.089578Z","title":"A stochastic approximation method","venue":null,"work_id":null,"year":1951},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":32,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.089578Z"},"links":{"citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:679bc32906b37fd16b45c670a2905ed13fe0595c32c679a28bb8cdd9b27636b8","observation_id":"08f47968-563a-4d63-bf27-8e1957d12973","resolution":{"observed_at":"2026-08-16T10:12:43.089578Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T10:12:43.093336Z","title":"A mathematical theory of communication","venue":null,"work_id":null,"year":1948},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.093336Z"},"links":{"citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:331b4c2df00f9b0e2d5f3830904acbbe88bfdd8e0b70217d4d3599a6a39e2699","observation_id":"652abb8c-24d3-4e9c-8baf-df4441118e9a","resolution":{"observed_at":"2026-08-16T10:12:43.093336Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.08699","last_updated":"2024-03-13T17:02:27Z","snapshot_observed_at":"2026-08-16T14:10:06.555220Z","submitted_at":"2024-03-13T17:02:27Z","title":"Implicit Regularization of Gradient Flow on One-Layer Softmax Attention","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.08699","snapshot_observed_at":"2026-08-16T10:12:43.098087Z","title":"Implicit regularization of gradient flow on one-layer softmax attention","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":34,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.098087Z"},"links":{"cited_paper":"/paper/2403.08699","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:b0b3ce43b1dd50660790bf3206b2e047e698d6b79b0c59b0667e515dd220a37d","observation_id":"132ff93b-ef9f-4524-bae8-bd34f1a0fe80","resolution":{"observed_at":"2026-08-16T10:12:43.098087Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2302.06461","last_updated":"2023-02-13T15:41:20Z","snapshot_observed_at":"2026-08-16T15:55:42.409032Z","submitted_at":"2023-02-13T15:41:20Z","title":"A Study on ReLU and Softmax in Transformer","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2302.06461","snapshot_observed_at":"2026-08-16T10:12:43.102563Z","title":"A study on relu and softmax in transformer","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":35,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.102563Z"},"links":{"cited_paper":"/paper/2302.06461","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:42c6d80c5148229fceb07ef0a8b1eb6525941e6714d895bae32f0c5a64c6d70f","observation_id":"708a9c69-c078-4977-b93c-f3f75d5766aa","resolution":{"observed_at":"2026-08-16T10:12:43.102563Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T10:12:43.729781Z","title":"An observation on generalization","venue":null,"work_id":"ec8fb7eb-f2b2-46ca-ae87-ae368dbf7f41","year":2023},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":36,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.106604Z"},"links":{"citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:71779ecdc66ff7bae57a8066b1a047cffa589e1b1506848dffe4c8add04d14e0","observation_id":"2a26c120-281c-4d76-86e2-95152e86b2bf","resolution":{"observed_at":"2026-08-16T10:12:43.733744Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.05738","last_updated":"2025-03-31T06:17:02Z","snapshot_observed_at":"2026-08-18T23:30:55.927572Z","submitted_at":"2024-02-08T15:15:09Z","title":"Implicit Bias and Fast Convergence Rates for Self-attention","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.05738","snapshot_observed_at":"2026-08-16T10:12:43.110604Z","title":"Implicit bias and fast convergence rates for self-attention","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":37,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.110604Z"},"links":{"cited_paper":"/paper/2402.05738","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:8106ca7856650543d338a69d83122134aaeb1720a1b60a1df413bf2083a37163","observation_id":"ff593e8a-10b1-43fe-b6cd-1eebebdb226c","resolution":{"observed_at":"2026-08-16T10:12:43.110604Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T10:12:43.114913Z","title":"Attention is all you need","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":38,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.114913Z"},"links":{"citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:4872a0aeb5a88e7366446475dda2feafc9f11966e7ed12c1254d66634c60b1aa","observation_id":"12cd0dc5-e932-428e-a4fe-75b4e48529aa","resolution":{"observed_at":"2026-08-16T10:12:43.114913Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.04827","last_updated":"2023-09-09T15:51:36Z","snapshot_observed_at":"2026-08-16T15:02:02.724232Z","submitted_at":"2023-09-09T15:51:36Z","title":"Neurons in Large Language Models: Dead, N-gram, Positional","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.04827","snapshot_observed_at":"2026-08-16T10:12:43.119358Z","title":"Neurons in large language models: Dead, n-gram, positional","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":39,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.119358Z"},"links":{"cited_paper":"/paper/2309.04827","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:e6fa3cd4d35f8fd65c1b30da5a0d4fb0977b3230e9804fd8cd6cdc95e170e4cb","observation_id":"7f106bef-6056-45e9-98a9-e54c5afeaf0c","resolution":{"observed_at":"2026-08-16T10:12:43.119358Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T10:12:43.705825Z","title":"Attention-only transformers via unrolled subspace denoising","venue":null,"work_id":"527d50e3-d703-401d-8074-3a8d76bd61e5","year":2025},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":40,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.123770Z"},"links":{"citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:09afed2f237d8031413d48f0ec03cf2a59487c3300a76ad4bfd3aa1123abebf3","observation_id":"f3e97eca-ac28-4fa8-9ce7-0795194d061c","resolution":{"observed_at":"2026-08-16T10:12:43.709868Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T10:12:43.692159Z","title":"Scaling white-box transformers for vision","venue":null,"work_id":"c108e228-76e7-4ef4-8d48-ef201173fda7","year":2024},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":41,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.127901Z"},"links":{"citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:b1b6265d53b0a81fac39d3266f64244c02f5aee3cd4f0a0eb49925d29a735d62","observation_id":"dc491a75-8b1e-4d39-b5b5-a8ea86afdfa0","resolution":{"observed_at":"2026-08-16T10:12:43.696358Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-16T10:12:43.675985Z","title":"White-box transformers via sparse rate reduction","venue":null,"work_id":"07a3bc1e-4d1a-45d8-bffb-5729b0e6477f","year":2023},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":42,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.131717Z"},"links":{"citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:61c50d59a908839eaea1f4e43dfd41d31d55c434668e0d599e4f770fbae2b5b2","observation_id":"8627024e-6241-4db6-9d42-a1235113f332","resolution":{"observed_at":"2026-08-16T10:12:43.682401Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1212.5701","last_updated":"2012-12-22T15:46:49Z","snapshot_observed_at":"2026-08-19T05:19:50.480481Z","submitted_at":"2012-12-22T15:46:49Z","title":"ADADELTA: An Adaptive Learning Rate Method","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1212.5701","snapshot_observed_at":"2026-08-16T10:12:43.135927Z","title":"Adadelta: an adaptive learning rate method","venue":null,"work_id":null,"year":2012},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":43,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.135927Z"},"links":{"cited_paper":"/paper/1212.5701","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:44f680815e3bd5dcd84fa08ff0175a2697496e74e0139a34771564dc7ef9ce86","observation_id":"57eb23cb-192b-4af9-9e16-9106a6bd7ca7","resolution":{"observed_at":"2026-08-16T10:12:43.135927Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2205.01068","last_updated":"2022-06-21T17:04:40Z","snapshot_observed_at":"2026-08-06T03:13:37.403059Z","submitted_at":"2022-05-02T17:49:50Z","title":"OPT: Open Pre-trained Transformer Language Models","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2205.01068","snapshot_observed_at":"2026-08-16T10:12:43.140463Z","title":"Opt: Open pre-trained transformer language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity","version":1},"reference_index":44,"source":"arxiv_source","source_observed_at":"2026-08-16T10:12:43.140463Z"},"links":{"cited_paper":"/paper/2205.01068","citing_paper":"/paper/2504.18929"},"observation_digest":"sha256:69d3ddf77b18d2c12496b9fac25d48fa8ee0084058a1900303f4ad5f504416f2","observation_id":"b7083555-3e07-48c3-a1ea-554a1daa3c93","resolution":{"observed_at":"2026-08-16T10:12:43.140463Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2504.18929","last_updated":"2025-04-26T14:02:07Z","latest_version":1,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-18T12:12:00.004360Z","submitted_at":"2025-04-26T14:02:07Z","title":"Revisiting Transformers through the Lens of Low Entropy and Dynamic Sparsity"},"reference_resolution":{"displayed":44,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":36,"verified_exact":0,"verified_fuzzy":8},"total_outbound_references":44},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"thesis":"As of 19 August 2026, this Paper Citation Record lists 44 of 44 outbound references and 2 inbound Pith citation observations for arXiv:2504.18929."}