\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{amssymb}

\title{Dynamic Sparse Gating: A Learned Approach to Feedforward Adaptation in Transformers}

\author{Aardvark}

\date{\today}

\begin{document}

\maketitle

\begin{abstract}
This paper presents Dynamic Sparse Gating (DSG), a novel approach to Transformer feedforward layers that combines learned sparsity patterns with input-dependent dynamic modulation. While our method achieves comparable performance to the SwiGLU baseline (validation loss of 4.935 vs 4.927 on FineWeb), it demonstrates the viability of learned conditional computation in feedforward networks. We provide extensive analysis of the training dynamics, architectural decisions, and computational tradeoffs.
\end{abstract}

\section{Introduction}
Transformer architectures have revolutionized natural language processing, with the feedforward layer playing a crucial role alongside attention mechanisms. While the self-attention component has received significant research attention, recent work has shown that the feedforward layer design is equally critical for model performance.

Our work builds on two key observations from prior research: (1) that gating mechanisms consistently outperform simple activation functions in feedforward layers, and (2) that sparse or conditional computation can improve model efficiency without sacrificing performance. Dynamic Sparse Gating (DSG) combines these directions by introducing learned position-wise gating modulated by input statistics.

\section{Related Work}
Modern Transformer architectures typically use variants of the Gated Linear Unit (GLU) for their feedforward layers. The original GLU formulation introduced multiplicative gating, while subsequent work demonstrated the effectiveness of Swish-gated variants (SwiGLU). Parallel research has explored sparse and conditional computation approaches, including expert mixtures and dynamic routing.

Recent work has also investigated input-dependent modulation of feedforward layers. The Gated Attention Unit demonstrated the benefits of dynamic feature transformation, while Adaptive Computation Time showed how neural networks can benefit from learned computation budgets.

\section{Method}
DSG consists of three key components that work together to enable dynamic, input-adaptive computation in the feedforward layer.

\subsection{Architecture}
The DSG layer maintains the same interface as standard feedforward layers. The core innovation lies in how the hidden representation is computed:

\begin{equation}
    \text{DSG}(x) = W_{\text{down}}(\text{SiLU}(W_{\text{gate}}x \cdot g(x) \cdot m(x)) \cdot W_{\text{up}}x)
\end{equation}

where $g(x)$ implements learned sparse gating and $m(x)$ provides dynamic modulation.

\subsection{Learned Sparse Gating}
DSG learns position-wise gating:

\begin{equation}
    g(x) = \sigma(W_gx + b_g)
\end{equation}

where $\sigma$ is the sigmoid function.

\section{Experimental Setup}
We evaluate DSG on the FineWeb dataset using a Transformer architecture with 134M parameters. Key experimental details:

\begin{itemize}
    \item Dataset: FineWeb (vocabulary size 50,304)
    \item Training: 400 steps with batch size 512
    \item Sequence length: 2048 tokens
    \item Hardware: 8 A100 GPUs
\end{itemize}

\begin{figure}[h]
  \centering
  \includegraphics[width=0.8\textwidth]{training_curve.png}
  \caption{Training dynamics showing DSG vs baseline performance.}
  \label{fig:training}
\end{figure}

\section{Results}
\begin{table}[h]
\centering
\begin{tabular}{lc}
\hline
Method & Validation Loss \\
\hline
DSG (Ours) & 4.935 \\
SwiGLU & 4.927 \\
\hline
\end{tabular}
\caption{Comparison of validation losses}
\label{tab:results}
\end{table}

\section{Conclusion}
We presented Dynamic Sparse Gating, demonstrating that learned conditional computation can approach the effectiveness of established feedforward variants. The work provides a foundation for future research into adaptive computation in Transformer architectures.

\end{document}