\documentclass{article}
\usepackage{graphicx}
\usepackage{booktabs}
\title{Analysis of Dynamic Gating in Transformer Feedforward Networks}
\author{Aardvark}
\begin{document}

\maketitle

\begin{abstract}
We present a comprehensive study of dynamic gating mechanisms in transformer feedforward networks. Our proposed architecture introduces a lightweight controller network to generate input-dependent gating coefficients. While initial results on smaller models showed promise, our analysis reveals significant challenges in scaling these approaches, with our final implementation achieving a validation loss of 4.935 compared to the SwiGLU baseline of 4.927.
\end{abstract}

\section{Introduction}
Transformer architectures have become foundational in modern NLP, with the feedforward layer playing a crucial role in their success. Recent work has explored various enhancements to feedforward layers through gating mechanisms and alternative activation functions.

Our work makes three key contributions:
\begin{itemize}
\item Systematic evaluation of dynamic gating
\item Detailed scaling analysis
\item Empirical evidence of limitations
\end{itemize}

\section{Methodology}
Our dynamic architecture modifies the standard feedforward layer through:

\subsection{Architecture}
The standard feedforward layer computes:
\begin{equation}
FFN(x) = W_2(GELU(W_1x))
\end{equation}

Our dynamic variant introduces a controller network:
\begin{equation}
C(x) = W_c^2(GELU(W_c^1x))
\end{equation}

The final output becomes:
\begin{equation}
DynamicFFN(x) = W_2(sigmoid(C(x)) \cdot GELU(W_1x))
\end{equation}

\subsection{Implementation}
\begin{itemize}
\item Orthogonal initialization for main weights
\item Normal initialization for controller
\item Hidden dimension expansion factor of 4
\end{itemize}

\section{Experiments}
We conducted experiments at two scales:

\subsection{Ablation Studies}
\begin{itemize}
\item Model size: 83M parameters
\item Batch size: 512
\item Learning rate: 6e-4
\end{itemize}

\subsection{Full-scale Training}
\begin{itemize}
\item Model size: 134M parameters
\item Batch size: 1024
\item Learning rate: 3e-4
\end{itemize}

\section{Results}
\begin{table}[h]
\centering
\caption{Performance Comparison}
\begin{tabular}{lc}
\toprule
Method & Validation Loss \\
\midrule
SwiGLU Baseline & 4.927 \\
Our Method & 4.935 \\
Best Method & 4.792 \\
\bottomrule
\end{tabular}
\end{table}

\section{Discussion}
Our results reveal:
\begin{itemize}
\item Scaling challenges from small to large models
\item Optimization difficulties with dynamic components
\item Questionable efficiency tradeoffs
\end{itemize}

\section{Conclusion}
Our study provides valuable negative results about dynamic gating in transformers. Future work should focus on more efficient adaptation mechanisms.

\end{document}