\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{subcaption}

\title{Adaptive Gated Feedforward Networks with Learnable Expansion}

\author{Aardvark}

\date{\today}

\begin{document}

\maketitle

\begin{abstract}
We introduce a modified feedforward network architecture for transformers that incorporates learnable gating expansion and intermediate transformations. While maintaining the simplicity of standard feedforward networks, our approach introduces two key modifications: (1) a learnable expansion factor for the gating mechanism that adapts during training, and (2) an intermediate transformation with fixed residual connection. Experiments on language modeling demonstrate that our approach achieves better perplexity (4.864) compared to the standard SwiGLU baseline (4.9266) while maintaining similar computational efficiency. We provide ablation studies showing the contribution of each component and analyze the training dynamics.
\end{abstract}

\section{Introduction}
Transformer architectures have become fundamental in modern language modeling, with the feedforward network playing a crucial role alongside attention mechanisms.

Our work builds on gated linear units (GLU) and their variants. We identify two key limitations in current approaches: (1) fixed scaling of gating mechanisms, and (2) limited capacity for intermediate feature transformations.

Our main contributions are:
\begin{itemize}
    \item A learnable expansion parameter for gating mechanisms
    \item An intermediate transformation with fixed residual connection
    \item Comprehensive empirical evaluation showing improvements
\end{itemize}

\section{Method}
Our feedforward network introduces two modifications:

\subsection{Gated Transformation}
The gated transformation combines:
\begin{equation}
    h = (1 + \alpha\sigma(\alpha_0)) \cdot \text{swish}(W_g x) \odot (W_u x)
\end{equation}
where $\alpha$ is a learnable parameter.

\subsection{Intermediate Transformation}
We apply:
\begin{equation}
    h' = h + \text{gelu}(W_m h)
\end{equation}

\section{Results}
Our modified feedforward network achieves a validation perplexity of 4.864, compared to 4.9266 for SwiGLU. Figure 1 shows the training curves.

\begin{figure}[h]
  \centering
  \includegraphics[width=0.8\textwidth]{loss_curve.png}
  \caption{Training curves}
\end{figure}

Table 1 compares our approach:

\begin{table}[h]
\centering
\begin{tabular}{lc}
\hline
Method & Validation Perplexity \\ \hline
Our Method & 4.864 \\
SwiGLU Baseline & 4.9266 \\ \hline
\end{tabular}
\caption{Comparison}
\end{table}

\section{Conclusions}
We presented a simple modification to transformer feedforward networks that introduces learnable gating expansion and intermediate transformations.

\begin{thebibliography}{10}

\bibitem{shazeer2020glu}
Shazeer, Noam.
Glu variants improve transformer.
arXiv:2002.05202 (2020).

\end{thebibliography}

\end{document}