\documentclass{article}
\usepackage{graphicx}
\usepackage[utf8]{inputenc}
\usepackage{url}
\usepackage{booktabs}
\usepackage{amsmath}

\title{Systematic Evaluation of Feedforward Network Variants in Transformer Architectures}
\author{Aardvark}
\date{\today}

\begin{document}

\maketitle

\begin{abstract}
This paper presents a systematic evaluation of feedforward network variations in transformer architectures, with particular focus on modifications to the gated activation mechanism. We conduct controlled experiments comparing four variants against the SwiGLU baseline: (1) polynomial-enhanced GEGLU, (2) normalized GEGLU, (3) scaled residual GEGLU, and (4) pure GEGLU. All experiments use a 134M parameter transformer trained on the FineWeb dataset with fixed hyperparameters (learning rate 6e-4, batch size 256, 100K steps). While our best variant achieved a marginal 0.6\% improvement in validation loss (4.898 vs 4.927), most modifications degraded performance. We discuss implications for architectural innovation and identify key limitations of our study, including the need for multi-run statistical validation and broader exploration of the design space.
\end{abstract}

[Previous sections remain unchanged until Methodology]

\subsection{FFN Variants}
We evaluated four variants:

\subsubsection{Polynomial GEGLU}
\begin{equation}
y = W_2\left(\text{GELU}(W_{1a}x) \circ W_{1b}x + \alpha(W_{1a}x)^2 \circ W_{1b}x + \beta W_{1a}x \circ (W_{1b}x)^2\right)
\end{equation}

\subsubsection{Normalized GEGLU}
\begin{equation}
y = \text{LayerNorm}\left(W_2\left(\text{GELU}(W_{1a}x) \circ W_{1b}x\right)\right) + x
\end{equation}

\subsubsection{Scaled Residual GEGLU}
\begin{equation}
y = \alpha W_2\left(\text{GELU}(W_{1a}x) \circ W_{1b}x\right) + x
\end{equation}

\subsubsection{Pure GEGLU}
\begin{equation}
y = W_2\left(\text{GELU}(W_{1a}x) \circ W_{1b}x\right)
\end{equation}

[Remaining sections unchanged]

\end{document}