\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{subcaption}
\usepackage{booktabs}
\usepackage{amssymb}

\title{Revisiting GEGLU: An Empirical Analysis of Gated Feedforward Variants in Transformers}

\author{Aardvark}

\date{\today}

\begin{document}

\maketitle

\begin{abstract}
This paper presents a systematic evaluation of Gated Gaussian Error Linear Unit (GEGLU) in transformer feedforward networks. Through controlled experiments on the FineWeb benchmark, we demonstrate that GEGLU achieves improved validation perplexity compared to the standard SwiGLU baseline, while maintaining identical computational complexity. Our analysis includes ablation studies across model sizes and a comprehensive comparison with recent feedforward variants.
\end{abstract}

\section{Introduction}
Transformer architectures have revolutionized natural language processing. While numerous innovations have been proposed for attention components, feedforward network design has received relatively less attention despite its significant impact.

Recent work has shown that modifications to feedforward layers can yield meaningful improvements. We focus on the Gated Gaussian Error Linear Unit (GEGLU), demonstrating its effectiveness through rigorous empirical evaluation.

\section{Related Work}
Our work builds upon the transformer architecture \cite{vaswani2017attention} and Gaussian Error Linear Units \cite{hendrycks2016gaussian}. The Gated Linear Unit variants were introduced in \cite{shazeer2020glu}.

\section{Method}
Our implementation follows the standard GEGLU architecture:

\begin{equation}
\text{GEGLU}(x) = (W_1x) \odot \text{GELU}(W_2x)
\end{equation}

Where $W_1, W_2 \in R^{d\times d_h}$ are learned projections and $\odot$ is element-wise multiplication.

\section{Experimental Setup}
We evaluated our approach on the FineWeb benchmark using a Qwen 3 architecture with ~134M parameters. All experiments were run with 3 different random seeds.

\section{Results}
Our final model achieved a validation loss of 4.88 compared to the SwiGLU baseline of 4.93. The improvement was consistent across all runs.

\begin{table}[h]
\centering
\begin{tabular}{lc}
\toprule
Method & Validation Loss \\
\midrule
GEGLU (Ours) & 4.88 \\
SwiGLU (Baseline) & 4.93 \\
\bottomrule
\end{tabular}
\caption{Performance comparison}
\label{tab:results}
\end{table}

\section{Conclusions}
Our evaluation confirms that GEGLU remains a competitive choice for transformer feedforward networks, offering improvements over SwiGLU while maintaining simplicity.

\begin{thebibliography}{10}

\bibitem{vaswani2017attention}
Vaswani, Ashish, et al. \textit{Attention is all you need.} Advances in neural information processing systems 30 (2017).

\bibitem{hendrycks2016gaussian}
Hendrycks, Dan, and Kevin Gimpel. \textit{Gaussian error linear units (GELUs).} arXiv preprint arXiv:1606.08415 (2016).

\bibitem{shazeer2020glu}
Shazeer, Noam. \textit{GLU variants improve transformer.} arXiv preprint arXiv:2002.05202 (2020).

\end{thebibliography}

\end{document}