\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}

\title{Gated MLP with Isotropy Maintenance: \\ A Systematic Study of Feedforward Network Design}

\author{Aardvark}

\date{\today}

\begin{document}

\maketitle

\begin{abstract}
This paper presents a comprehensive investigation of gated multi-layer perceptron (MLP) architectures with explicit isotropy maintenance for transformer feedforward networks. Through extensive experimentation and ablation studies, we systematically evaluate the potential benefits of combining gated linear units with isotropy-preserving pathways. While our final model achieves a validation loss of 4.997 on the FineWeb benchmark, slightly underperforming the SwiGLU baseline (4.9266), the study provides valuable insights into the challenges of improving feedforward network design.
\end{abstract}

\input{theory}
\input{related_work}
\input{method}
\input{experiments}
\input{ablations}
\input{results}
\input{conclusions}

\begin{thebibliography}{10}

\bibitem{dauphin2016language}
Dauphin, Y. N., et al. 
Language modeling with gated convolutional networks. 
In \textit{International Conference on Machine Learning}, 2017.

\bibitem{shazeer2020glu}
Shazeer, N. 
GLU variants improve transformer. 
\textit{arXiv:2002.05202}, 2020.

\bibitem{ethayarajh2019contextual}
Ethayarajh, K. 
How contextual are contextualized word representations? 
In \textit{Proceedings of EMNLP}, 2019.

\end{thebibliography}

\end{document}