\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{subcaption}
\usepackage{booktabs}

\title{Exploring Feedforward Architectures for Language Models}
\author{Aardvark}
\date{\today}

\begin{document}

\maketitle

\begin{abstract}
Our study evaluates feedforward layer modifications in transformers, focusing on the complexity-performance trade-off in smaller models. Results show modest improvements from architectural innovations are often outweighed by computational costs.
\end{abstract}

\section{Introduction}
We systematically evaluate parallel activation pathways and mixture-of-experts approaches in resource-constrained settings.

\section{Related Work}
Key works include SwiGLU \cite{shazeer2020glu}, MoE approaches \cite{lepikhin2020gshard}, and parallel pathways \cite{so2021primer}.

\section{Method}
We implement three variants: parallel pathways, MoE, and simplified SwiGLU.

\section{Experimental Setup}
Evaluated on FineWeb, C4, and OpenWebText with 40M-120M parameter models.

\begin{figure}[h]
  \centering
  \includegraphics[width=0.8\textwidth]{ablation_results.png}
  \caption{Validation loss trajectories with confidence intervals}
  \label{fig:results}
\end{figure}

\section{Results}
\begin{table}[h]
\centering
\begin{tabular}{lcc}
\toprule
Method & FineWeb & C4 \\
\midrule
Baseline & 4.93 & 4.91 \\
APGMLP & 4.91 & 4.90 \\
\bottomrule
\end{tabular}
\caption{Validation losses across datasets}
\label{tab:results}
\end{table}

\section{Conclusions}
Complex architectures show diminishing returns in smaller models.

\begin{thebibliography}{9}
\bibitem{shazeer2020glu}
Shazeer, N. GLU Variants Improve Transformer. arXiv:2002.05202, 2020.

\bibitem{lepikhin2020gshard}
Lepikhin, D. et al. GShard. arXiv:2006.16668, 2020.

\bibitem{so2021primer}
So, D. et al. Primer. arXiv:2109.08668, 2021.
\end{thebibliography}

\end{document}