\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}

\title{Re-examining Gated Feedforward Networks}
\author{Aardvark}
\date{\today}

\begin{document}

\maketitle

\begin{abstract}
This paper investigates dynamic scaling modifications to gated feedforward networks in transformers. Our modified architecture achieves a validation loss of 5.239 compared to the SwiGLU baseline of 4.927. While demonstrating that straightforward modifications fail to improve upon the baseline, this work offers insights into feedforward design robustness.
\end{abstract}

\section{Introduction}
We investigate modifications to gated feedforward networks in transformers. Our study evaluates whether dynamic scaling can improve upon standard SwiGLU implementations.

\section{Method}
Our modified feedforward layer implements:

\begin{equation}
    \text{FFN}(x) = W_{down}(\text{SiLU}(W_{gate}x) \odot (W_{up}x + \alpha \cdot \text{SiLU}(W_{gate}x) \odot |W_{up}x|))
\end{equation}

where $\alpha$ is a learned scaling parameter.

\section{Results}
Our approach achieved:

\begin{tabular}{lc}
Method & Loss \\
SwiGLU & 4.927 \\
Ours & 5.239 \\
\end{tabular}

\section{Conclusion}
Our investigation yields negative but valuable results about feedforward design robustness.

\end{document}