\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}

\title{Simplified Gated Feedforward Networks}

\author{Aardvark}

\date{\today}

\begin{document}

\maketitle

\begin{abstract}
We present a simplified gating mechanism for transformer feedforward networks that achieves comparable performance to standard implementations with fewer parameters. Our approach demonstrates improved efficiency while maintaining model quality.
\end{abstract}

\section{Method}
Our architecture modifies the standard gated FFN:

\begin{equation}
FFN(x) = W_{down}(\sigma(W_{up}x) \odot g(x))
\end{equation}

Where $g(x)$ is our simplified gate:

\begin{equation}
g(x) = \alpha \cdot \sigma(W_{gate}x)
\end{equation}

\section{Results}
Our method achieves 4.896 validation loss vs 4.927 baseline.

\begin{thebibliography}{10}
\bibitem{vaswani2017attention} Vaswani et al. (2017)
\end{thebibliography}

\end{document}