\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}

\title{Position-Aware Gompertz Gating for Transformer Feedforward Networks}

\author{Aardvark}

\date{\today}

\begin{document}

\maketitle

\begin{abstract}
We present Position-Aware Gompertz Gating (PAGG), an improved feedforward module for transformers that systematically addresses three limitations of standard gated linear units (GLUs). Our method combines asymmetric activation with position-aware scaling and achieves a 4.889 validation loss on FineWeb, improving upon SwiGLU (4.927) while maintaining similar computational cost.
\end{abstract}

\section{Introduction}
Transformer architectures rely heavily on their feedforward networks. The prevalent SwiGLU architecture uses a symmetric SiLU activation and lacks position-awareness. Our Position-Aware Gompertz Gating addresses these limitations through:
\begin{itemize}
    \item Gompertz Linear Unit activation
    \item Learned position-dependent scaling
    \item Complete bias terms
\end{itemize}

\section{Method}
\subsection{Gompertz Linear Unit}
The Gompertz function provides desirable properties for activation functions:
\begin{equation}
    \text{GoLU}(x) = x \cdot e^{-e^{-x}}
\end{equation}

\subsection{Position-Aware Scaling}
We augment the gating mechanism with position-dependent scaling:
\begin{equation}
    s_p = 1 + \text{softplus}(W_p p + b_p)
\end{equation}

\section{Results}
Our implementation achieves:
\begin{itemize}
    \item 4.889 validation loss (vs 4.927 baseline)
    \item Consistent improvements across ablations
\end{itemize}

\section{Limitations}
\begin{itemize}
    \item Modest improvement over baseline
    \item Only validated on one model size
\end{itemize}

\begin{thebibliography}{10}
\bibitem{vaswani2017attention}
Vaswani, A. et al. \textquotedblleft Attention is all you need.\textquotedblright\ NeurIPS 2017.

\bibitem{shazeer2020glu}
Shazeer, N. \textquotedblleft GLU variants improve transformer.\textquotedblright\ arXiv:2002.05202 (2020).
\end{thebibliography}

\end{document}