\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}

\title{Probabilistic Asymmetric Gating Units for Transformer Networks}
\author{Aardvark}
\date{\today}

\begin{document}

\maketitle

\begin{abstract}
We present Probabilistic Asymmetric Gating Units (PAGU), a novel activation function combining Gompertz asymmetry with probabilistic gating. Experimental results on language modeling show PAGU achieves competitive performance (validation loss 5.115) compared to SwiGLU (4.927), with faster early convergence.
\end{abstract}

\section{Introduction}
Transformer architectures rely heavily on their feedforward components. We introduce PAGU to improve activation function design through:
\begin{itemize}
\item Asymmetric gating better matching activation distributions
\item Explicit probabilistic interpretation
\end{itemize}

\section{Method}
PAGU is defined as:
\begin{equation}
\text{PAGU}(x) = \text{clip}(\exp(-\exp(-0.8x)) \cdot (1.2 + 0.035x^3), 0, 1.5)
\end{equation}

\section{Results}
Key results on FineWeb dataset:
\begin{itemize}
\item Validation loss: 5.115 (PAGU) vs 4.927 (SwiGLU)
\item 15% faster early convergence
\end{itemize}

\section{Conclusion}
While not surpassing SwiGLU, PAGU offers valuable insights into activation design. Future work will explore parameter optimizations.

\begin{thebibliography}{1}
\bibitem{shazeer2020glu}
Shazeer, N. \textit{GLU Variants Improve Transformer}. arXiv:2002.05202, 2020.
\end{thebibliography}

\end{document}