\documentclass{article}
\usepackage{graphicx}
\usepackage{booktabs}

\title{xATLU: Expanded Gating Ranges for Transformer Feedforward Networks}
\author{Aardvark}

\begin{document}

\maketitle

\begin{abstract}
We introduce xATLU (Expanded ArcTan Linear Unit), a novel activation function for transformer feedforward networks that generalizes traditional gating mechanisms through learnable range expansion. Comprehensive experiments demonstrate that xATLU achieves consistent improvements over SwiGLU, with a 0.038 reduction in validation loss on FineWeb.
\end{abstract}

\section{Methodology}
The xATLU function is defined as:

\[
f(x) = (\frac{\arctan(x) + \pi/2}{\pi}) \cdot (1 + 2\alpha) - \alpha \cdot x
\]

Where $\alpha$ is initialized to 0 and learned during training.

\section{Experiments}
We evaluate on FineWeb using a 134M parameter model:

\begin{table}[h]
\centering
\caption{Validation Loss Comparison}
\begin{tabular}{lc}
\toprule
Method & Validation Loss \\
\midrule
SwiGLU (baseline) & 4.927 \\
xATLU (ours) & 4.889 \\
\bottomrule
\end{tabular}
\end{table}

\section{Conclusion}
xATLU demonstrates that learnable range expansion can provide improvements over fixed-range gating mechanisms.

\bibliographystyle{unsrt}
\bibliography{references}

\end{document}