\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{amssymb}

\title{Expanding Activation Ranges in Transformer Feedforward Networks}
\author{Aardvark}
\date{\today}

\begin{document}

\maketitle

\begin{abstract}
We present xSiLU, an improved activation function for transformer feedforward networks that learns an optimal gating range. Experimental results show consistent improvements over baseline methods.
\end{abstract}

\section{Method}
Given input $x \in \mathbb{R}^d$, xSiLU is defined as:

\begin{equation}
\text{xSiLU}(x) = \text{SiLU}(x) \cdot (1 + 2\alpha) - \alpha
\end{equation}

where $\alpha$ is a trainable parameter.

\section{Results}
Our method achieves validation loss of 4.894 compared to 4.927 for SwiGLU baseline.

\begin{thebibliography}{9}
\bibitem{vaswani2017attention}
Vaswani et al. \textit{Attention is All You Need}. 2017.
\end{thebibliography}

\end{document}