\documentclass[10pt]{article}
\title{Adaptive Threshold Gating in Transformer Feedforward Networks}
\author{Aardvark}
\date{\today}

\begin{document}

\maketitle

\begin{abstract}
This paper investigates Adaptive Threshold Gating (ATG) for transformer feedforward networks. Our experiments show ATG achieves a validation loss of 4.966, slightly underperforming the SwiGLU baseline of 4.9266. The results suggest adaptive thresholds may have limited benefits in standard architectures.
\end{abstract}

\section{Introduction}
We propose ATG, which adjusts gating based on input statistics. This is inspired by biological neurons and prior work on adaptive activations.

\section{Method}
ATG modifies SwiGLU with:
\begin{equation}
FFN(x) = W_d(SiLU(W_g x) \cdot \sigma(\alpha\mu + \beta\sigma^2) \cdot W_u x)
\end{equation}
where $\mu,\sigma^2$ are input statistics and $\alpha,\beta$ are learned scalars.

\section{Results}
On a 134M parameter model:
\begin{itemize}
\item ATG loss: 4.966
\item SwiGLU loss: 4.9266
\item Memory: 39.5GB vs 31.5GB
\end{itemize}

\section{Conclusion}
ATG shows modest results, suggesting input statistics may not be optimal for gating decisions in transformers.

\bibliographystyle{plain}
\bibliography{references}

\end{document}