\documentclass{article}
\usepackage{graphicx}
\usepackage{url}
\title{Adaptive Activation Mixing for Transformer Feedforward Networks}
\author{Aardvark}
\begin{document}

\maketitle

\begin{abstract}
We analyze adaptive activation mixing in transformer feedforward networks, combining GELU and SiLU activations with a learned gating mechanism. Our approach achieves 5.108 validation loss versus 4.927 for SwiGLU baseline, demonstrating the feasibility of dynamic activation selection with minimal parameter overhead.
\end{abstract}

\section{Introduction}
We explore mixing activation functions in transformer FFNs using a simple gating mechanism to combine GELU and SiLU paths. This allows adaptive activation selection while maintaining computational efficiency.

\section{Method}
Our architecture implements parallel GELU and SiLU paths with weights w1, w2 = softmax(Wgx). The output is w1*GELU(W1x) + w2*SiLU(W2x).

\section{Results}
On FineWeb with 83M parameter models:
\begin{itemize}
\item Our method: 5.108 loss
\item SwiGLU baseline: 4.927 loss
\item Added only 3,072 parameters
\end{itemize}

\section{Conclusion}
While not outperforming SwiGLU, our approach shows promise for dynamic activation selection with minimal overhead.

\bibliographystyle{plain}
\bibliography{references}
\end{document}