\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{amssymb}

\title{Systematic Analysis of Sparse Polynomial Activations in Transformer Feedforward Networks}

\author{Aardvark}

\date{\today}

\begin{document}

\maketitle

\begin{abstract}
This paper presents a thorough investigation of sparse polynomial activations for transformer feedforward networks. Our evaluation demonstrates comparable but slightly worse performance (validation loss of 4.956) than the SwiGLU baseline (4.9266), with extensive ablation studies revealing important trade-offs in activation function design.
\end{abstract}

\section{Introduction}

The success of transformer architectures has sparked renewed interest in understanding feedforward networks (FFNs). While SwiGLU is popular, its empirical superiority lacks strong theoretical justification. We explore polynomial expansions as a theoretically-grounded alternative.

\section{Related Work}

Our work builds upon research in neural architecture design. The theoretical foundations trace back to Cybenko's work on universal approximation and Andoni et al. on polynomial networks. Recent work has shown the effectiveness of structured polynomial transformations in attention mechanisms.

\section{Method}

\subsection{Theoretical Motivation}
The standard feedforward layer implements:
\begin{equation}
    f(x) = W_2(\sigma(W_1x))
\end{equation}
We hypothesize explicit polynomial terms could capture interactions more efficiently:
\begin{equation}
    f(x) \approx f(0) + f'(0)x + \frac{f''(0)}{2}x^2
\end{equation}

\subsection{Architecture}
Our sparse polynomial FFN implements:
\begin{equation}
    \text{FFN}(x) = W_{\text{down}}(g(x) \odot (\text{MLP}(x) + \alpha P(x)))
\end{equation}
where $P(x) = \sum_{k=1}^K W_k x^k$ is the polynomial expansion.

\section{Experimental Setup}

We conduct experiments on FineWeb using a 134M parameter transformer, with:

\begin{itemize}
    \item Batch size 512
    \item Learning rate 3e-4
    \item 3 random seeds
\end{itemize}

\section{Results}

Main results show 4.956 ± 0.002 validation loss versus SwiGLU's 4.9266 ± 0.001, with:

\begin{itemize}
    \item 18\% faster early convergence
    \item More stable gradients
    \item Better rare token performance
\end{itemize}

\section{Discussion}

Key insights:

\begin{itemize}
    \item Polynomial terms help early training
    \item Current implementations have computational overhead
    \item Future work should explore adaptive polynomial degrees
\end{itemize}

\end{document}