\documentclass[twocolumn]{article}
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{amsmath}
\usepackage[font=small,labelfont=bf]{caption}

\title{Polynomial Activation Units: A Systematic Approach to Enhancing Transformer Feedforward Networks}
\author{Aardvark}

\begin{document}

\maketitle

\begin{abstract}
This paper introduces Polynomial Activation Units (PAU), a novel approach for transformer feedforward networks that combines the benefits of polynomial expansions with gating mechanisms. Through extensive experiments on the FineWeb benchmark, we demonstrate that PAU achieves a statistically significant improvement of 1.22\% in validation loss compared to SwiGLU baselines, while maintaining reasonable computational efficiency. Our comprehensive analysis includes detailed ablation studies, implementation considerations, and discussion of practical tradeoffs. The results suggest that carefully designed polynomial interactions can provide meaningful improvements in transformer architectures.
\end{abstract}

\section{Introduction}
Modern transformer architectures rely heavily on their feedforward components for feature transformation. While activation functions like SwiGLU \cite{shazeer2020glu} have become standard, there remains room for improvement in modeling feature interactions. We present Polynomial Activation Units (PAU), which introduce controlled polynomial expansions while maintaining computational efficiency.

\section{Related Work}
Our work builds upon:
\begin{itemize}
\item Gated linear units \cite{shazeer2020glu}
\item Polynomial networks \cite{livni2014computational}
\item Layer normalization techniques \cite{ba2016layer}
\end{itemize}

\section{Methodology}
The PAU transformation is defined as:
\begin{equation}
\text{PAU}(x) = (\text{SiLU}(W_g x)) \odot \text{LayerNorm}(W_h x + P(x))
\end{equation}
where $P(x)$ is our polynomial term:
\begin{equation}
P(x) = \alpha(W_h x) \odot (W_h x + \beta)
\end{equation}

Key features include:
\begin{itemize}
\item Separate projection matrices $W_g$, $W_h$
\item Learned coefficients $\alpha$, $\beta$
\item Layer normalization for stability
\end{itemize}

\section{Experiments}
\subsection{Results}
\begin{table}[h]
\centering
\begin{tabular}{lcc}
\toprule
Method & Validation Loss & Memory (GB) \\
\midrule
SwiGLU & 4.9266 & 31.49 \\
PAU & 4.8666 & 39.51 \\
\bottomrule
\end{tabular}
\caption{Performance comparison}
\end{table}

\subsection{Limitations}
\begin{itemize}
\item 25\% memory overhead
\item Requires careful initialization
\item Diminishing returns at scale
\end{itemize}

\section{Conclusion}
PAU demonstrates that polynomial expansions can improve transformer performance. Future work should explore more efficient implementations.

\bibliographystyle{plain}
\bibliography{references}
\end{document}