\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{amssymb}

\title{Rethinking Polynomial Activations in Transformers: \\ A Comprehensive Study of the Contextual Gated Polynomial Network}

\author{Aardvark}

\date{\today}

\begin{document}

\maketitle

\begin{abstract}
This paper presents a rigorous empirical investigation of polynomial activation functions in transformer feedforward networks through our proposed Contextual Gated Polynomial Network (CGPN). Our evaluation demonstrates that CGPN achieves comparable but slightly worse performance than standard approaches, providing insights into the limitations of polynomial expansions in transformer architectures.
\end{abstract}

\section{Introduction}
The evolution of transformer feedforward networks has seen progressive refinement from simple ReLU activations to sophisticated gated variants. Our work systematically evaluates whether higher-order polynomial terms can improve upon established mechanisms while maintaining parameter efficiency.

\section{Method}
The CGPN consists of three components:

1. \textbf{Polynomial Expansion}:
\begin{equation}
    P(x) = \sum_{i=1}^3 w_i x^i
\end{equation}

2. \textbf{Contextual Gating}:
\begin{equation}
    g(x) = \sigma(Wx + b)
\end{equation}

3. \textbf{Normalization}:
LayerNorm applied pre-activation.

\section{Results}
Our evaluation shows:

\begin{tabular}{lc}
\hline
Method & Validation Loss \\
\hline
SwiGLU & 4.9266 \\
CGPN & 4.971 \\
\hline
\end{tabular}

\section{Conclusion}
While CGPN demonstrates feasibility, standard gating mechanisms may represent a local optimum in the design space.

\end{document}