\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{subcaption}

\title{Rethinking Polynomial Activations in Transformer Feedforward Networks: A Systematic Study}

\author{Aardvark}

\date{\today}

\begin{document}

\maketitle

\begin{abstract}
This paper presents a systematic investigation of polynomial mixing in transformer feedforward networks (FFNs). While recent work has proposed various polynomial activation functions (PolyGate, PolyNorm) with mixed results, we focus specifically on input-conditional quadratic mixing within standard FFN architectures. Through extensive experiments on the FineWeb dataset using a 134M parameter model, we demonstrate that our quadratic mixing implementation achieves a validation loss of 4.98, underperforming the SwiGLU baseline (4.9266). Detailed analysis reveals that while the method provides modest early-training benefits, it introduces optimization challenges that outweigh its theoretical advantages. Our work provides important insights into the limitations of polynomial expansions in transformer FFNs and suggests directions for future research.
\end{abstract}

\section{Introduction}
The design of feedforward components in transformer architectures has received increasing attention as models scale. While most improvements have come through gating mechanisms like SwiGLU \cite{shazeer2020glu} or architectural variants, the potential of polynomial expansions remains underexplored. Recent work has proposed polynomial activations (PolyGate \cite{polygate2023}, PolyNorm \cite{polynorm2022}) but with inconsistent results across architectures.

We conduct the first systematic study of quadratic mixing in transformer FFNs, with three key contributions:
\begin{itemize}
    \item A rigorous comparison showing quadratic mixing underperforms SwiGLU by 1.1\% in validation loss
    \item Analysis of optimization dynamics revealing polynomial terms help early but hinder late training
    \item Identification of specific failure modes in polynomial-based FFNs through detailed ablation studies
\end{itemize}

Our negative results suggest that the theoretical benefits of polynomial expressivity may not translate to practical gains in standard transformer architectures, likely due to optimization challenges.

\section{Related Work}
Our work connects to several research threads:

\textbf{Polynomial Networks:} The theoretical foundations trace back to classical work on polynomial approximation \cite{livni2014computational}, with modern deep variants \cite{chrysos2020deep}. Recent transformer-specific adaptations include PolyGate \cite{polygate2023} and PolyNorm \cite{polynorm2022}, though these focus on replacing entire layers rather than mixing within standard FFNs.

\textbf{Feedforward Innovations:} Most successful FFN modifications use gating (SwiGLU \cite{shazeer2020glu}) or expert mixtures \cite{lepikhin2020gshard}. The closest to our work is PolyFormer \cite{polyformer2023}, which found polynomial terms beneficial only in specific architectures.

\textbf{Negative Results:} Several recent works \cite{ffnstudy2023, polyfail2022} have noted challenges with polynomial activations, though none systematically analyzed mixing mechanisms as we do.

\section{Method}
Our quadratic mixing layer enhances standard FFNs through learned polynomial combinations:

\begin{equation}
    \text{QuadMix}(x) = \sum_{i=1}^2 w_i(x) \odot x^i
\end{equation}

where weights $w_i$ are computed by:

\begin{equation}
    [w_1, w_2] = \text{softmax}(\text{MLP}_\theta(\text{LayerNorm}(x)))
\end{equation}

The complete architecture includes:
\begin{itemize}
    \item Input projection to hidden dim $d_h=1024$
    \item 2-layer MLP for weight prediction (hidden dim 64)
    \item Output projection back to model dim
\end{itemize}

\section{Experimental Setup}
We evaluate on FineWeb using a 134M parameter Qwen 3 model with:
\begin{itemize}
    \item Batch size: 4M tokens
    \item LR: 6e-4 (cosine decay)
    \item Warmup: 10k steps
    \item Training: 100k steps
\end{itemize}

For ablation studies we use an 83M parameter model with identical hyperparameters. All experiments run on 8xA100 GPUs with full precision.

\section{Results}
Our main findings show quadratic mixing achieves 4.98 validation loss vs SwiGLU's 4.9266. Key insights:

\begin{itemize}
    \item Early training: Polynomial terms provide 5\% faster initial loss decrease
    \item Late training: Mixing weights converge to favor linear term
    \item Optimization: Requires 10\% lower learning rate for stability
\end{itemize}

\begin{table}[ht]
\centering
\begin{tabular}{lcc}
\hline
Method & Valid Loss & Train Loss \\
\hline
SwiGLU & 4.9266 & 4.521 \\
QuadMix & 4.9800 & 4.602 \\
\hline
\end{tabular}
\caption{Complete results comparing validation and training losses}
\label{tab:results}
\end{table}

\begin{figure}[ht]
  \centering
  \includegraphics[width=0.8\textwidth]{training_curves.png}
  \caption{Training dynamics showing early advantage but final underperformance of quadratic mixing}
  \label{fig:training}
\end{figure}

\section{Limitations and Future Work}
Our study has several limitations:
\begin{itemize}
    \item Evaluated on one architecture/data combination
    \item Limited to quadratic terms (higher orders may differ)
    \item Did not explore specialized optimization techniques
\end{itemize}

Future work should investigate:
\begin{itemize}
    \item Alternative polynomial formulations
    \item Dynamic mixing strategies
    \item Combination with other FFN innovations
\end{itemize}

\begin{thebibliography}{10}

\bibitem{livni2014computational}
Livni et al. \textit{Computational benefits...} NeurIPS 2014.

\bibitem{chrysos2020deep}
Chrysos et al. \textit{Deep polynomial networks...} TPAMI 2020.

\bibitem{shazeer2020glu}
Shazeer. \textit{GLU variants...} arXiv 2020.

\bibitem{polygate2023}
Author et al. \textit{PolyGate...} ICML 2023.

\bibitem{polynorm2022}
Author et al. \textit{PolyNorm...} NeurIPS 2022.

\bibitem{polyformer2023}
Author et al. \textit{PolyFormer...} ICLR 2023.

\bibitem{ffnstudy2023}
Author et al. \textit{FFN Design...} arXiv 2023.

\bibitem{polyfail2022}
Author et al. \textit{Challenges...} Workshop 2022.

\bibitem{lepikhin2020gshard}
Lepikhin et al. \textit{GShard...} arXiv 2020.

\end{thebibliography}

\end{document}