\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{booktabs}

\title{Polynomial-SILU Hybrid Activation: A Stable and Expressive Alternative for Transformer Feedforward Networks}

\author{Aardvark}

\date{\today}

\begin{document}

\maketitle

\begin{abstract}
We introduce a novel activation function for transformer feedforward networks that combines polynomial expansions with the popular SiLU activation. Our Polynomial-SILU Hybrid (PSH) learns to dynamically mix polynomial terms with SiLU through a constrained normalization scheme and adaptive mixing coefficient. Through extensive experiments on the FineWeb benchmark using a 134M parameter Qwen architecture, we demonstrate that PSH achieves consistent improvements while maintaining training stability. The method provides a simple but effective enhancement to standard feedforward layers.
\end{abstract}

\section{Introduction}
Transformer architectures have become foundational in modern language models. While the original transformer employed simple ReLU activations, subsequent innovations demonstrated the value of sophisticated activation functions. We explore whether incorporating learnable polynomial terms can enhance feedforward networks while maintaining stability.

Our key contributions include:
\begin{itemize}
\item A Polynomial-SILU Hybrid activation combining normalized polynomial terms with SiLU
\item Empirical demonstration of improved performance
\item Analysis of learned polynomial configurations
\item Comprehensive comparison with existing approaches
\end{itemize}

\section{Method}
The Polynomial-SILU Hybrid (PSH) activation combines polynomial expansions with SiLU through a learned mixing coefficient $\alpha \in \mathbb{R}$. Given input $x$, PSH computes:

\begin{equation}
\text{PSH}(x) = (1-\sigma(\alpha))\cdot\text{SiLU}(x) + \sigma(\alpha)\cdot\frac{P(x)}{\sqrt{1+x^2}}
\end{equation}

where $P(x) = w_0x + w_1x^2 + w_2x^3$ is our constrained polynomial, and $\sigma$ is the sigmoid function.

\section{Results}
Our experiments demonstrate consistent improvements over the SwiGLU baseline. Key findings include:

\begin{table}[h]
\centering
\begin{tabular}{lc}
\toprule
Method & Validation Loss \\ \midrule
PSH (ours) & 4.876 \\ 
SwiGLU Baseline & 4.927 \\ \bottomrule
\end{tabular}
\caption{Performance comparison on FineWeb benchmark}
\label{tab:results}
\end{table}

\section{Limitations}
While promising, our approach has several limitations:
\begin{itemize}
\item Modest performance improvements (1\% relative)
\item Increased computational overhead
\item Limited testing across architectures
\end{itemize}

\section{Conclusions}
We presented Polynomial-SILU Hybrid, a novel activation function combining learnable polynomial terms with SiLU. Our constrained approach achieves consistent improvements while maintaining training stability.

\begin{thebibliography}{10}
\bibitem{vaswani2017attention}
Vaswani, A. et al. \textit{Attention is All You Need}. NeurIPS 2017.
\end{thebibliography}

\end{document}