\documentclass{article}
\title{Quadratic Interaction Networks}
\author{Aardvark}
\begin{document}

\maketitle

\begin{abstract}
We present Quadratic Interaction Networks (QIN), a novel feedforward layer that models polynomial feature interactions. QIN achieves 4.904 perplexity on language modeling, outperforming SwiGLU (4.926). The method combines gated activations with quadratic terms while maintaining efficiency.
\end{abstract}

\section{Introduction}
Transformer architectures rely heavily on their feedforward layers. Recent work has shown improvements through gating mechanisms \cite{shazeer2020glu}. We propose Quadratic Interaction Networks (QIN) that incorporate polynomial feature interactions.

\section{Method}
QIN extends SwiGLU with quadratic terms:

$QIN(x) = W_o(gate \odot value + \alpha(interactions)^2)$

where:
\begin{itemize}
\item $gate = SiLU(W_gx)$
\item $value = W_ux$
\item $interactions = tanh(Qx)W_i$
\item $\alpha$ is a learned scalar
\end{itemize}

\section{Experiments}
We evaluate on language modeling:

\begin{tabular}{|l|l|}
\hline
Method & Val PPL \\ \hline
QIN & 4.904 \\
SwiGLU & 4.926 \\
\hline
\end{tabular}

\section{Conclusion}
QIN shows promise for modeling feature interactions.

\bibliographystyle{plain}
\bibliography{references}
\end{document}