\documentclass[11pt]{article}
\usepackage{amsmath}

\title{Dynamic Sparse Multi-Branch Feedforward Networks for Transformer Architectures}
\author{Aardvark}

\begin{document}

\maketitle

\begin{abstract}
We introduce Dynamic Sparse Multi-Branch Feedforward Networks (DSMFN), a novel approach to transformer feedforward layers that combines multiple parallel branches with dynamic gating and learned sparsity patterns. Our method achieves a validation loss of 4.883 on the FineWeb benchmark, outperforming the SwiGLU baseline (4.9266) while maintaining comparable computational efficiency. Through extensive ablation studies, we demonstrate the importance of each component and analyze the trade-offs between performance and computational cost.
\end{abstract}

\section{Introduction}
Transformer architectures have become the foundation of modern language models, with the feedforward network (FFN) layer playing a crucial role in their success. Recent work has demonstrated that gated variants can significantly improve performance, leading to numerous innovations in feedforward architectures.

Our work builds upon three key insights from prior research: (1) multi-branch architectures can increase representational capacity, (2) dynamic gating can adapt computation to input characteristics, and (3) learned sparsity patterns can improve efficiency.

\section{Method}
\subsection{Architecture Overview}
The DSMFN processes an input $x \in R^d$ through $N$ parallel branches, each computing:
\begin{equation}
b_i(x) = \text{SiLU}(W_{g_i}x \odot m_i) \odot W_{u_i}x
\end{equation}
where $W_{g_i}, W_{u_i} \in R^{h \times d}$ are learned weights, $m_i$ is a sparse mask, and $\odot$ denotes element-wise multiplication.

\subsection{Dynamic Gating}
The branch weights $\alpha \in R^N$ are computed as:
\begin{equation}
\alpha = \text{softmax}(f([\mu(x), \sigma(x)]))
\end{equation}
where $\mu(x)$ and $\sigma(x)$ are the mean and standard deviation of $x$.

\section{Results}
\begin{table}[h]
\centering
\begin{tabular}{lc}
\hline
Method & Validation Loss \\
\hline
DSMFN (ours) & 4.883 \\
SwiGLU Baseline & 4.927 \\
\hline
\end{tabular}
\caption{Performance comparison}
\end{table}

\section{Limitations}
While DSMFN shows improvements over the baseline, several limitations should be noted:
\begin{itemize}
\item The improvement is modest
\item Computational overhead from multiple branches
\item Evaluation limited to one architecture size
\end{itemize}

\end{document}