\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{subcaption}
\usepackage{booktabs}

\title{SparseGLU: A Study of Dynamic Neuron Selection in Transformer Feedforward Networks}

\author{Aardvark}

\date{\today}

\begin{document}

\maketitle

\begin{abstract}
We present a comprehensive investigation of SparseGLU, an approach to feedforward networks that dynamically selects neurons through a learned predictor. While the concept of input-dependent sparsity is theoretically appealing for efficiency, our experiments reveal significant challenges in implementation. On the FineWeb dataset with a 134M parameter Qwen model, SparseGLU achieved a validation loss of 5.02 compared to the SWiGLU baseline of 4.9266. We analyze the failure modes, including gradient flow issues from hard masking and the limitations of our predictor architecture. While not practically viable in its current form, this work provides valuable insights into the difficulties of implementing sparse activation in feedforward networks and suggests directions for future research.
\end{abstract}

\section{Introduction}
Transformer feedforward networks typically employ dense activation patterns, potentially wasting computation on irrelevant features. Recent work has explored sparse approaches like mixture-of-experts \cite{lepikhin2020gshard} and dynamic depth \cite{elbayad2020depth}, but neuron-level sparsity remains understudied. Our work systematically evaluates whether learned neuron selection can maintain model quality while reducing computation.

We make three key contributions:
\begin{itemize}
    \item A thorough empirical evaluation of predictor-based neuron selection, revealing fundamental challenges
    \item Analysis of gradient propagation issues caused by hard masking
    \item Quantitative comparison with alternative sparsity approaches
\end{itemize}

\section{Related Work}
Our work connects to several research threads:

\textbf{Sparse Transformers}: Recent work like Activator \cite{you2023activator} and Lazy Transformer \cite{li2023sparse} has explored sparse attention patterns, while Spark \cite{zhou2022sparse} examined feedforward sparsity. Our approach differs by using input-dependent neuron selection.

\textbf{Dynamic Networks}: Methods like Switch Transformers \cite{fedus2021switch} route examples to experts, while our work selects individual neurons. Depth-adaptive approaches \cite{elbayad2020depth} modify layer count rather than width.

\textbf{Gated Linear Units}: Building on GLU variants \cite{shazeer2020glu}, we explore whether sparsity can improve efficiency without quality loss.

\section{Method}
\subsection{Architecture}
SparseGLU consists of:
\begin{itemize}
    \item Standard up/gate/down projections (dim=1536, hidden=8960)
    \item Predictor network (input=32, hidden=64, output=8960)
    \item Top-k selection with k=2240 (25\%)
\end{itemize}

The predictor uses the first 32 dimensions of $x$ to compute importance scores $s = f(x_{1:32})$, then applies hard masking:

\begin{equation}
    M_{ij} = \begin{cases}
    1 & \text{if } s_{ij} \in \text{top-k}(s_i)\\
    0 & \text{otherwise}
    \end{cases}
\end{equation}

\subsection{Limitations}
Key challenges emerged during implementation:
\begin{itemize}
    \item Gradient flow through the hard mask is problematic
    \item The small predictor may lack capacity to properly rank neurons
    \item Fixed sparsity ratio may be too aggressive
\end{itemize}

\section{Experimental Setup}
We evaluate on FineWeb using:
\begin{itemize}
    \item Base Qwen architecture (134M params)
    \item Fixed compute budget (50K steps)
    \item Learning rate 6e-4 with cosine decay
    \item Batch size 256
\end{itemize}

\section{Results and Analysis}
\begin{table}[h]
\centering
\caption{Performance Comparison}
\begin{tabular}{llr}
\toprule
Method & Description & Loss \\
\midrule
SWiGLU & Baseline & 4.9266 \\
SparseGLU (Ours) & 25\% Sparse & 5.0202 \\
GEGLU & Common Variant & 4.8734 \\
\bottomrule
\end{tabular}
\end{table}

Key findings:
\begin{itemize}
    \item 1.3\% relative loss increase vs baseline
    \item Training instability from masking gradients
    \item Predictor fails to consistently identify useful neurons
\end{itemize}

\section{Discussion}
While SparseGLU underperformed, we identify several promising directions:

\textbf{Soft Sparsity}: Replace hard masking with differentiable alternatives like sparsemax \cite{louizos2018learning}.

\textbf{Predictor Design}: Larger predictors or attention-based importance scoring may improve neuron selection.

\textbf{Progressive Sparsity}: Gradually increase sparsity during training as in \cite{evci2020rigging}.

\section{Conclusion}
Our investigation of neuron-level sparsity reveals significant implementation challenges, particularly around gradient flow and predictor design. While not immediately practical, these findings provide valuable negative results for researchers exploring efficient transformer architectures.

\begin{thebibliography}{10}

\bibitem{lepikhin2020gshard}
Lepikhin, Dmitry, et al. 
Gshard: Scaling giant models with conditional computation and automatic sharding. 
arXiv preprint arXiv:2006.16668 (2020).

\bibitem{elbayad2020depth}
Elbayad, Maha, et al. 
Depth-adaptive transformer. 
ICLR 2020.

\bibitem{you2023activator}
You, Haoran, et al. 
Activator: Towards a Generalist Neural Network for Dense Prediction.
NeurIPS 2023.

\bibitem{li2023sparse}
Li, Yongjian, et al.
The Lazy Neuron Phenomenon: Understanding Emergent Sparse Networks in Transformers.
ICML 2023.

\bibitem{zhou2022sparse}
Zhou, Hattie, et al. 
Spark: A Sparse Kernel Acceleration Architecture for Transformer Networks.
MLSys 2022.

\bibitem{fedus2021switch}
Fedus, William, et al. 
Switch transformers: Scaling to trillion parameter models with simple and efficient sparsity.
JMLR 2021.

\bibitem{shazeer2020glu}
Shazeer, Noam. 
Glu variants improve transformer. 
arXiv preprint arXiv:2002.05202 (2020).

\bibitem{louizos2018learning}
Louizos, Christos, et al. 
Learning sparse neural networks through L0 regularization.
ICLR 2018.

\bibitem{evci2020rigging}
Evci, Utku, et al. 
Rigging the lottery: Making all tickets winners.
ICML 2020.

\end{thebibliography}

\end{document}