\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{subcaption}
\usepackage{booktabs}

\title{Layer-Adaptive Feedforward Networks with Dynamic Scaling: A Systematic Study}

\author{Aardvark}

\date{\today}

\begin{document}

\maketitle

\begin{abstract}
We present a systematic study of layer-adaptive feedforward networks in Transformers, examining three established techniques in combination: depth-dependent activations, input-dependent scaling, and learned sparsity. While each component has been explored individually in prior work, we provide the first comprehensive analysis of their combined effects. On the FineWeb benchmark using a 134M parameter Qwen 3 model, our approach shows a modest but consistent improvement (validation loss 4.910 vs 4.927 baseline), with analysis suggesting these gains come primarily from the layer-adaptive components. We discuss the practical tradeoffs and limitations of this approach, particularly the diminishing returns relative to implementation complexity.
\end{abstract}

\section{Introduction}
Recent work has shown that Transformer feedforward networks (FFNs) can benefit from specialized architectures \cite{shazeer2020glu}. However, most approaches apply uniform modifications across all layers. We investigate whether systematic layer-wise adaptation can provide additional benefits, building on three established techniques:

1) \textbf{Layer-adaptive activations}: Inspired by prior work on depth-dependent nonlinearities
2) \textbf{Dynamic scaling}: Following input-dependent feature modulation approaches
3) \textbf{Learned sparsity}: Extending adaptive sparsity methods

Our contribution is a careful empirical study of these techniques in combination.

\section{Method}
\subsection{Layer-Adaptive Activations}
We use different activation functions at different depths:

\begin{equation}
    f_l(x) = \begin{cases}
    \text{GELU}(x) & l < 6 \\
    x\sigma(1.702x) & 6 \leq l < 12 \\
    \text{SiLU}(x) & \text{otherwise}
    \end{cases}
\end{equation}

\subsection{Dynamic Scaling}
We implement input-dependent feature modulation:

\begin{equation}
    s(x) = 1 + \sigma(W_2\text{SiLU}(W_1\bar{x}))
\end{equation}

\subsection{Learned Sparsity}
We use layer-wise sparsity thresholds:

\begin{equation}
    \text{sparse}(x)_l = \text{ReLU}(x - \tau_l)
\end{equation}

\section{Experimental Setup}
We evaluate on FineWeb with a 134M parameter Qwen 3 model, using:
\begin{itemize}
    \item Batch size: 512
    \item Learning rate: 3e-4 (cosine decay)
    \item Context: 2048 tokens
\end{itemize}

\section{Results}
\begin{table}[h]
\centering
\begin{tabular}{lc}
\toprule
Component & Validation Loss \\
\midrule
Baseline (SwiGLU) & 4.927 \\
Full approach & 4.910 \\
\bottomrule
\end{tabular}
\caption{Performance comparison}
\end{table}

\section{Limitations}
Key limitations include:
\begin{itemize}
    \item Marginal performance gains
    \item Increased implementation complexity
    \item Narrow evaluation scope
\end{itemize}

\begin{thebibliography}{10}
\bibitem{shazeer2020glu}
Shazeer, N. 
Glu variants improve transformer. 
arXiv preprint arXiv:2002.05202 (2020).
\end{thebibliography}

\end{document}