\documentclass{article}
\usepackage{amsmath}
\usepackage{amssymb}
\title{Dynamic Range Gated Linear Units: An Empirical Study}
\author{Aardvark}
\date{\today}

\begin{document}

\maketitle

\begin{abstract}
We present an empirical study of Dynamic Range Gated Linear Units (DRGLU), a simplified feedforward architecture for transformers. Our experiments on FineWeb with 134M parameter models show DRGLU achieves 4.996 validation loss compared to SwiGLU's 4.9266, while requiring 25\% more memory. These negative results provide insights into the limitations of simple scaling approaches.
\end{abstract}

\section{Introduction}
We investigate whether a single learned scaling parameter can effectively replace more complex gating mechanisms in transformer feedforward networks.

\section{Methodology}
Given input $x \in R^d$, DRGLU computes:
\begin{equation}
    \text{DRGLU}(x) = W_d(\text{SiLU}(W_g x) \odot (\alpha W_u x))
\end{equation}
where $\alpha$ is a learned scalar initialized to 0.5.

\section{Results}
\begin{tabular}{lc}
\hline
Method & Validation Loss \\
\hline
SwiGLU & 4.9266 \\
DRGLU & 4.9962 \\
\hline
\end{tabular}

\section{Discussion}
Our results suggest single-parameter scaling is insufficient for modern transformers. Future work should explore more sophisticated approaches.

\end{document}