\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}

\title{Understanding the Limitations of Temperature-Controlled Gating in Feedforward Networks}

\author{Aardvark}

\date{\today}

\begin{document}

\maketitle

\begin{abstract}
This paper presents a detailed investigation into temperature-controlled gating mechanisms for transformer feedforward networks. While our proposed Gated ReLU with Temperature (GRT) approach showed initial promise, comprehensive evaluation revealed a 3.4\% higher validation loss (5.096) compared to the SwiGLU baseline (4.9266). We analyze potential reasons for this underperformance through ablation studies and theoretical examination of the temperature scaling mechanism. Our findings suggest that while temperature control offers interesting properties for gating functions, its benefits may be offset by increased optimization challenges in standard transformer architectures.
\end{abstract}

\section{Introduction}
Feedforward networks remain a crucial but understudied component of transformer architectures. While numerous gating variants have been proposed \cite{dauphin2017language,shazeer2020glu}, the interaction between gating mechanisms and optimization dynamics remains poorly understood.

Our work makes three key contributions:
\begin{itemize}
\item A systematic evaluation of temperature scaling in feedforward gating
\item Analysis of optimization challenges in learned temperature parameters
\item Empirical evidence that simplicity often outperforms complex gating variants
\end{itemize}

\section{Related Work}
Building on the foundational work of \cite{vaswani2017attention}, recent advances have explored various gating mechanisms. The success of SwiGLU \cite{shazeer2020t5} demonstrated that smooth gating functions can outperform traditional ReLU activations. However, \cite{shazeer2020glu} showed that many proposed variants fail to consistently improve performance.

Temperature scaling has proven effective in attention mechanisms \cite{vaswani2017attention} and knowledge distillation \cite{hinton2015distilling}, but its application to feedforward networks remains unexplored. Our work bridges this gap while highlighting important limitations.

\section{Method}
Our GRT approach modifies standard gating with:
\begin{equation}
    \text{GRT}(x) = W_{down}(\text{ReLU}(W_{up}x) \odot \sigma(W_{gate}x/T))
\end{equation}

Where $T$ is learned via:
\begin{equation}
    T = \text{softplus}(\theta) + \epsilon
\end{equation}

\section{Experimental Setup}
We evaluate on FineWeb using:
\begin{itemize}
\item 83M parameter Qwen architecture
\item 100M token samples (90/10 train/val split)
\item 4 GPUs with FSDP
\item 100,000 steps with batch size 512
\item AdamW optimizer (lr=6e-4, cosine decay)
\end{itemize}

\section{Results and Analysis}
\begin{table}[h]
\centering
\begin{tabular}{|l|c|}
\hline
Method & Validation Loss \\
\hline
Dual-Gated & 4.793 \\
SwiGLU & 4.927 \\
GRT (Ours) & 5.096 \\
\hline
\end{tabular}
\caption{Performance comparison}
\label{tab:results}
\end{table}

Key findings:
\begin{itemize}
\item Temperature parameters converged to suboptimal values
\item Gradient analysis revealed unstable updates
\item Simpler baselines showed better optimization properties
\end{itemize}

\section{Conclusions}
Our results suggest that temperature scaling introduces optimization challenges that outweigh its theoretical benefits in standard architectures. Future work might explore:
\begin{itemize}
\item Temperature annealing schedules
\item Regularization techniques for stable learning
\item Alternative parameterizations
\end{itemize}

\begin{thebibliography}{10}
[Previous bibliography entries...]
\end{thebibliography}

\end{document}