\documentclass[10pt]{article}

\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{amsmath}

\title{SpectraMix: Analyzing the Failure Modes of a Dual Momentum Optimizer for Language Models}
\author{Aardvark}
\date{\today}

\begin{document}

\maketitle

\begin{abstract}
This paper presents a thorough investigation of SpectraMix, an optimizer combining fast and slow exponential moving averages (EMAs) with adaptive mixing coefficients for language model training. Despite promising theoretical properties and successful ablation tests (loss: 11.93), SpectraMix significantly underperformed AdamW (4.93) in full-scale evaluation (loss: 12.00). We provide complete implementation details, theoretical analysis, and extensive diagnostic experiments to understand this performance gap. Our findings suggest that while dual momentum strategies appear theoretically appealing, their practical benefits for transformer optimization may be limited by complex interactions between gradient statistics across layers. This work contributes a cautionary case study in optimizer development and provides concrete recommendations for evaluating novel optimization methods.
\end{abstract}

\section{Method Details}

The SpectraMix algorithm maintains three state variables per parameter:

1. \textbf{Fast EMA} ($m_t$): $m_t = \beta_1 m_{t-1} + (1-\beta_1)g_t$
2. \textbf{Slow EMA} ($s_t$): $s_t = \beta_3 s_{t-1} + (1-\beta_3)g_t$
3. \textbf{Gradient variance} ($v_t$): $v_t = 0.9 v_{t-1} + 0.1 g_t^2$

The update combines both EMAs with adaptive mixing:
\begin{equation}
\Delta_t = \frac{m_t + \alpha_{base}/(1 + \sqrt{v_t} + \epsilon)s_t}{\sqrt{\hat{v}_t} + \epsilon}
\end{equation}

\section{Experimental Results}

\begin{table}[h]
\centering
\begin{tabular}{lc}
\toprule
Method & Validation Loss \\
\midrule
AdamW & 4.93 \\
Ademamix & 5.42 \\
SpectraMix (ours) & 12.00 \\
\bottomrule
\end{tabular}
\caption{Performance comparison}
\end{table}

\section{Conclusion}
Our analysis suggests that:
\begin{itemize}
    \item Dual momentum requires careful layer-specific tuning
    \item Gradient variance tracking needs stabilization
    \item The overhead outweighs benefits in current architectures
\end{itemize}

\end{document}
