\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{amsmath}

\title{Analysis of Adaptive Muon-Adam: Lessons from a Failed Optimizer}
\author{Aardvark}
\date{\today}

\begin{document}

\maketitle

\begin{abstract}
We analyze Adaptive Muon-Adam, an optimizer combining orthogonal gradient processing, adaptive momentum, and second-order information. Despite careful implementation, our method underperformed baselines (loss=10.854 vs Muon=3.537). We identify key failure modes and provide recommendations for future optimizer designs.
\end{abstract}

\section{Method}
\subsection{Algorithm}
The optimizer processes gradients as:

\begin{itemize}
    \item For 2D weights:
    \begin{itemize}
        \item Compute SVD: $G = U\Sigma V^T$
        \item Clip singular values: $\Sigma_{ii} = \min(\sigma_i, \tau)$
        \item Estimate diagonal Hessian: $h_i = u_i^T (\nabla^2 L) u_i$
        \item Update: $\Delta = -\eta G/(|h| + \epsilon)$
    \end{itemize}
    \item For other parameters: Standard Adam update
\end{itemize}

\section{Experiments}
\subsection{Setup}
\begin{itemize}
    \item Model: Qwen 3 (134M params)
    \item Dataset: FineWeb (10B tokens)
    \item Learning rate: $10^{-4}$ with warmup
    \item Gradient clipping: 5.0
\end{itemize}

\subsection{Results}
\begin{table}[h]
\centering
\caption{Validation Loss}
\label{tab:results}
\begin{tabular}{lc}
\toprule
Method & Loss \\
\midrule
Our Method & 10.854 \\
AdamW & 4.927 \\
Muon & 3.537 \\
\bottomrule
\end{tabular}
\end{table}

\section{Analysis}
Failure modes:

\begin{itemize}
    \item Orthogonalization caused instability
    \item Second-order estimates were noisy
    \item Update rules conflicted
\end{itemize}

\section{Conclusion}
Key lessons:

\begin{itemize}
    \item Need better orthogonalization scaling
    \item Second-order methods require stabilization
    \item Parameter grouping needs validation
\end{itemize}

\bibliographystyle{unsrt}
\bibliography{references}

\end{document}
