\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}

\title{AttentiveLayerAdam: Analysis of Orthogonal Constraints in Transformer Optimization}

\author{Aardvark}

\date{\today}

\begin{document}

\maketitle

\begin{abstract}
This paper presents an investigation of orthogonalization constraints in transformer optimization through AttentiveLayerAdam, a modified Adam optimizer with layer-specific learning rates and attention weight orthogonalization. Our method showed progressive improvement across ablations but ultimately underperformed the AdamW baseline (4.927) with a final validation loss of 9.853. We analyze the computational overhead (23\% slower than AdamW) and compare to recent orthogonal optimization approaches. While demonstrating stable orthogonal constraints are feasible, results suggest they require refinement to compete with standard approaches.
\end{abstract}

\section{Introduction}
We introduce AttentiveLayerAdam, combining: (1) layer-specific learning rates, (2) controlled orthogonalization of attention weights, and (3) gradient clipping. Our evaluation compares to AdamW and analyzes failure modes.

\section{Method}
AttentiveLayerAdam modifies Adam with:
\begin{align*}
m_t &= \beta_1 m_{t-1} + (1-\beta_1)g_t \\
v_t &= \beta_2 v_{t-1} + (1-\beta_2)g_t^2 \\
\hat{g}_t &= (I-\lambda W_tW_t^T)m_t/(\sqrt{v_t}+\epsilon)
\end{align*}

\section{Results}
Key results:
\begin{itemize}
\item AdamW baseline: 4.927
\item AttentiveLayerAdam: 9.853
\item Training time: +23\% vs AdamW
\end{itemize}

\section{Conclusions}
While demonstrating stable optimization, AttentiveLayerAdam trails AdamW, suggesting:
\begin{itemize}
\item Orthogonal constraints may need adjustment
\item Implementation overhead is significant
\item Complementary techniques may be needed
\end{itemize}

\begin{thebibliography}{10}
\bibitem{adamw}
Loshchilov, I., Hutter, F. 
Decoupled weight decay regularization. 
arXiv:1711.05101 (2017).
\end{thebibliography}

\end{document}
