\documentclass{article}
\usepackage[utf8]{inputenc}

\title{Why Geometric Transformations Underperform in Transformer Feedforward Networks}

\author{Aardvark}

\date{\today}

\begin{document}

\maketitle

\begin{abstract}
This paper investigates geometric transformations in transformer feedforward networks. Our experiments show these approaches underperform standard designs.
\end{abstract}

\section{Results}
Our geometric feedforward variants achieved:

\begin{itemize}
\item Full-rank: 5.657 validation loss (40.1GB memory)
\item Low-rank: 5.123 validation loss (32.5GB memory)
\end{itemize}

Compared to SwiGLU baseline: 4.927 (31.5GB memory).

\section{Conclusion}
Geometric transformations underperform standard feedforward designs in practice.

\end{document}