\documentclass[10pt]{article}
\usepackage[utf8]{inputenc}

\title{Dual-Activation Feedforward Networks with Dynamic Residual Scaling}
\author{Aardvark}
\date{\today}

\begin{document}

\maketitle

\begin{abstract}
We investigate a novel feedforward network architecture combining SiLU and GELU activations with dynamic residual scaling. Our experiments on the FineWeb dataset using a 134M parameter model show competitive performance (validation loss 4.993 vs SwiGLU baseline 4.927). The method provides insights into activation function interactions while maintaining simplicity.
\end{abstract}

\section{Introduction}
Transformer architectures have become fundamental in machine learning. While gated architectures dominate current practice, we explore an alternative approach through strategic activation function combination.

\section{Method}
Our architecture features:
\begin{itemize}
\item Parallel SiLU and GELU activation pathways
\item Dynamic residual scaling
\item Layer normalization
\end{itemize}

\section{Results}
Main results (validation loss):
\begin{itemize}
\item Multi-Scale Gated: 4.792
\item Dual-Gated: 4.793
\item SwiGLU Baseline: 4.927
\item Our Method: 4.993
\end{itemize}

\section{Discussion}
While not surpassing the baseline, our method shows promise. Future work should explore larger scales and alternative activations.

\end{document}