\documentclass[10pt]{article}

\usepackage{fullpage}
\usepackage{setspace}
\usepackage{parskip}
\usepackage{titlesec}
\usepackage[section]{placeins}
\usepackage{xcolor}
\usepackage{breakcites}
\usepackage{lineno}
\usepackage{hyphenat}





\PassOptionsToPackage{hyphens}{url}
\usepackage[colorlinks = true,
            linkcolor = blue,
            urlcolor  = blue,
            citecolor = blue,
            anchorcolor = blue]{hyperref}
\usepackage{etoolbox}
\makeatletter
\patchcmd\@combinedblfloats{\box\@outputbox}{\unvbox\@outputbox}{}{%
  \errmessage{\noexpand\@combinedblfloats could not be patched}%
}%
\makeatother


\usepackage{natbib}




\renewenvironment{abstract}
  {{\bfseries\noindent{\abstractname}\par\nobreak}\footnotesize}
  {\bigskip}

\titlespacing{\section}{0pt}{*3}{*1}
\titlespacing{\subsection}{0pt}{*2}{*0.5}
\titlespacing{\subsubsection}{0pt}{*1.5}{0pt}


\usepackage{authblk}


\usepackage{graphicx}
\usepackage[space]{grffile}
\usepackage{latexsym}
\usepackage{textcomp}
\usepackage{longtable}
\usepackage{tabulary}
\usepackage{booktabs,array,multirow}
\usepackage{amsfonts,amsmath,amssymb}
\providecommand\citet{\cite}
\providecommand\citep{\cite}
\providecommand\citealt{\cite}
% You can conditionalize code for latexml or normal latex using this.
\newif\iflatexml\latexmlfalse
\providecommand{\tightlist}{\setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}%

\AtBeginDocument{\DeclareGraphicsExtensions{.pdf,.PDF,.eps,.EPS,.png,.PNG,.tif,.TIF,.jpg,.JPG,.jpeg,.JPEG}}

\usepackage[utf8]{inputenc}
\usepackage[english]{babel}



\usepackage{float}








\begin{document}

\title{The CUR Decomposition of Self-attention}



\author[1]{Chong Wu}%
\author[1,2]{Maolin Che}%
\author[1]{Hong Yan}%
\affil[1]{Department of Electrical Engineering and Centre for Intelligent Multidimensional Data Analysis, City University of Hong Kong}%
\affil[2]{School of Mathematics, Southwestern University of Finance and Economics}%


\vspace{-1em}



  
  \date{June 15, 2024}


\begingroup
\let\center\flushleft
\let\endcenter\endflushleft
\maketitle
\endgroup





\selectlanguage{english}
\begin{abstract}
Transformers have achieved great success in natural language processing
and computer vision. The core and basic technique of transformers is the
self-attention mechanism. The vanilla self-attention mechanism has
quadratic complexity, which limits its applications to vision tasks.
Most of the existing linear self-attention mechanisms will sacrifice
performance to some extent for reducing complexity. In this paper, we
propose a novel linear approximation of the vanilla self-attention
mechanism named CURSA to achieve both high performance and low
complexity at the same time. CURSA is based on the CUR decomposition to
decompose the multiplication of large matrices into the multiplication
of several small matrices to achieve linear complexity. Experiment
results of CURSA in image classification tasks show that it outperforms
state-of-the-art self-attention mechanisms with better data efficiency,
faster speed, and higher performance.%
\end{abstract}%



\sloppy




\selectlanguage{english}
\FloatBarrier
\end{document}

