114b24e2Vaishali Kulkarni%----------------------------------------------------------------------------------------
314b24e2Vaishali Kulkarni%----------------------------------------------------------------------------------------
414b24e2Vaishali Kulkarni
514b24e2Vaishali Kulkarni\documentclass[11pt,fleqn,hidelinks,oneside]{book} % Default font size and left-justified equations
614b24e2Vaishali Kulkarni\usepackage[nottoc,notlot,notlof]{tocbibind}
714b24e2Vaishali Kulkarni\makeindex % Tells LaTeX to create the files required for indexing
814b24e2Vaishali Kulkarni%----------------------------------------------------------------------------------------
914b24e2Vaishali Kulkarni
1014b24e2Vaishali Kulkarni% Create a command to cleanly insert a snippet with the style above anywhere in the document
1114b24e2Vaishali Kulkarni\newcommand{\insertcode}[2]{\begin{itemize}\item[]\lstinputlisting[caption=#2,label=#1,style=Style1,float=h!]{#1}\end{itemize}} % The first argument is the script location/filename and the second is a caption for the listing
1214b24e2Vaishali Kulkarni
1314b24e2Vaishali Kulkarni\newcommand{\myref}[1]
1414b24e2Vaishali Kulkarni	{\textcolor{blue}{[\ref{#1}]}}
1514b24e2Vaishali Kulkarni
1614b24e2Vaishali Kulkarni\newcommand{\myindex}[1]
1714b24e2Vaishali Kulkarni	{\index{#1@\texttt{#1}}#1}
1814b24e2Vaishali Kulkarni
1914b24e2Vaishali Kulkarni\newcommand{\ChapterFuncs}{}
2014b24e2Vaishali Kulkarni
2114b24e2Vaishali Kulkarni%Fpr some reason, this doesn't work inside \item so we can't have this as part of \myfunc
2214b24e2Vaishali Kulkarni\newcommand{\silentfunc}[1]
2314b24e2Vaishali Kulkarni{\expandafter\def\expandafter\ChapterFuncs\expandafter{\ChapterFuncs { } \insertcode{snippets/#1_generated.h}{}}}
2414b24e2Vaishali Kulkarni
2514b24e2Vaishali Kulkarni\newcommand{\myfunc}[2]
2614b24e2Vaishali Kulkarni{\index{ZZZ@API Function!ecore\_#1@\texttt{ecore\_#1}}%
2714b24e2Vaishali Kulkarni\silentfunc{#2}\texttt{ecore\_#1()}}
2814b24e2Vaishali Kulkarni	
2914b24e2Vaishali Kulkarni\newenvironment{bottompar}{\par\vspace*{\fill}}{\clearpage}	
3014b24e2Vaishali Kulkarni	
3114b24e2Vaishali Kulkarni\newcommand{\SpillChapterFuncs}%
3214b24e2Vaishali Kulkarni%{\begin{bottompar}
3314b24e2Vaishali Kulkarni{%
3414b24e2Vaishali Kulkarni%\texttt{\textbf{\\API functions in this chapter: \\}}%
3514b24e2Vaishali Kulkarni\section{API functions discussed in this chapter}
3614b24e2Vaishali Kulkarni\ChapterFuncs{}%
3714b24e2Vaishali Kulkarni%\end{bottompar}%
3814b24e2Vaishali Kulkarni\renewcommand{\ChapterFuncs}{}}
3914b24e2Vaishali Kulkarni	
4014b24e2Vaishali Kulkarni%----------------------------------------------------------------------------------------
4114b24e2Vaishali Kulkarni
4214b24e2Vaishali Kulkarni\input{structure} % Insert the commands.tex file which contains the majority of the structure behind the template
4314b24e2Vaishali Kulkarni
4414b24e2Vaishali Kulkarni%\lstset{belowskip=-20pt plus 2pt}
4514b24e2Vaishali Kulkarni\lstset{belowskip=\smallskipamount,aboveskip=\smallskipamount,boxpos=h!,float=h!}
4614b24e2Vaishali Kulkarni\makeatletter
4714b24e2Vaishali Kulkarni\setlength{\@fptop}{5pt}
4814b24e2Vaishali Kulkarni\makeatother
4914b24e2Vaishali Kulkarni
5014b24e2Vaishali Kulkarni
5114b24e2Vaishali Kulkarni\usepackage{hyperref}
5214b24e2Vaishali Kulkarni\usepackage{verbatim}
5314b24e2Vaishali Kulkarni
5414b24e2Vaishali Kulkarni%Macros
5514b24e2Vaishali Kulkarni\newcommand{\mlist}[1]{\begin{itemize}{#1}\end{itemize}}
5614b24e2Vaishali Kulkarni\newcommand{\mlisti}[2]{\item {\textcolor{red}{#1} -- #2}}
5714b24e2Vaishali Kulkarni        
5814b24e2Vaishali Kulkarni\long\def\greybox#1{%
5914b24e2Vaishali Kulkarni    \newbox\contentbox%
6014b24e2Vaishali Kulkarni    \newbox\bkgdbox%
6114b24e2Vaishali Kulkarni    \setbox\contentbox\hbox to \hsize{%
6214b24e2Vaishali Kulkarni        \vtop{
6314b24e2Vaishali Kulkarni            \kern\columnsep
6414b24e2Vaishali Kulkarni            \hbox to \hsize{%
6514b24e2Vaishali Kulkarni                \kern\columnsep%
6614b24e2Vaishali Kulkarni                \advance\hsize by -2\columnsep%
6714b24e2Vaishali Kulkarni                \setlength{\textwidth}{\hsize}%
6814b24e2Vaishali Kulkarni                \vbox{
6914b24e2Vaishali Kulkarni                    \parskip=\baselineskip
7014b24e2Vaishali Kulkarni                    \parindent=0bp
7114b24e2Vaishali Kulkarni                    #1
7214b24e2Vaishali Kulkarni                }% 
7314b24e2Vaishali Kulkarni                \kern\columnsep%
7414b24e2Vaishali Kulkarni            }%
7514b24e2Vaishali Kulkarni            \kern\columnsep%
7614b24e2Vaishali Kulkarni        }%
7714b24e2Vaishali Kulkarni    }%
7814b24e2Vaishali Kulkarni    \setbox\bkgdbox\vbox{
7914b24e2Vaishali Kulkarni        \pdfliteral{0.75 0.75 0.75 rg}
8014b24e2Vaishali Kulkarni        \hrule width  \wd\contentbox %
8114b24e2Vaishali Kulkarni               height \ht\contentbox %
8214b24e2Vaishali Kulkarni               depth  \dp\contentbox
8314b24e2Vaishali Kulkarni        \pdfliteral{0 0 0 rg}
8414b24e2Vaishali Kulkarni    }%
8514b24e2Vaishali Kulkarni    \wd\bkgdbox=0bp%
8614b24e2Vaishali Kulkarni    \vbox{\hbox to \hsize{\box\bkgdbox\box\contentbox}}%
8714b24e2Vaishali Kulkarni    \vskip\baselineskip%
8814b24e2Vaishali Kulkarni}
8914b24e2Vaishali Kulkarni
9014b24e2Vaishali Kulkarni\newcommand{\greycom}[2]{\greybox{\textcolor{red}{#1} -- #2}}
9114b24e2Vaishali Kulkarni
9214b24e2Vaishali Kulkarni
9314b24e2Vaishali Kulkarni\global \mdfdefinestyle{MyMdStyle}{%
9414b24e2Vaishali Kulkarni	linecolor=black, linewidth=1,%
9514b24e2Vaishali Kulkarni	outerlinecolor=red,outerlinewidth=2pt,%
9614b24e2Vaishali Kulkarni	roundcorner=5pt,backgroundcolor=brown!10,nobreak=true}
9714b24e2Vaishali Kulkarni
9814b24e2Vaishali Kulkarni\newenvironment{warning}
9914b24e2Vaishali Kulkarni	{\par\begin{mdframed}[style=MyMdStyle] \begin{Warning}}
10014b24e2Vaishali Kulkarni	{\end{Warning}\end{mdframed}\vspace{5pt}\par}
10114b24e2Vaishali Kulkarni
10214b24e2Vaishali Kulkarni\newcommand{\HRule}{\rule{\linewidth}{0.5mm}}
10314b24e2Vaishali Kulkarni\newenvironment{TBD}
10414b24e2Vaishali Kulkarni	{\par\vspace{3pt}\begin{mdframed}[style=MyMdStyle,outerlinecolor=blue,%
10514b24e2Vaishali Kulkarni									  backgroundcolor=blue!10]%
10614b24e2Vaishali Kulkarni		\begin{question}}
10714b24e2Vaishali Kulkarni  {\end{question}\end{mdframed}\par}
10814b24e2Vaishali Kulkarni
10914b24e2Vaishali Kulkarni\newenvironment{NOTICE}
11014b24e2Vaishali Kulkarni  {\par\begin{mdframed}[style=MyMdStyle,outerlinecolor=black,% 
11114b24e2Vaishali Kulkarni  						linecolor=black, outerlinewidth=1.5pt]%
11214b24e2Vaishali Kulkarni    \begin{itemize}{}{\leftmargin=1cm
11314b24e2Vaishali Kulkarni                   \labelwidth=\leftmargin}\item[\Large\Info]}
11414b24e2Vaishali Kulkarni  {\end{itemize}\end{mdframed}\par}
11514b24e2Vaishali Kulkarni
11614b24e2Vaishali Kulkarni\newenvironment{REMINDER}
11714b24e2Vaishali Kulkarni	{\par\begin{mdframed}[style=MyMdStyle,outerlinecolor=blue,% 
11814b24e2Vaishali Kulkarni  						  linecolor=blue, outerlinewidth=2pt]%
11914b24e2Vaishali Kulkarni		\begin{reminder}}
12014b24e2Vaishali Kulkarni  {\end{reminder}\end{mdframed}\par}
12114b24e2Vaishali Kulkarni
12214b24e2Vaishali Kulkarni\bibliographystyle{plain}
12314b24e2Vaishali Kulkarni
12414b24e2Vaishali Kulkarni\begin{document}
12514b24e2Vaishali Kulkarni
12614b24e2Vaishali Kulkarni\begin{titlepage}
12714b24e2Vaishali Kulkarni\begin{center}
12814b24e2Vaishali Kulkarni
12914b24e2Vaishali Kulkarni% Upper part of the page. The '~' is needed because \\
13014b24e2Vaishali Kulkarni% only works if a paragraph has started.
13114b24e2Vaishali Kulkarni\includegraphics[width=0.5\textwidth]{./qlogic-logo}~\\[3cm]
13214b24e2Vaishali Kulkarni
13314b24e2Vaishali Kulkarni% Title
13414b24e2Vaishali Kulkarni\HRule \\[0.4cm]
13514b24e2Vaishali Kulkarni{ \huge \bfseries E4 ecore \\[0.4cm] }
13614b24e2Vaishali Kulkarni
13714b24e2Vaishali Kulkarni\HRule \\[1.5cm]
13814b24e2Vaishali Kulkarni
13914b24e2Vaishali Kulkarni\begin{minipage}{0.4\textwidth}
14014b24e2Vaishali Kulkarni\begin{flushleft} \large
14114b24e2Vaishali Kulkarni\emph{Authors:}\\
14214b24e2Vaishali KulkarniAriel \textsc{Elior} \\
14314b24e2Vaishali KulkarniMichal \textsc{Kalderon} \\
14414b24e2Vaishali KulkarniYuval \textsc{Mintz} \\
14514b24e2Vaishali KulkarniMerav \textsc{Sicron} \\
14614b24e2Vaishali KulkarniTomer \textsc{Tayar} \\
14714b24e2Vaishali KulkarniSudarsana Reddy \textsc{Kalluru} \\
14814b24e2Vaishali Kulkarni\end{flushleft}
14914b24e2Vaishali Kulkarni\end{minipage}
15014b24e2Vaishali Kulkarni\begin{minipage}{0.4\textwidth}
15114b24e2Vaishali Kulkarni\begin{flushright} \large
15214b24e2Vaishali Kulkarni\emph{Version:} \\
15314b24e2Vaishali Kulkarni0.0.10
15414b24e2Vaishali Kulkarni\end{flushright}
15514b24e2Vaishali Kulkarni\end{minipage}
15614b24e2Vaishali Kulkarni
15714b24e2Vaishali Kulkarni\vfill
15814b24e2Vaishali Kulkarni
15914b24e2Vaishali Kulkarni% Bottom of the page
16014b24e2Vaishali Kulkarni{\large \today}
16114b24e2Vaishali Kulkarni
16214b24e2Vaishali Kulkarni\end{center}
16314b24e2Vaishali Kulkarni\end{titlepage}
16414b24e2Vaishali Kulkarni
16514b24e2Vaishali Kulkarni\pagestyle{empty} % No headers
16614b24e2Vaishali Kulkarni\chapterimage{qlogic-full-36}
16714b24e2Vaishali Kulkarni\tableofcontents % Print the table of contents itself
16814b24e2Vaishali Kulkarni
16914b24e2Vaishali Kulkarni\cleardoublepage % Forces the first chapter to start on an odd page so it's on the right
17014b24e2Vaishali Kulkarni
17114b24e2Vaishali Kulkarni\pagestyle{fancy} % Print headers again
17214b24e2Vaishali Kulkarni
17314b24e2Vaishali Kulkarni
17414b24e2Vaishali Kulkarni%----------------------------------------------------------------------------------------
17514b24e2Vaishali Kulkarni%	Real Content
17614b24e2Vaishali Kulkarni%----------------------------------------------------------------------------------------
17714b24e2Vaishali Kulkarni\chapterimage{pictures/qlogic-full-36.jpg}
17814b24e2Vaishali Kulkarni\chapter{Introduction}
17914b24e2Vaishali KulkarniBy definition, a driver is the entity which allows an OS to drive a hardware device.
18014b24e2Vaishali KulkarniAs such the driver contains both device-specific parts and OS-specific parts.
18114b24e2Vaishali KulkarniThe Everest architecture, with programmable fastpath processors (Storms), host-based device-dedicated memory (ILT), and minimal on-chip management presents a device which requires a driver with significant portions of device-specific code.
18214b24e2Vaishali Kulkarni
18314b24e2Vaishali KulkarniDrivers will be implemented for Everest 4 devices in many OSs (linux, windows, freebsd, solaris, esx, aix, hpux���).
18414b24e2Vaishali KulkarniImplementing the device-specific code again and again in each OS is both wasteful and difficult to maintain.
18514b24e2Vaishali KulkarniFor this purpose the ecore was conceived.
18614b24e2Vaishali KulkarniA large mass of code for operating and interacting with the Everest 4 device, to be incorporated into and used by OS drivers.
18714b24e2Vaishali Kulkarni
18814b24e2Vaishali KulkarniIn the abstract, the ecore is a layer between the HW/FW and the OS.
18914b24e2Vaishali KulkarniIt is device-specific and OS-agnostic. When ecore code requires OS services (e.g. memory allocation, pci configuration space access, etc.) it calls an abstract OS function for that purpose. These are implemented in OS-specific layers.
19014b24e2Vaishali KulkarniEcore flows may be driven by the HW (e.g. by an interrupt) or by the OS specific portion of the driver (e.g. driver load/unload).
19114b24e2Vaishali Kulkarni
19214b24e2Vaishali Kulkarni\begin{itemize}
19314b24e2Vaishali Kulkarni
19414b24e2Vaishali Kulkarni	\item Slowpath flows tend to reside largely in ecore and less so in OS specific layers. As much of the functionality as possible is placed in the ecore to leverage it across multiple platforms. \\
19514b24e2Vaishali Kulkarni
19614b24e2Vaishali Kulkarni	\item Fastpath flows tend to be in the OS specific layer as too much layering and abstraction is out of place in fastpath.
19714b24e2Vaishali KulkarniHowever, the fastpath would usually be set up by ecore flows, for example the address where transmission flow should write a doorbell to the BAR is determined by the ecore at init phase and this address is supplied by ecore to the OS specific layer. \\
19814b24e2Vaishali Kulkarni
19914b24e2Vaishali Kulkarni\end{itemize}
20014b24e2Vaishali Kulkarni
20114b24e2Vaishali KulkarniDifferent drivers in the same OS may have the ecore within them, and may use it for similar or different purposes:
20214b24e2Vaishali Kulkarni
20314b24e2Vaishali Kulkarni\begin{exampleT}
20414b24e2Vaishali Kulkarni	In linux there will be an ethernet driver, an fcoe driver, an iscsi driver, a roce driver and also a slim driver for the diag utility.
20514b24e2Vaishali Kulkarni	All of these may exists in the same system.
20614b24e2Vaishali Kulkarni	All of these will have an ecore instance incorporated in them.
20714b24e2Vaishali Kulkarni	Either one of the drivers might use the ecore to initialize the device, or the sections of the device pertaining to that driver���s operation.
20814b24e2Vaishali Kulkarni	A storage driver may use the ecore for storage specific purposes, such as the initialization and allocation of task context.
20914b24e2Vaishali Kulkarni\end{exampleT}
21014b24e2Vaishali Kulkarni
21114b24e2Vaishali KulkarniThe ecore is not a driver in its own capacity, but only code which is used by other drivers. Thus, separate drivers, including separate instances of the same driver within an OS, have separate instances of the ecore within them, which are concurrently active.
21214b24e2Vaishali Kulkarni
21314b24e2Vaishali Kulkarni\section{scope}
21414b24e2Vaishali KulkarniThis document strives to define and detail what is the ecore.
21514b24e2Vaishali KulkarniThe first parts of the document deal with the concept of the ecore, and its place in the software layers between the device and the OS.
21614b24e2Vaishali KulkarniThe rest of the document deals with the content of the ecore.
21714b24e2Vaishali KulkarniThis document does not deal with the needs and use cases of any specific OS or tool, but only with the common ground which is the ecore.
21814b24e2Vaishali Kulkarni
21914b24e2Vaishali KulkarniThe document sometimes delves in-depth into the inner-workings of the ecore; Since the programmer coming to utilize the ecore might not need [or want] to know those inner workings, such a person should look into specific sections in each chapter, specifically:
22014b24e2Vaishali Kulkarni\begin{enumerate}
22114b24e2Vaishali Kulkarni	\item Chapter \ref{cha:overview}'s introduction and section \ref{sec:overview-api} for a listing of the ecore API files and their locations.
22214b24e2Vaishali Kulkarni
22314b24e2Vaishali Kulkarni	\item OS abstraction layer [\ref{sec:osal}] for functions needed to be implemented by upper-layer driver in order to support the ecore.
22414b24e2Vaishali Kulkarni	
22514b24e2Vaishali Kulkarni	\item Register-access [\ref{cha:reg}], mainly for learning about PTTs which are required by various ecore API functions.
22614b24e2Vaishali Kulkarni	
22714b24e2Vaishali Kulkarni	\item Initialization and De-initialization of the HW [ \ref{sec:init-init}, \ref{sec:init-de-init}].
22814b24e2Vaishali Kulkarni	
22914b24e2Vaishali Kulkarni	\item Status block initialization [\ref{ssec:sb-init}] and Interrupt handling flow [\ref{sec:sb-flow}].
23014b24e2Vaishali Kulkarni	
23114b24e2Vaishali Kulkarni	\item Link interface [\ref{sec:mfw-link}].
23214b24e2Vaishali Kulkarni	
23314b24e2Vaishali Kulkarni	\item Protocol related initialization/de-initialization:
23414b24e2Vaishali Kulkarni	\begin{enumerate}
23514b24e2Vaishali Kulkarni		\item L2-related, see Chapter [\ref{cha:l2}].
23614b24e2Vaishali Kulkarni	\end{enumerate}
23714b24e2Vaishali Kulkarni\end{enumerate}
23814b24e2Vaishali Kulkarni
23914b24e2Vaishali KulkarniIn addition, each chapter which includes ecore API functions that can be called by the upper-layer driver lists those functions' prototypes at its end.
24014b24e2Vaishali Kulkarni
24114b24e2Vaishali Kulkarni%\bibliography{ecore}
24214b24e2Vaishali Kulkarni
24314b24e2Vaishali Kulkarni\chapterimage{qlogic-full-36}
24414b24e2Vaishali Kulkarni\chapter{Ecore interface overview}
24514b24e2Vaishali Kulkarni\label{cha:overview}
24614b24e2Vaishali KulkarniThe ecore can be found at the perforce servers under:
24714b24e2Vaishali Kulkarni\begin{center}
24814b24e2Vaishali Kulkarni	//servers/main/nx2/579xx/drivers/ecore
24914b24e2Vaishali Kulkarni\end{center}
25014b24e2Vaishali Kulkarni
25114b24e2Vaishali KulkarniMost of the ecore consists of the \textit{inner} parts, i.e., HW-oriented implementation to which the upper-layer driver writer is oblivious.
25214b24e2Vaishali KulkarniAbove that is a concise API layer, through which the upper-layer driver should manipulate the ecore code.
25314b24e2Vaishali Kulkarni
25414b24e2Vaishali Kulkarni\section{Ecore API}
25514b24e2Vaishali Kulkarni\label{sec:overview-api}
25614b24e2Vaishali KulkarniThe Ecore API contains two types of files:
25714b24e2Vaishali Kulkarni\begin{enumerate}
25814b24e2Vaishali Kulkarni	\item Files of the format \texttt{ecore\_<module>\_api.h} -- these files are the SW API between the ecore and the upper-layer driver:
25914b24e2Vaishali Kulkarni	\begin{enumerate}
26014b24e2Vaishali Kulkarni		\item \texttt{ecore\_cxt\_api.h}.
26114b24e2Vaishali Kulkarni		\item \texttt{ecore\_dev\_api.h}.
26214b24e2Vaishali Kulkarni		\item \texttt{ecore\_fcoe\_api.h}.
26314b24e2Vaishali Kulkarni		\item \texttt{ecore\_int\_api.h}.
26414b24e2Vaishali Kulkarni		\item \texttt{ecore\_iov\_api.h}.
26514b24e2Vaishali Kulkarni		\item \texttt{ecore\_iscsi\_api.h}.
26614b24e2Vaishali Kulkarni		\item \texttt{ecore\_ll2\_api.h}.
26714b24e2Vaishali Kulkarni		\item \texttt{ecore\_roce\_api.h}.
26814b24e2Vaishali Kulkarni		\item \texttt{ecore\_sp\_api.h}.
26914b24e2Vaishali Kulkarni		\item \texttt{ecore\_vf\_api.h}.
27014b24e2Vaishali Kulkarni		\item \texttt{ecore\_mcp\_api.h}.
27114b24e2Vaishali Kulkarni	\end{enumerate}
27214b24e2Vaishali Kulkarni	\item Files of the format \texttt{ecore\_hsi\_<protocol>.h} -- these files contain the API between FW/HW and the the ecore/upper-layer driver:
27314b24e2Vaishali Kulkarni	\begin{enumerate}
27414b24e2Vaishali Kulkarni		\item \texttt{ecore\_hsi\_common.h}.
27514b24e2Vaishali Kulkarni		\item \texttt{ecore\_hsi\_eth.h}.
27614b24e2Vaishali Kulkarni		\item \texttt{ecore\_hsi\_fcoe.h}.
27714b24e2Vaishali Kulkarni		\item \texttt{ecore\_hsi\_iscsi.h}.
27814b24e2Vaishali Kulkarni		\item \texttt{ecore\_hsi\_roce.h}.
27914b24e2Vaishali Kulkarni		\item \texttt{ecore\_hsi\_tcp.h}.
28014b24e2Vaishali Kulkarni		\item \texttt{ecore\_hsi\_toe.h}.
28114b24e2Vaishali Kulkarni	\end{enumerate}
28214b24e2Vaishali Kulkarni\end{enumerate}
28314b24e2Vaishali KulkarniUpper-layer driver should not include any other ecore header file, as the rest of the header files are internal, with the following exceptions:
28414b24e2Vaishali Kulkarni\begin{itemize}
28514b24e2Vaishali Kulkarni	\item \texttt{ecore\_chain.h} -- Networking drivers will probably want to include this to benefit from the already-implemented chain.
28614b24e2Vaishali Kulkarni	\item \texttt{ecore\_utils.h} -- Useful macros which can be used by upper-layer driver.
28714b24e2Vaishali Kulkarni	\item \texttt{ecore\_status.h} -- contains \texttt{enum \_ecore\_status\_t}. Many of the ecore return values are of this type.
28814b24e2Vaishali Kulkarni\end{itemize}
28914b24e2Vaishali Kulkarni
29014b24e2Vaishali Kulkarni\begin{warning}
29114b24e2Vaishali KulkarniCurrently \texttt{ecore.h, ecore\_proto\_if.h} should also be included by upper-layer driver; This will (hopefully) be fixed shortly.
29214b24e2Vaishali Kulkarni\end{warning}
29314b24e2Vaishali Kulkarni
29414b24e2Vaishali Kulkarni
29514b24e2Vaishali Kulkarni\section{Ecore Internal files}
29614b24e2Vaishali KulkarniThis lists the ecore files, giving each a short description:
29714b24e2Vaishali Kulkarni
29814b24e2Vaishali Kulkarni\begin{itemize}
29914b24e2Vaishali Kulkarni	\item \texttt{ecore\_attn\_values.h}
30014b24e2Vaishali Kulkarni	
30114b24e2Vaishali Kulkarni	\item \texttt{ecore\_chain.h} -- Implements a cyclic chain; Used for various interfaces with the FW [Buffer-Descriptoss, Event Queues, etc.].
30214b24e2Vaishali Kulkarni	
30314b24e2Vaishali Kulkarni	\item \texttt{ecore\_cxt\_api.[ch]} -- Handles the allocation, configuration and distribution of contexts to the various clients. 
30414b24e2Vaishali Kulkarni	
30514b24e2Vaishali Kulkarni	\item \texttt{ecore\_dbg\_fw\_funcs.[ch], ecore\_dbg\_values.h, ecore\_fw\_defs} -- Files which contain code related for various debug features ecore can provide [e.g., grcDump].
30614b24e2Vaishali Kulkarni	
30714b24e2Vaishali Kulkarni	\item \texttt{ecore\_fcoe.[ch], ecore\_iscsi.[ch], ecore\_ll2.[ch], ecore\_roce.[ch]} -- files containing specific ecore code for the storage protocols.
30814b24e2Vaishali Kulkarni	
30914b24e2Vaishali Kulkarni	\item \texttt{ecore\_dev.[ch]} -- Contains much of the functionality of starting/stopping the hardware. See chapter \ref{cha:hwinit}.
31014b24e2Vaishali Kulkarni	
31114b24e2Vaishali Kulkarni	\item \texttt{ecore\_hw.[ch], ecore\_gtt\_reg\_addr.h, ecore\_gtt\_values.h} -- contains the functionality for register access and DMAE. See chapter \ref{cha:reg}.
31214b24e2Vaishali Kulkarni	
31314b24e2Vaishali Kulkarni	\item \texttt{ecore.h} -- contains the defintion of the most \textit{elementary} structures in the ecore, the \texttt{ecore\_dev} and the \texttt{ecore\_hwfn}.
31414b24e2Vaishali Kulkarni	
31514b24e2Vaishali Kulkarni	\item \texttt{ecore\_init\_defs.h, ecore\_init\_fw\_funcs.[ch], ecore\_init\_ops.[ch], \\ ecore\_init\_values.h, ecore\_rt\_defs} -- Code responsible for initialization and configuration of the HW and loading of the FW, mostly in relation with the init-tool. See chapter \ref{cha:hwinit}.
31614b24e2Vaishali Kulkarni	\begin{REMINDER}
31714b24e2Vaishali Kulkarni			Chapter \ref{cha:hwinit} doesn't really give a thorough explanation of the init tool - at most it mentions it. Do we want a section/chapter of it somewhere?
31814b24e2Vaishali Kulkarni	\end{REMINDER}
31914b24e2Vaishali Kulkarni	
32014b24e2Vaishali Kulkarni	\item \texttt{ecore\_int.[ch]} -- Handles interrupts and attentions. See chapter \ref{cha:int}.
32114b24e2Vaishali Kulkarni
32214b24e2Vaishali Kulkarni	\item \texttt{ecore\_iro.h, ecore\_iro\_values.h} -- Generated FW files. Enables ecore to access [or supply to upper-layer] addresses inside the \texttt{storm}'s RAM.
32314b24e2Vaishali Kulkarni
32414b24e2Vaishali Kulkarni	\item \texttt{ecore\_mcp.[ch]} -- Contains the interface between the ecore and the MFW. See chapter \ref{cha:mfw}.
32514b24e2Vaishali Kulkarni
32614b24e2Vaishali Kulkarni	\item \texttt{ecore\_sp\_commands.[ch], ecore\_spq.[ch]} -- Contained the slowpath logic required for sending ramrods and configuring \& handling the various slowpath events.
32714b24e2Vaishali Kulkarni	
32814b24e2Vaishali Kulkarni	\item \texttt{ecore\_sriov.[ch], ecore\_vf.[ch], ecore\_vfpf\_if.h} -- Contains the SRIOV implementation both from the PF and VF sides.
32914b24e2Vaishali Kulkarni\end{itemize}
33014b24e2Vaishali Kulkarni
33114b24e2Vaishali Kulkarni\section{OS abstraction Layer}
33214b24e2Vaishali Kulkarni\label{sec:osal}
33314b24e2Vaishali Kulkarni
33414b24e2Vaishali Kulkarni%\section{Driver Core}
33514b24e2Vaishali Kulkarni%As the ecore contains most of the lowlevel code operating the non-fastpath parts of the working with the HW and FW, it can be thought of as some sort of library ��� it contains bits of code meant to be operated from an outside source. Each OS needs to implement its own driver, calling the various functions in the ecore API in a place fitting for that OS driver flows.
33614b24e2Vaishali Kulkarni%Each OS will independently need to create a driver that incorporates the ecore, both filling the OS dependent callbacks required by the ecore to perform and supply an upper level of abstraction which best suits that OS. Notice this upper layer is sometimes also, mistakenly, referred to as ecore [e.g., bnx2c for linux drivers] but there���s an important distinction:
33714b24e2Vaishali Kulkarni%\begin{itemize}
33814b24e2Vaishali Kulkarni%	\item Ecore ��� shared code between ALL operating systems.
33914b24e2Vaishali Kulkarni%	\item Upper-Layer ��� shared code by all drivers on a single operating system.
34014b24e2Vaishali Kulkarni%\end{itemize}
34114b24e2Vaishali Kulkarni
34214b24e2Vaishali Kulkarni%It���s possible [and likely] that an operating system will break the various protocols into different sub-drivers, where each sub-driver will be designated for a specific protocol. Notice that if such separation is made, the preferred implementation is that the OS will implement a ���core��� driver consisting of the Ecore and an upper-layer, and define an API through which the various protocol drivers communicate with the OS core driver\footnote{Although notice there should be no inter-dependencies between HW-functions in the ecore, so the alternative method where each contains the ecore is also feasible}.
34314b24e2Vaishali Kulkarni
34414b24e2Vaishali KulkarniThe ecore utilizes various functions which should be implemented by the upper layer. There are two main ���types��� of functions:
34514b24e2Vaishali Kulkarni\begin{enumerate}
34614b24e2Vaishali Kulkarni	\item Basic OS-specific operations that the ecore needs in order to perform it���s work; e.g., memory allocations ��� the ecore needs to allocate memory for various reasons, and it needs the upper layer to supply the method by which it can do so.
34714b24e2Vaishali Kulkarni	\item Hooks by which the upper-layer can run additional OS specific code, or make decisions affecting the work of the ecore. E.g., in the SRIOV flows, the mechanism for passing messages from VF to PF is implemented in the ecore but the decision whether a request is valid or not might be OS specific ��� as in the case of unicast filters.
34814b24e2Vaishali Kulkarni\end{enumerate}
34914b24e2Vaishali Kulkarni
35014b24e2Vaishali KulkarniThe various functions that need to be implemented by the upper-layer can be found in Appendix \ref{app:osal} -- OSAL Documentation.
35114b24e2Vaishali Kulkarni
35214b24e2Vaishali Kulkarni
35314b24e2Vaishali Kulkarni\section{Ecore print scheme}
35414b24e2Vaishali KulkarniThe ecore utilizes several printing methods to print messages to the system logs; It requires some functions to be implemented by the upper-layer for this to work ��� the required documentation can be found in Appendix \ref{app:osal} -- OSAL Documentation.
35514b24e2Vaishali KulkarniIn order to support this, the verbosity mechanism contains two distinct values \myindex{\texttt{DP\_LEVEL}} and \myindex{\texttt{DP\_MODULE}} [both can be found in \texttt{ecore.h}]. Since the printing scheme in the ecore was defined with the linux limitations in mind ��� that is, the API [via ethtool] allowing the setting of the debug message level is only 32-bit long, both \texttt{DP\_MODULE} and \texttt{DP\_LEVEL} together contain only 32-bits.
35614b24e2Vaishali KulkarniThe \texttt{DP\_LEVEL} determines which prints will actually reach the logs based on the message urgency, defining 4 levels ��� verbose, info, notice and error. When level is set, all prints which are at least as urgent will be printed. Notice this means there���s a single level ��� e.g., you can���t have a configuration in which you���ll get all the `info��� level prints, but not the `notice��� level.
35714b24e2Vaishali KulkarniThe \texttt{DP\_MODULE} is relevant only when level is set to verbose, and it defines which of the verbose prints should reach system logs, based mostly on component/flow. When setting the module level, a bit mask of the requested components/flows is set.
35814b24e2Vaishali KulkarniIn order to set which prints should reach system logs, the upper layer should utilize the ecore function \myfunc{init\_dp}{init_dp} defined in \texttt{ecore\_dev.c}.
35914b24e2Vaishali Kulkarni
36014b24e2Vaishali Kulkarni\section{Compilation flags}
36114b24e2Vaishali KulkarniThe ecore project contains several optional compilation flags that if passed would affect the content compiled. A few notable flags:
36214b24e2Vaishali Kulkarni\begin{itemize}
36314b24e2Vaishali Kulkarni	\item ASIC\_ONLY -- By default, this is `off'. Setting this would remove content that is relevant only for simulations of the hardware, I.e., emulations and FPGAs.
36414b24e2Vaishali Kulkarni
36514b24e2Vaishali Kulkarni	\item REAL\_ASIC\_ONLY -- By default, this is `off'. Setting this would remove content that is relevant for non-productized hardware, E.g., workarounds for BigBear A0.
36614b24e2Vaishali Kulkarni
36714b24e2Vaishali Kulkarni	\item REMOVE\_DBG -- By default, this is `off'. There are several structures and field in ecore which aren't functional; there sole purpose is to store interesting data for memory dumps in case of failures. Setting this would remove all such data items.
36814b24e2Vaishali Kulkarni\end{itemize}
36914b24e2Vaishali Kulkarni
37014b24e2Vaishali Kulkarni\SpillChapterFuncs
37114b24e2Vaishali Kulkarni
37214b24e2Vaishali Kulkarni
37314b24e2Vaishali Kulkarni\chapterimage{qlogic-full-36}
37414b24e2Vaishali Kulkarni\chapter{Register Access}
37514b24e2Vaishali Kulkarni\label{cha:reg}
37614b24e2Vaishali KulkarniThis section describes the ecore API for accessing registers. 
37714b24e2Vaishali KulkarniThe E4 bar is a reduced BAR, i.e., it does not map the entire register address range.
37814b24e2Vaishali KulkarniTo access the entire range, windows are defined that can be configured to point to a certain address within the device and allow reading and writing of registers / memory from that address.
37914b24e2Vaishali KulkarniThere are two types of windows, \textbf{PTT} (per PF Translation Table) and \textbf{GTT} (Global Translation Table). 
38014b24e2Vaishali Kulkarni
38114b24e2Vaishali KulkarniThe \textit{external BAR} is the BAR accessed by the ecore. It is divided into configurable windows which point to different areas within the device (Image \ref{fig:bars}, Internal BAR vs. External BAR, demonstrates this).
38214b24e2Vaishali Kulkarni
38314b24e2Vaishali Kulkarni\begin{figure}[ht]
38414b24e2Vaishali Kulkarni	\caption{Internal BAR vs. External BAR}
38514b24e2Vaishali Kulkarni	\centering
38614b24e2Vaishali Kulkarni	\includegraphics[width=0.8\paperwidth]{reg_access}
38714b24e2Vaishali Kulkarni	\label{fig:bars}
38814b24e2Vaishali Kulkarni\end{figure}
38914b24e2Vaishali Kulkarni
39014b24e2Vaishali KulkarniFor more details on the E4 BAR access scheme the reader is referred to the ���Reduced PF BAR0 size��� section of \cite{doc:PXP}. \\
39114b24e2Vaishali Kulkarni
39214b24e2Vaishali Kulkarni
39314b24e2Vaishali KulkarniAll register access should be done within the ecore layer and it is not expected for the upper layers to access registers at all.
39414b24e2Vaishali KulkarniFor this reason, there is no description here on how to find the register address and how to distinguish whether the address is mapped into a \myindex{GTT} or a \myindex{PTT}.
39514b24e2Vaishali KulkarniHowever, in case a need does rise in the future, API for reading/writing is detailed below as well.
39614b24e2Vaishali Kulkarni
39714b24e2Vaishali KulkarniEcore requires an OSAL implementation of the macros:
39814b24e2Vaishali Kulkarni\begin{enumerate}
39914b24e2Vaishali Kulkarni	\item \myindex{REG\_RD}
40014b24e2Vaishali Kulkarni	\item \myindex{REG\_WR}
40114b24e2Vaishali Kulkarni\end{enumerate}
40214b24e2Vaishali KulkarniThese macros are a direct read / write from the BAR with the absolute address offset given. 
40314b24e2Vaishali KulkarniImplementation should add the offset to the mapped BAR address and call the appropriate OS specific API. 
40414b24e2Vaishali Kulkarni
40514b24e2Vaishali KulkarniSeveral ecore interface functions require a PTT. There is a pool of PTTs maintained by ecore.
40614b24e2Vaishali KulkarniThe reason there are several PTTs is to enable simultaneous access to device registers from different flows.
40714b24e2Vaishali KulkarniThe PTT is reserved per flow, and it is the responsibility of the upper layer to make sure it does not use the same PTT in flows that can run concurrently. Upper-layer requests for a PTT entry using \myfunc{ptt\_acquire}{ptt_acquire}. 
40814b24e2Vaishali KulkarniHowever, to avoid running out of this resource, it is also the responsibility of the upper layer not to acquire too many PTTs without releasing them. Returning a PTT entry back to the pool is done via \myfunc{ptt\_release}{ptt_release}.
40914b24e2Vaishali Kulkarni
41014b24e2Vaishali KulkarniUsing a PTT, ecore [and upper-driver] can access registers/memories using inner BAR addresses; The ecore is responsible for configuring the memory windows, and translates the inner address into an external address [i.e., one which resides on the actual BAR as seen by the host]. The register access is then made by calling \texttt{ecore\_wr} and \texttt{ecore\_rd}.
41114b24e2Vaishali Kulkarni\SpillChapterFuncs
41214b24e2Vaishali Kulkarni
41314b24e2Vaishali Kulkarni
41414b24e2Vaishali Kulkarni\chapterimage{qlogic-full-36}
41514b24e2Vaishali Kulkarni\chapter{Hardware/Firmware initialization}
41614b24e2Vaishali Kulkarni\label{cha:hwinit}
41714b24e2Vaishali Kulkarni
41814b24e2Vaishali Kulkarni\section{Basic concepts -- inner-working of the ecore}
41914b24e2Vaishali Kulkarni\begin{itemize}
42014b24e2Vaishali Kulkarni	\item \myindex{ILT} ��� one of the features of our device is that the memories used by various HW blocks are allocated on the host memory [as opposed to large embedded memory segment on chip]. The driver is responsible for allocating the memory needed for those HW blocks [DMA-coherent memory] and configure both the HW blocks themselves and a communal sub-block known as ILT. The ecore contains complicated code that decides exactly how much memory each such block needs, allocates it in an ���ilt\_shadow���, and then uses that shadow to configure the ILT itself with all the allocated chunks of memory. 
42114b24e2Vaishali Kulkarni	
42214b24e2Vaishali KulkarniAdditional ILT documentation is at \cite{doc:ILT}.	
42314b24e2Vaishali Kulkarni
42414b24e2Vaishali Kulkarni	\item \myindex{RT array} ��� when the ecore initializes the HW, it utilizes a common, tool-generated code known as the init-tool. Since there are quite a few values which depend upon actual setup configuration and thus must receive feedback during the initialization from the ecore, instead of adding many such hooks there���s the concept of the RunTime array ��� an array of values filled by the ecore prior to the init-tool run based on the complex ecore logic. The init-tool will then utilize the values in that array to configure the HW according to the correct order of configuration [i.e., writing the values set by ecore in the array in the correct place in the initialization flow where they���re required/the block that contains them is configured].
42514b24e2Vaishali Kulkarni\end{itemize}
42614b24e2Vaishali Kulkarni
42714b24e2Vaishali Kulkarni\section{Initialization}
42814b24e2Vaishali Kulkarni\label{sec:init-init}
42914b24e2Vaishali KulkarniThe functions required for initializing the HW/FW mostly reside in \texttt{ecore\_dev.[ch]}; More accurately, most of the outside API [toward the upper-layer] is in \texttt{ecore\_dev.h} ��� the functions themselves utilize many other ecore files.
43014b24e2Vaishali KulkarniThis section gives a brief description of the functions that need to be called, what they do, requirements, etc., in order to successfully initialize the ecore structs and load the HW/FW.
43114b24e2Vaishali Kulkarni
43214b24e2Vaishali Kulkarni\silentfunc{init_struct}
43314b24e2Vaishali Kulkarni\silentfunc{hw_prepare}
43414b24e2Vaishali Kulkarni\silentfunc{resc_alloc}
43514b24e2Vaishali Kulkarni\silentfunc{resc_setup}
43614b24e2Vaishali Kulkarni\silentfunc{hw_init}
43714b24e2Vaishali Kulkarni\begin{itemize}
43814b24e2Vaishali Kulkarni	\item \myfunc{init\_struct}{init\_struct} ��� After allocating and setting of zeroes of the ecore\_dev [the upper-layer responsibility], a pointer to it should be passed to this function for some early initialization of the data structure. \\
43914b24e2Vaishali Kulkarni	
44014b24e2Vaishali Kulkarni	\item \myfunc{hw\_prepare}{hw_prepare} ��� This function serves two purposes [plus some additional inner ecore workings]:
44114b24e2Vaishali Kulkarni	\begin{enumerate}
44214b24e2Vaishali Kulkarni		\item It enables the ecore to access its BAR, doing things such as enabling the PTT pool and opening the access in the PGLUE\_B block.
44314b24e2Vaishali Kulkarni		Notice this doesn���t actually do anything to the PCI BAR itself ��� the upper-layer should have initialized those before calling this function, and must guarantee that its REG\_WR/RD functions actually point to valid, accessible addresses.
44414b24e2Vaishali Kulkarni		\item It learns as much as it can about system configuration from HW and SHMEM. 
44514b24e2Vaishali Kulkarni	\end{enumerate}
44614b24e2Vaishali Kulkarni
44714b24e2Vaishali KulkarniTrying to access registers except for pci-related ones prior to calling this function will fail. \\
44814b24e2Vaishali Kulkarni
44914b24e2Vaishali Kulkarni	\item \myfunc{resc\_alloc}{resc_alloc} ��� Allocates the various ecore-related memory, e.g., contexts, slowpath queue, SRIOV information, etc. Notice that before calling this function, each HW-function of the \texttt{ecore\_dev} should have its `pf\_params��� set, as the function depends upon the protocol-specific resources for its calculations. \\
45014b24e2Vaishali Kulkarni	
45114b24e2Vaishali Kulkarni	\item \myfunc{resc\_setup}{resc_setup} ��� Configures the various slowpath elements. Notice that since there���s no guarantee chip is alive at this point [i.e., it���s very likely the chip is reset at this point], it fills the configuration in the runtime array instead of actually writing it to chip. \\
45214b24e2Vaishali Kulkarni
45314b24e2Vaishali Kulkarni	\item \myfunc{hw\_init}{hw_init} ��� This function actually initializes the chip, using the init-tool and the runtime array to make the correct configuration.
45414b24e2Vaishali Kulkarni	 As part of the slowpath interrupt enablement, ecore invokes OSAL\_SLOWPATH\_IRQ\_REQ() callback for each HW function. The client implementation should setup the IRQ handlers for slowpath interrupt handling.
45514b24e2Vaishali Kulkarni	 This is required since as part of the flow the \texttt{function\_start} ramrod will be sent to FW; Once FW finishes handling it, an \myindex{EQE} [Event Queue Element] will be placed in the slowpath event queue and an interrupt will be fired. The flow is dependent on the EQE being processed.
45614b24e2Vaishali Kulkarni	 
45714b24e2Vaishali Kulkarni	Some interesting sub-functions of the \texttt{ecore\_hw\_init()} method, at least for debugging purposes as many possible errors can be caught there:
45814b24e2Vaishali Kulkarni	\begin{itemize}
45914b24e2Vaishali Kulkarni		\item \texttt{ecore\_get\_init\_mode()} ��� this creates a bitmask which will be later passed to the init-tool which describes the configured mode ��� Multi function vs. Single function, 40G vs. 100G etc. A wrong configuration here could explain many peculiar events later on. \\
46014b24e2Vaishali Kulkarni
46114b24e2Vaishali Kulkarni%		\item ecore\_mcp\_load\_req() ��� the MFW [assuming it is present] will answer with one of 3 possible answers: ENGINE, PORT or FUNCTION.
46214b24e2Vaishali Kulkarni%		The MFW is responsible for initializing the common blocks [i.e., the HW blocks shared between the 2 engines], but the driver is responsible for the rest.
46314b24e2Vaishali Kulkarni%		Each function needs to perform different initialization based on whether it���s the first to load on its engine [ENGINE], the first to load on its port [PORT] or if it���s being loaded on an already initialized port [FUNCTION]\footnote{Initialization which is common for both engines will be performed by the MFW.}.
46414b24e2Vaishali Kulkarni%	Some very basic errors can be detected here, if the function receives an unexpected answer from MFW.
46514b24e2Vaishali Kulkarni	\end{itemize}
46614b24e2Vaishali Kulkarni
46714b24e2Vaishali KulkarniOnce this function returns, the chip is initialized, FW is functional and slowpath event queues are operational.	
46814b24e2Vaishali Kulkarni
46914b24e2Vaishali Kulkarni\end{itemize}
47014b24e2Vaishali Kulkarni
47114b24e2Vaishali Kulkarni\section{Zipped and Binary firmware}
47214b24e2Vaishali Kulkarni\label{sec:init-Zipped and Binary firmware}
47314b24e2Vaishali Kulkarni\begin{itemize}
47414b24e2Vaishali Kulkarni	\item \myindex{Zipped Firmware} - There are two types of firmware files generated in ecore.\\
47514b24e2Vaishali KulkarniNon-zipped firmware [ecore\_init\_values.h and ecore\_init\_values.bin] and Zipped firmware [ecore\_init\_values\_zipped.h and
47614b24e2Vaishali Kulkarniecore\_init\_values\_zipped.bin] files. Each type of file is generated in two formats that is a C header file and binary file,
47714b24e2Vaishali Kulkarniwhere each has all relevant data needed to initialize the firmware. Either of these file types can be used for firmware initialization.
47814b24e2Vaishali KulkarniThe difference is that Zipped firmware files has lot of dmae firmware data zipped which is beneficiary in reducing the code size.\\
47914b24e2Vaishali Kulkarni
48014b24e2Vaishali KulkarniBy default, the non-zipped variant is used. If ecore clients want to use zipped version of firmware then they need to have
48114b24e2Vaishali KulkarniCONFIG\_ECORE\_ZIPPED\_FW defined/enabled by their operating system drivers to make feature operational. For unzipping the
48214b24e2Vaishali Kulkarnizipped firmware data ecore clients need to implement OSAL\_UNZIP\_DATA() as well. This OSAL is meant for unzipping the
48314b24e2Vaishali Kulkarnizipped firmware data in order to do firmware initialization.\\
48414b24e2Vaishali Kulkarni
48514b24e2Vaishali Kulkarni	\item \myindex{Binary Firmware} - As explained above there are two formats of firmware files
48614b24e2Vaishali Kulkarnigenerated by ecore, C header files [ecore\_init\_values.h and ecore\_init\_values\_zipped.h] and
48714b24e2Vaishali Kulkarnibinary firmware files [ecore\_init\_values.bin and ecore\_init\_values\_zipped.bin]. Either of those files formats
48814b24e2Vaishali Kulkarnican be used by ecore clients to utilize firmware data. By default, ecore uses the .h files which are compiled as part of the ecore,
48914b24e2Vaishali Kulkarnibut using binary firmware files has the advantage where the code size is reduced and the FW can be loaded from a file imported by
49014b24e2Vaishali Kulkarnithe system.\\
49114b24e2Vaishali Kulkarni
49214b24e2Vaishali KulkarniIf ecore clients want to use firmware data from binary files then they need to have CONFIG\_ECORE\_BINARY\_FW defined/enabled by their
49314b24e2Vaishali Kulkarnioperating system drivers to make feature operational. Ecore clients must store all binary firmware data from the
49414b24e2Vaishali Kulkarnifile in to a void* pointer and pass that firmware data buffer pointer in ecore\_hw\_init() as an argument.
49514b24e2Vaishali KulkarniIf ecore client is not using binary firmware file or instead using firmware from regular header files then they
49614b24e2Vaishali Kulkarnishould pass NULL as an argument for binary firmware data buffer in ecore\_hw\_init().
49714b24e2Vaishali Kulkarni
49814b24e2Vaishali Kulkarni
49914b24e2Vaishali Kulkarni\end{itemize}
50014b24e2Vaishali Kulkarni
50114b24e2Vaishali Kulkarni\section{De-Initialization}
50214b24e2Vaishali Kulkarni\label{sec:init-de-init}
50314b24e2Vaishali Kulkarni\silentfunc{hw_stop}
50414b24e2Vaishali Kulkarni\silentfunc{resc_free}
50514b24e2Vaishali Kulkarni\silentfunc{hw_remove}
50614b24e2Vaishali Kulkarni\begin{itemize}
50714b24e2Vaishali Kulkarni	\item \myfunc{hw\_stop}{hw_stop} ��� this function notifies the MFW that the HW-functions unload, stops the FW/HW for all HW-functions in the \texttt{ecore\_dev} including sending the common PF\_STOP ramrod for each HW-function, and disables the HW-functions in various HW blocks.
50814b24e2Vaishali Kulkarni	Notice that before calling this, all the protocol specifics done after initializing the HW should have already been reversed by the upper-layer [e.g., L2 VPORTs which were started by the upper layer should be stopped before calling this].
50914b24e2Vaishali Kulkarni	Following this function, it is guaranteed HW will not generate any more slowpath interrupts, so the interrupt handler can be released [and slowpath DPC context can be stopped]. \\
51014b24e2Vaishali Kulkarni	
51114b24e2Vaishali Kulkarni	\item \myfunc{int\_disable\_post\_isr\_release}{ecore_int_disable_post_isr_release} ��� this function performs the required IRQ related cleanup post the ISR release. The function need to be called after releasing all slowpath IRQs of the device.
51214b24e2Vaishali Kulkarni
51314b24e2Vaishali Kulkarni	\item \myfunc{resc\_free}{resc_free} ��� Releases the memory allocated by the ecore during \texttt{ecore\_resc\_alloc()}. \\
51414b24e2Vaishali Kulkarni	
51514b24e2Vaishali Kulkarni	\item \myfunc{hw\_remove}{hw_remove} ��� Release the memory allocated early by the ecore during \texttt{ecore\_hw\_prepare()}.
51614b24e2Vaishali Kulkarni	Following this, REG\_RD/REG\_WR are no longer operational - upper layer can disable the PCI BAR.
51714b24e2Vaishali Kulkarni\end{itemize}
51814b24e2Vaishali Kulkarni\SpillChapterFuncs
51914b24e2Vaishali Kulkarni
52014b24e2Vaishali Kulkarni%\chapterimage{qlogic-full-36}
52114b24e2Vaishali Kulkarni%\chapter{Firmware hsi}
52214b24e2Vaishali Kulkarni%\begin{NOTICE}
52314b24e2Vaishali Kulkarni%Placeholder - owner Michal
52414b24e2Vaishali Kulkarni%\end{NOTICE}
52514b24e2Vaishali Kulkarni
52614b24e2Vaishali Kulkarni
52714b24e2Vaishali Kulkarni\chapterimage{qlogic-full-36}
52814b24e2Vaishali Kulkarni\chapter{Interrupts}
52914b24e2Vaishali Kulkarni\label{cha:int}
53014b24e2Vaishali KulkarniThis chapter describes how the device notifies the driver about operations -
53114b24e2Vaishali Kulkarniit describes how firmware status is reflected on host memory via status blocks, and how the firmware initiates an interrupt toward the driver.
53214b24e2Vaishali Kulkarni
53314b24e2Vaishali KulkarniA reference document that fully describes status blocks can be found at \cite{doc:SB}.
53414b24e2Vaishali Kulkarni
53514b24e2Vaishali Kulkarni
53614b24e2Vaishali Kulkarni\section{Status blocks - host point of view}
53714b24e2Vaishali KulkarniThe \myindex{status block} structures are allocated on host memory. The status block is an array of indices which are updated by firmware (mainly ring consumer values). 
53814b24e2Vaishali KulkarniThere are 288 status blocks per path in Big Bear and 368 in K2.
53914b24e2Vaishali Kulkarni
54014b24e2Vaishali KulkarniWhen one of the indices on a status block is updated (because some event occurred at the device), the status block is copied from internal device memory to host memory, and an interrupt is generated.
54114b24e2Vaishali KulkarniThe CAU unit may aggregate several events and generate a single update of the status block and a single interrupt, in order to lower the number of interrupts sent to host CPU.
54214b24e2Vaishali Kulkarni 
54314b24e2Vaishali KulkarniThe indices of the status blocks are referred to as \myindex{protocol indices} (abbreviated to \textit{pi}).
54414b24e2Vaishali KulkarniOriginally, the motivation behind multiple status blocks was to enable multiple protocols to work with the same status block, giving each protocol a different index.
54514b24e2Vaishali KulkarniHowever, with single personality this is no longer the case.
54614b24e2Vaishali KulkarniMultiple indices are used for L2 to differentiate between RX / TX and different class of service operations. 
54714b24e2Vaishali Kulkarni
54814b24e2Vaishali Kulkarni\subsection{Initialization}
54914b24e2Vaishali Kulkarni\label{ssec:sb-init}
55014b24e2Vaishali KulkarniThere is a dedicated status block for ecore usage which is allocated and maintained by ecore.
55114b24e2Vaishali KulkarniThe fastpath status blocks used for traffic need to be allocated by the protocol driver.
55214b24e2Vaishali KulkarniThis memory must be DMA-coherent memory.
55314b24e2Vaishali KulkarniThe ecore defines a structure called \texttt{ecore\_sb\_info} which should be allocated by the protocol driver and initialized using the function \myfunc{int\_sb\_init}{int_sb_init} 
55414b24e2Vaishali Kulkarni%[code snippet \ref{snippets/ecore_int_sb_init.h}].
55514b24e2Vaishali KulkarniThis structure is later used for calling the functions \texttt{ecore\_sb\_update\_sb\_idx()} and \texttt{ecore\_sb\_ack()}.
55614b24e2Vaishali Kulkarni
55714b24e2Vaishali Kulkarni%\insertcode{snippets/ecore_int_sb_init.h}{Initialize status blocks}
55814b24e2Vaishali Kulkarni
55914b24e2Vaishali Kulkarni\begin{NOTICE}
56014b24e2Vaishali Kulkarni	Status blocks need to be allocated and initialized before queues are created.
56114b24e2Vaishali Kulkarni\end{NOTICE}
56214b24e2Vaishali Kulkarni
56314b24e2Vaishali Kulkarni\section{Mode and configuration}
56414b24e2Vaishali KulkarniThe device can work in one of the following interrupt modes:
56514b24e2Vaishali Kulkarni\begin{enumerate}
56614b24e2Vaishali Kulkarni	\item INTA ��� Physical interrupt line.
56714b24e2Vaishali Kulkarni	\item MSI ���  Message signaled interrupts. Device is programmed with one address to write to, and 16-bit data to identify the interrupt.
56814b24e2Vaishali Kulkarni	\item MSIX ��� Large number of interrupts (up to 2048) and each one gets a separate target address, making it possible to designate different interrupts to different processors.
56914b24e2Vaishali Kulkarni	This is the preferred mode for performance.
57014b24e2Vaishali Kulkarni	\item POLL ��� HW increments producers on status blocks in case of interrupts but it doesn't generate any message nor does it assert any physical line. It's the upper-layer responsibility to periodically poll on those changes to identify interrupts. \\
57114b24e2Vaishali Kulkarni\end{enumerate}
57214b24e2Vaishali Kulkarni
57314b24e2Vaishali KulkarniEnabling and disabling interrupts is OS specific and done differently by the OS specific layer of the driver.
57414b24e2Vaishali KulkarniHowever, the device needs to be configured differently according to the selected interrupt mode; This initialization is done by the ecore.
57514b24e2Vaishali Kulkarni
57614b24e2Vaishali KulkarniIn order to so, the proper interrupt mode using an \myindex{ecore\_int\_mode} enum [can be seen in code snippet [\ref{snippets/ecore_int_mode.h}]] needs to be passed when calling \texttt{ecore\_hw\_init}.
57714b24e2Vaishali Kulkarni
57814b24e2Vaishali Kulkarni\insertcode{snippets/ecore_int_mode.h}{Enum for the interrupt mode}
57914b24e2Vaishali Kulkarni
58014b24e2Vaishali KulkarniIf upper-layer driver would later wish to change the interrupt mode, it can do so by calling \myfunc{int\_igu\_enable\_int}{int_igu_enable_int},
58114b24e2Vaishali Kulkarnior to \myfunc{int\_igu\_disable\_int}{int_igu_disable_int} when wishing to disable interrupt generation altogether.
58214b24e2Vaishali Kulkarni
58314b24e2Vaishali Kulkarni%\insertcode{snippets/ecore_int_endis.h}{Functions for enabling/disabling interrupts}
58414b24e2Vaishali Kulkarni
58514b24e2Vaishali KulkarniIn MSIX mode, each status block should generate it's own interrupt message, meaning in reasonable OSes it should be possible to connect each interrupt with the specific handler of that interrupt's source.
58614b24e2Vaishali KulkarniThe \textit{sb\_id} passed as value to \textit{ecore\_int\_sb\_init()} will indicate the index of the vector in the MSI-X table that would be used to generate interrupts for this specific SB.
58714b24e2Vaishali KulkarniI.e., if the value passed is $X$, then the $X^{th}$ MSI-X vector will generate interrupts for this SB.
58814b24e2Vaishali Kulkarni
58914b24e2Vaishali KulkarniWhen working in INTA / MSI we work in single-ISR multiple-DPC mode; The same interrupt line can signify interrupts from many possible status blocks. In this case the information of which status block generated an interrupt needs to be read from a register in the IGU. Use \myfunc{int\_igu\_read\_sisr\_reg}{int_igu_read_sisr_reg} to get the information [returned value is a bitmask of status blocks which asserted the interrupt].
59014b24e2Vaishali Kulkarni
59114b24e2Vaishali Kulkarni%\insertcode{snippets/ecore_int_sisr.h}{INTA mechanism for reading interrupt source}
59214b24e2Vaishali Kulkarni
59314b24e2Vaishali Kulkarni\section{IGU block operation}
59414b24e2Vaishali KulkarniThe IGU block has a mapping of status blocks to interrupts.
59514b24e2Vaishali KulkarniThe mapping is done inside the IGU CAM and maps a (function, vector) pair to an MSI-X message.
59614b24e2Vaishali KulkarniIn case of INTA / MSI, each function has a register in the IGU stating which status block gave the interrupt.
59714b24e2Vaishali KulkarniThe IGU block is responsible for generating the interrupt. It receives the command to generate an interrupt from the CAU block.
59814b24e2Vaishali KulkarniThe IGU block maintains producer-consumer pairs per status block.
59914b24e2Vaishali KulkarniThe CAU updates the producer after it wrote the status block to host memory.
60014b24e2Vaishali KulkarniThe driver updates the consumer after it finished processing the status block.
60114b24e2Vaishali KulkarniThe IGU block generates an interrupt when there is a prod-cons difference on the status block.
60214b24e2Vaishali Kulkarni
60314b24e2Vaishali KulkarniCAU also handles coalescing of status block writes and interrupt generation.
60414b24e2Vaishali KulkarniThe CAU unit may aggregate several events and generate a single update of the status block and a single interrupt, in order to lower the number of interrupts sent to host CPU.
60514b24e2Vaishali Kulkarni
60614b24e2Vaishali Kulkarni\section{Interrupt handling flow}
60714b24e2Vaishali Kulkarni\label{sec:sb-flow}
60814b24e2Vaishali KulkarniThe flow of handling an interrupt in the device and driver is as follows:
60914b24e2Vaishali Kulkarni\silentfunc{sb_update_sb_idx}
61014b24e2Vaishali Kulkarni\silentfunc{sb_ack}
61114b24e2Vaishali Kulkarni\begin{enumerate}
61214b24e2Vaishali Kulkarni	\item The device (Firmware/CAU) updates a status block index.
61314b24e2Vaishali Kulkarni	
61414b24e2Vaishali Kulkarni	\item The device copies the status block to host memory and generates an interrupt.
61514b24e2Vaishali Kulkarni	
61614b24e2Vaishali Kulkarni	\item OS is triggered, calling the driver's Interrupt Service Routine [ISR].
61714b24e2Vaishali Kulkarni	
61814b24e2Vaishali Kulkarni	\item (Possible upper-half handling and bottom-half scheduling, or other OS-specifics which are outside the scope of this document).
61914b24e2Vaishali Kulkarni	
62014b24e2Vaishali Kulkarni	\item Driver identifies a producer update on the status block (as the producer is written as part of the status block on host memory) using \myfunc{sb\_update\_sb\_idx}{sb_update_sb_idx}.
62114b24e2Vaishali Kulkarni	
62214b24e2Vaishali Kulkarni	\item Driver scans the protocol indices in the status block to determine the interrupt source.
62314b24e2Vaishali Kulkarni	\begin{NOTICE}
62414b24e2Vaishali Kulkarni		It's likely the upper-layer doesn't really need to scan the status block, but rather compare values in some previous-supplied addresses against a shadow copy. E.g., In L2 the ecore callbacks configuring the queues will return the addresses which upper-layer should test for producer updates. See section [\ref{sec:l2-start}].
62514b24e2Vaishali Kulkarni	\end{NOTICE}
62614b24e2Vaishali Kulkarni
62714b24e2Vaishali Kulkarni	\item When Driver completes processing all the indices on the status block, it writes the producer value from the status block into the IGU consumer address, using \myfunc{sb\_ack}{sb_ack}.
62814b24e2Vaishali Kulkarni	
62914b24e2Vaishali Kulkarni	\item The IGU compares the producer and consumer -- if they differ it will generate an additional interrupt.
63014b24e2Vaishali Kulkarni	
63114b24e2Vaishali Kulkarni\end{enumerate}
63214b24e2Vaishali Kulkarni
63314b24e2Vaishali Kulkarni\begin{exampleT}
63414b24e2Vaishali Kulkarni	Assume an Rx packet is received by device. After FW places the packet in the Rx rings, it updates the status block of that Rx ring; This in turn is copied into host memory and an MSI-X interrupt for the appropriate Rx queue's status block is triggered.
63514b24e2Vaishali Kulkarni	Driver reads the status blocks, scanning the indicies and identifies the interrupt is an Rx CQE consumer and handles the incoming packet. Assuming this is the only interrupt source [and there was also a single packet] driver than acks the status block.
63614b24e2Vaishali Kulkarni\end{exampleT}
63714b24e2Vaishali Kulkarni\SpillChapterFuncs
63814b24e2Vaishali Kulkarni
63914b24e2Vaishali Kulkarni
64014b24e2Vaishali Kulkarni\chapterimage{qlogic-full-36}
64114b24e2Vaishali Kulkarni\chapter{Management firmware [MFW] interface}
64214b24e2Vaishali Kulkarni\label{cha:mfw}
64314b24e2Vaishali Kulkarni
64414b24e2Vaishali KulkarniThe management firmware runs on its own processor on the chip [\myindex{MCP}] and has many responsibilities ��� it serves as the entity initially configuring the chip [during bios phase], answering the various management protocols, synchronizing between PFs, configuring the physical link, etc.
64514b24e2Vaishali KulkarniHW functions and the \myindex{MFW} may interact with each other in both ways ��� driver may send messages to the MFW in the form of commands on a buffer, while the MFW generates attentions for the driver and posts messages in a designated mailbox in the SHMEM. The implementation of the interface resides in \texttt{ecore\_mcp.[ch]}, with the addition of \texttt{.h} files generated by the MFW owners, e.g., \texttt{mcp\_public.h} which contains the SHMEM structure and the list of commands.
64614b24e2Vaishali KulkarniThe API that should be included by upper-layer driver is defined in \texttt{ecore\_mcp\_api.h}.
64714b24e2Vaishali Kulkarni
64814b24e2Vaishali KulkarniThe interface between driver and MFW is initialized as early as possible in the initial initialization flow [specifically as part of \texttt{ecore\_hw\_prepare()}],  as this initializes the Driver access to SHMEM which is used later during initialization to learn about the chip configuration [which was read from NVRAM by MFW and written into SHMEM].
64914b24e2Vaishali KulkarniThe upper layer doesn���t need to take care of allocating/releasing of this interface ��� it���s part of the greater initialization/de-initialization of the ecore.
65014b24e2Vaishali Kulkarni
65114b24e2Vaishali Kulkarni\section{Shared Memory [SHMEM]}
65214b24e2Vaishali KulkarniThe \myindex{shared memory} is a segment of memory accessible to all functions as well as the MFW. The memory is used for various purposes:
65314b24e2Vaishali Kulkarni\begin{enumerate}
65414b24e2Vaishali Kulkarni	\item MFW fills it with current HW configuration, either based on the default found in the NVRAM or based on some management-protocol [e.g., it���s possible vlans configuration is determined by switch and communicated to the MFW]. Driver reads those values and decides upon its logical state/configures HW appropriately. \\
65514b24e2Vaishali Kulkarni	
65614b24e2Vaishali Kulkarni	\item The driver--MFW interface is based on mailboxes in well-known addresses in the SHMEM. \\
65714b24e2Vaishali Kulkarni	
65814b24e2Vaishali Kulkarni	\item It���s possible [as in E3] that there will be driver-held information that will be requested by some management-protocol, and the driver will have to fill it in some well-known address in the SHMEM.
65914b24e2Vaishali Kulkarni\end{enumerate}
66014b24e2Vaishali Kulkarni
66114b24e2Vaishali KulkarniAn upper-layer driver is not supposed to access the SHMEM directly; It should only do so by using ecore functions and accessing ecore structs. The ecore \textit{mcp\_info} struct contains as one of its fields \textit{func\_info} which is filled by the ecore during early device initialization with all the function-specific static\footnote{i.e., data that shouldn't change while driver is running} data. Upper-layer driver can read those values for its own usage.
66214b24e2Vaishali Kulkarni
66314b24e2Vaishali Kulkarni\section{Ecore - MFW interface}
66414b24e2Vaishali Kulkarni\begin{itemize}
66514b24e2Vaishali Kulkarni	\item Sending messages from driver to MFW -- Each HW-function has an address in the SHMEM in which the MFW will poll for messages from that HW-function.
66614b24e2Vaishali Kulkarni	A message is a u32 consisting of a command bit-mask which indicates of the message the HW-functions sends and a cyclic sequential number.
66714b24e2Vaishali Kulkarni	In addition there���s another u32 field which might contain additional parameters [command-specific].
66814b24e2Vaishali Kulkarni	The driver increases the sequence number and writes the message and then polls until the MFW writes its response [with the correct sequence number] to another known address in SHMEM\footnote{Obviously, this is a one-pending mechanism.}
66914b24e2Vaishali Kulkarni	The MFW can also send an additional parameter [command-specific]. \\
67014b24e2Vaishali Kulkarni	
67114b24e2Vaishali Kulkarni	\item Messages from MFW to driver -- MFW will trigger a general HW attention which will be handled by the specific HW-function [there���s a different general HW attention per HW-function].
67214b24e2Vaishali Kulkarni	Per-HW-function there���s an array of message producers in SHMEM,  of which the ecore maintains a copy.
67314b24e2Vaishali Kulkarni	Before sending the attention, the MFW will increment the producer of the message it wishes to inform the driver and the driver will recognize the message by noticing the difference in producers.
67414b24e2Vaishali Kulkarni	After handling said message, the driver will ack the message by writing the new producer back to SHMEM and disabling the general HW attention.
67514b24e2Vaishali Kulkarni	Notice it's [at least theoretically] possible for the ecore to encounter multiple MFW messages following a single attention from HW. \\
67614b24e2Vaishali Kulkarni\end{itemize}
67714b24e2Vaishali Kulkarni
67814b24e2Vaishali KulkarniNotice the commands��� content vary -- some of the commands will require additional parameters to be filled in specific fields in the SHMEM before the commands are passed.
67914b24e2Vaishali Kulkarni
68014b24e2Vaishali Kulkarni\section{API between ecore's MCP interface and upper-layer driver}
68114b24e2Vaishali Kulkarni\myfunc{mcp\_cmd}{mcp_cmd} --  this is the very core of message-passing from driver to MFW. Upper-layer driver should pass the command (FW\_MSG\_CODE\_* from \texttt{mcp\_public.h}) and a parameter, as well as pointers for the MFW response and additional possible parameter. The function will pass the command for MFW and await [sleep] for its reply. \\
68214b24e2Vaishali Kulkarni
68314b24e2Vaishali KulkarniA ���special��� instance of this function is \texttt{ecore\_mcp\_load\_req()} [which isn���t an API function] - that function sends an indication to the MCP that the HW-function is being loaded.
68414b24e2Vaishali KulkarniThe MFW is used as both a book-keeper and synchronization mechanism for the loading of PFs, as there are communal resources. The response will be (FW\_MSG\_CODE\_DRV\_LOAD\_<X>), where X can be either ENGINE, PORT or FUNCTION:
68514b24e2Vaishali Kulkarni\begin{itemize}
68614b24e2Vaishali Kulkarni	\item Engine ��� HW-function is the first being loaded on its engine.
68714b24e2Vaishali Kulkarni	\item Port ��� Another HW-function has already initialized the engine, but this HW-function is first on its port.
68814b24e2Vaishali Kulkarni	\item Function ��� Another HW-function has already initialized the port.
68914b24e2Vaishali Kulkarni\end{itemize}
69014b24e2Vaishali KulkarniAccording to the MFW response the ecore knows what need to be initialized. \\
69114b24e2Vaishali Kulkarni
69214b24e2Vaishali Kulkarni\texttt{ecore\_handle\_mcp\_events()} ��� This function is called from the slowpath interrupt context [sleepless] upon MFW attention to the driver.
69314b24e2Vaishali KulkarniDependent on the exact message received from the MFW, it���s possible that this will eventually will call some OSAL which will need to be implemented by the upper-layer driver, e.g., in case of link change indication [The upper-layer needs to be notified and should decide on its own what to do with that information].
69414b24e2Vaishali Kulkarni
69514b24e2Vaishali Kulkarni\section{Link Interface}
69614b24e2Vaishali Kulkarni\label{sec:mfw-link}
69714b24e2Vaishali KulkarniThe MFW is responsible for configuring the physical link [i.e., MAC, PHY, etc.]. The ecore encapsulates the entire interface with MFW for configuring the link, leaving a relatively narrow API with the upper-layer driver.
69814b24e2Vaishali KulkarniThe ecore HW-function contains 2 related strctures ���
69914b24e2Vaishali Kulkarni\silentfunc{mcp_get_link_params}
70014b24e2Vaishali Kulkarni\silentfunc{mcp_get_link_state}
70114b24e2Vaishali Kulkarni\begin{itemize}
70214b24e2Vaishali Kulkarni	\item Link\_params ��� The ecore uses this as inputs for configuring the link; According to the values in this struct, the ecore will later configure shmem in the appropriate places so that once the MFW receives the command to set the link it will use this configuratio.
70314b24e2Vaishali Kulkarni	During ecore initialization, the ecore will fill this structure with the default values from SHMEM [values set by MFW according to NVRAM configuration]
70414b24e2Vaishali KulkarniWhen upper-layer driver wishes to update link configuration, it should change this struct.
70514b24e2Vaishali KulkarniIt can access it by calling \myfunc{mcp\_get\_link\_params}{mcp_get_link_params} \\
70614b24e2Vaishali Kulkarni
70714b24e2Vaishali Kulkarni	\item Link\_output ��� The ecore fills the structure from attention handling context whenever the MFW indicates that a link change has occurred. Upper layer driver can read this to get information about the current state of the physical link. It can access this struct by calling \myfunc{mcp\_get\_link\_state}{mcp_get_link_state}.\\
70814b24e2Vaishali Kulkarni\end{itemize}
70914b24e2Vaishali Kulkarni
71014b24e2Vaishali KulkarniIn order to work with the ecore link interface, upper driver needs to implement an OSAL [\texttt{osal\_link\_update()}] which will be called whenever the link state has changed ��� this will notify the upper driver that the link has changed and that it should probably read link\_output and act upon it. \\
71114b24e2Vaishali Kulkarni
71214b24e2Vaishali KulkarniIn order to set/reset the link, the upper driver should call \myfunc{mcp\_set\_link}{mcp_set_link} after overriding the link\_params fields with its required link configured [optional, as without doing anything the structure will contain the default link configuration found in SHMEM].
71314b24e2Vaishali KulkarniPassing true will cause MFW to try setting the link [either by force or via auto-negotiation, based on the configuration], while passing false will cause the MFW to reset the link.
71414b24e2Vaishali Kulkarni
71514b24e2Vaishali KulkarniNotice the logic for link-flap-avoidance should be contained in MFW, e.g., in multi-function mode there���s no need for the upper-layer driver to count the number of functions loaded in order to decide whether during unload it should request a link reset; It should do it regardless.
71614b24e2Vaishali KulkarniIt���s the MFW's duty to decide whether the unloading function is actually the last loaded function on its port and thus whether to actually reset the link.
71714b24e2Vaishali Kulkarni
71814b24e2Vaishali Kulkarni\subsection{Energy Efficient Ethernet (EEE)}
71914b24e2Vaishali KulkarniEEE feature enables the device to put its transistors in sleep mode when there is no data activity on the wire. Hence achieves the significant reduction in the power consumption of the device. It's a Base-T feature, more details of which are captured under IEEE 802.3az standard. MFW negotiates the EEE parameters with the peer device and the results will be shared to the ecore as part of link notification. Following are the negotiated parameters which will be encapsulated in the struct \texttt{ecore\_mcp\_link\_state}.
72014b24e2Vaishali Kulkarni\begin{itemize}
72114b24e2Vaishali Kulkarni	\item eee\_active ��� EEE is negotiated and is currently operational.
72214b24e2Vaishali Kulkarni	\item eee\_adv\_caps ��� Device advertized capabilities.
72314b24e2Vaishali Kulkarni	\item eee\_lpi\_adv\_caps ��� Peer device advertized capabilities.
72414b24e2Vaishali Kulkarni\end{itemize}
72514b24e2Vaishali KulkarniFollowing are the EEE link parameters which can be queried by upper layer driver using \myfunc{mcp\_get\_link\_params}{mcp_get_link_params} API.
72614b24e2Vaishali Kulkarni\begin{itemize}
72714b24e2Vaishali Kulkarni	\item eee\_enable ��� EEE is enabled.
72814b24e2Vaishali Kulkarni	\item eee\_supported ��� Device supports EEE.
72914b24e2Vaishali Kulkarni	\item eee\_tx\_lpi\_enable ��� Determines whether the device should assert its Tx LPI.
73014b24e2Vaishali Kulkarni	\item eee\_tx\_lpi\_timer ��� EEE delay timer value, i.e., amount of time device should stay in idle mode prior to asserting its Tx LPI  (in  microseconds).
73114b24e2Vaishali Kulkarni\end{itemize}
73214b24e2Vaishali KulkarniUpper layer driver can configure the one or more of the EEE following parameters.
73314b24e2Vaishali Kulkarni\begin{itemize}
73414b24e2Vaishali Kulkarni	\item eee\_enable
73514b24e2Vaishali Kulkarni	\item eee\_adv\_caps
73614b24e2Vaishali Kulkarni	\item eee\_tx\_lpi\_enable
73714b24e2Vaishali Kulkarni	\item eee\_tx\_lpi\_timer
73814b24e2Vaishali Kulkarni\end{itemize}
73914b24e2Vaishali Kulkarni
74014b24e2Vaishali Kulkarni\section{Dcbx Interface}
74114b24e2Vaishali Kulkarni\label{sec:mfw-dcbx}
74214b24e2Vaishali KulkarniThe MFW is responsible for negotiating the dcbx parameters [e.g., per priority flow control (PFC)] with peer device. During initialization, MFW reads the dcbx parameters from NVRAM (called local parameters) and negotiates these with the peer. The negotiated/agreed parameters are called operational dcbx parameters. MFW provides driver interfaces for querying and configuring the dcbx parameters. The ecore dcbx implementation provides three APIs, one for querying the dcbx paramters and the other two for updating the dcbx configuration.
74314b24e2Vaishali Kulkarni\silentfunc{dcbx_query_params}
74414b24e2Vaishali Kulkarni\silentfunc{dcbx_get_config_params}
74514b24e2Vaishali Kulkarni\silentfunc{dcbx_config_params}
74614b24e2Vaishali Kulkarni\begin{itemize}
74714b24e2Vaishali Kulkarni	\item \myfunc{dcbx\_query\_params}{dcbx\_query\_params} ��� The API returns the current dcbx configuration. It expects type (i.e., local/remote/operational) and the buffer for storing the dcbx parameters of that type.\\
74814b24e2Vaishali Kulkarni
74914b24e2Vaishali Kulkarni	\item \myfunc{dcbx\_get\_config\_params}{dcbx\_get\_config\_params} - The API returns the currently cached dcbx parameter set that can be modified for making the dcbx update requests. \\
75014b24e2Vaishali Kulkarni
75114b24e2Vaishali Kulkarni	\item \myfunc{dcbx\_config\_params}{dcbx\_config\_params} ��� The API is used for sending the dcbx parameters update request. The API expects dcbx parameters to be configured and the flag specifying whether the parameters need to be sent to hardware or just cache at the ecore. When driver sends dcbx config to the hardware, device initiates the dcbx negotiation with the peer using lldp protocol. The negotiation takes few seconds to complete, and also the lldp requests are rate limited (using a predefined credit value). The dcbx API option ���hw\_commit��� specifies whether the dcbx parameters need to be committed to the hardware or just cache at the driver. When client requests the commit, all the cached parameters are sent to the device and the parameter negotiation will be initiated with the peer. \\
75214b24e2Vaishali Kulkarni\end{itemize}
75314b24e2Vaishali KulkarniThe steps for configuring the dcbx parameters are, upper layer driver invokes ecore\_dcbx\_get\_config\_params() API to get the current config parameter set, and update the required parameters, and then invoke ecore\_dcbx\_config\_params() API.
75414b24e2Vaishali Kulkarni
75514b24e2Vaishali KulkarniIf there is any change in the dcbx configuration at the host (for example due to a negotiation with the peer), then MFW notifies the same to ecore. OSAL\_DCBX\_AEN() would be called after such notification, ecore client would need to provide the implementation for this OSAL.
75614b24e2Vaishali Kulkarni
75714b24e2Vaishali Kulkarni\section{Management protocol APIs}
75814b24e2Vaishali Kulkarni\label{sec:mfw-protocols}
75914b24e2Vaishali KulkarniMFW needs various bits of information from the driver, and it gathers those in one of two methods:
76014b24e2Vaishali Kulkarni\begin{itemize}
76114b24e2Vaishali Kulkarni	\item Pulling ��� if ecore can���t provide information on its own, ecore-client would be required to implement an OSAL.\\
76214b24e2Vaishali Kulkarni	\item Pushing ��� it���s the ecore and ecore-client���s responsibility to push the data.\\
76314b24e2Vaishali Kulkarni\end{itemize}
76414b24e2Vaishali KulkarniIn some cases, ���Push��� is done without involvement of the ecore-client. If that���s not possible, it becomes more risky as the responsibility of doing things correctly passes to the ecore-client. Ecore-client shouldn���t presume to do ���push��� only for calls which match the configured management mode. Instead it should always do them and let the ecore be the arbiter of whether those are needed by MFW or not. Ecore provides the following APIs for updating the configuration attributes, it is the client's responsibility to invoke these APIs at the appropriate time.
76514b24e2Vaishali Kulkarni\silentfunc{mcp_ov_update_current_config}
76614b24e2Vaishali Kulkarni\silentfunc{mcp_ov_update_mtu}
76714b24e2Vaishali Kulkarni\silentfunc{mcp_ov_update_mac}
76814b24e2Vaishali Kulkarni\silentfunc{mcp_ov_update_wol}
76914b24e2Vaishali Kulkarni\silentfunc{mcp_ov_update_driver_state}
77014b24e2Vaishali Kulkarni\silentfunc{mcp_update_fcoe_cvid}
77114b24e2Vaishali Kulkarni\silentfunc{mcp_update_fcoe_fabric_name}
77214b24e2Vaishali Kulkarni\begin{itemize}
77314b24e2Vaishali Kulkarni	\item \myfunc{mcp\_ov\_update\_current\_config}{mcp\_ov\_update\_current\_config} ��� Drivers need to call this API when user updates one (or more) of the following: mtu, primary mac or Wake on LAN settings (to a non-default value). In addition, it also needs to call a unique API per each:
77414b24e2Vaishali Kulkarni	\begin{itemize}
77514b24e2Vaishali Kulkarni		\item \myfunc{mcp\_ov\_update\_mtu}{mcp\_ov\_update\_mtu} ��� called when user sets the mtu to a value other than the default provided by the ecore.\\
77614b24e2Vaishali Kulkarni
77714b24e2Vaishali Kulkarni		\item \myfunc{mcp\_ov\_update\_mac}{mcp\_ov\_update\_mac} ��� called when user updates the primary mac address.\\
77814b24e2Vaishali Kulkarni
77914b24e2Vaishali Kulkarni		\item \myfunc{mcp\_ov\_update\_wol}{mcp\_ov\_update\_wol} ��� called when Wake-on-LAN settings are updated.\\
78014b24e2Vaishali Kulkarni	\end{itemize}
78114b24e2Vaishali Kulkarni	\item \myfunc{mcp\_ov\_update\_driver\_state}{mcp\_ov\_update\_driver\_state} ��� notify about a change in the driver state. Following are the possible driver states,
78214b24e2Vaishali Kulkarni	\begin{itemize}
78314b24e2Vaishali Kulkarni		\item ECORE\_OV\_DRIVER\_STATE\_NOT\_LOADED - Firmware is not loaded.\\
78414b24e2Vaishali Kulkarni
78514b24e2Vaishali Kulkarni		\item ECORE\_OV\_DRIVER\_STATE\_DISABLED - Driver is not ready yet.\\
78614b24e2Vaishali Kulkarni
78714b24e2Vaishali Kulkarni		\item ECORE\_OV\_DRIVER\_STATE\_ACTIVE - Driver is operational.\\
78814b24e2Vaishali Kulkarni	\end{itemize}
78914b24e2Vaishali Kulkarni	Ecore sets the following driver states,
79014b24e2Vaishali Kulkarni	\begin{itemize}
79114b24e2Vaishali Kulkarni		\item DISABLED - After firmware is successfully loaded on the device, ecore updates the driver state as DISABLED (as part of ecore\_hw\_init() implementation). \\
79214b24e2Vaishali Kulkarni		\item NOT\_LOADED - Ecore sets this state when the protocol driver is unloaded (as part of ecore\_hw\_remove()).\\
79314b24e2Vaishali Kulkarni	\end{itemize}
79414b24e2Vaishali Kulkarni	It's the protocol driver's responsibility to alternate between the states,
79514b24e2Vaishali Kulkarni	\begin{itemize}
79614b24e2Vaishali Kulkarni		\item ACTIVE - Set when the required initialization is done from the driver side and the device is ready for traffic switching.\\
79714b24e2Vaishali Kulkarni		\item DISABLED - Set when device is not operational (e.g., fastpath queues are released or not configured).\\
79814b24e2Vaishali Kulkarni	\end{itemize}
79914b24e2Vaishali Kulkarni	\item \myfunc{mcp\_update\_fcoe\_cvid}{mcp_update_fcoe_cvid} - Update MFW with the 802.1q fcoe vlan id assigned for the PF.\\
80014b24e2Vaishali Kulkarni	\item \myfunc{mcp\_update\_fcoe\_fabric\_name}{mcp_update_fcoe_fabric_name} - Update fabric name value to the MFW. Fabric name is the value returned by the fabric domain controller in response to a GS-FC ���Get Fabric Name��� command from the adapter.\\
80114b24e2Vaishali Kulkarni\end{itemize}
80214b24e2Vaishali Kulkarni
80314b24e2Vaishali KulkarniEcore also provides the TLV request interface for MFW for querying the driver/device attributes. MFW uses mailbox interface to notify ecore on the required TLV information. Ecore parses the request, populates the required information with the help of ecore clients and sends it to the MFW. Ecore client need to provide necessary infrastructure and the OSALs for implementing this interface. 
80414b24e2Vaishali Kulkarni\begin{itemize}
80514b24e2Vaishali Kulkarni	\item OSAL\_MFW\_TLV\_REQ - The call indicates that ecore has received a TLV request notification from the MFW. The execution context in interrupt mode, hence ecore client need to schedule a thread/bottom-half context to handle this task, and return the control immediately. The bottom-half thread will need to invoke \myfunc{mfw\_process\_tlv\_req}{mfw_process_tlv_req} for further processing of the TLV request.\\
80614b24e2Vaishali Kulkarni	\item OSAL\_MFW\_FILL\_TLV\_DATA - Ecore invokes this callback to get the TLV values of a given type. Ecore client need to fill in the values for all the fields that it's aware of, and also need to set the flags associated with the respective fields. For instance,  if client sets value for 'npiv\_enabled' field, it needs to set the flag 'npiv\_enabled\_set' to true.\\
80714b24e2Vaishali Kulkarni\end{itemize}
80814b24e2Vaishali Kulkarni
80914b24e2Vaishali Kulkarni\SpillChapterFuncs
81014b24e2Vaishali Kulkarni
81114b24e2Vaishali Kulkarni
81214b24e2Vaishali Kulkarni\chapterimage{qlogic-full-36}
81314b24e2Vaishali Kulkarni\chapter{L2 protocol}
81414b24e2Vaishali Kulkarni\label{cha:l2}
81514b24e2Vaishali Kulkarni
81614b24e2Vaishali Kulkarni\section{L2-related terminology}
81714b24e2Vaishali KulkarniThis section describes in a very highlevel manner several FW objects which are related to L2. Developers implementing L2 support over the ecore should be familiar with these\footnote{Probably even more than is in the scope of this document.}.
81814b24e2Vaishali Kulkarni\begin{itemize}
81914b24e2Vaishali Kulkarni	\item Virtual port [\myindex{VPORT}] -- Can simply be seen as a collection of queues, each HW-function will have at least one VPORT configured\footnote{And in most scenarios one will suffice.}. Classifications are configured per-VPORT. \\
82014b24e2Vaishali Kulkarni	
82114b24e2Vaishali Kulkarni	\item Queues -- Either Rx/Tx, queues are attached to a VPORT. There can multiple queues per-VPORT [e.g., if RSS/TSS is supported]. Usually, each Rx queue will use it's own status block for interrupts upon Rx packets but Tx queues can utilize the same status blocks, using different protocol indices. \\
82214b24e2Vaishali Kulkarni\end{itemize}
82314b24e2Vaishali Kulkarni
82414b24e2Vaishali Kulkarni\section{Starting an L2 device}
82514b24e2Vaishali Kulkarni\label{sec:l2-start}
82614b24e2Vaishali KulkarniThis section begins after section \ref{sec:init-init}, I.e., assuming the HW-function has already been initialized by the init tool and the PF\_START ramrod has already been sent.
82714b24e2Vaishali Kulkarni
82814b24e2Vaishali Kulkarni	\begin{NOTICE}
82914b24e2Vaishali Kulkarni	Although VPORTs' and queues' indices are shared between all HW-function on the same engine, the resource allocation scheme determines a range of VPORTs per-HW-function to use for configuration [i.e., developer can assume starting index is always 0 per-HW-function].
83014b24e2Vaishali Kulkarni	\end{NOTICE}
83114b24e2Vaishali Kulkarni
83214b24e2Vaishali Kulkarni
83314b24e2Vaishali Kulkarni\silentfunc{sp_vport_start}
83414b24e2Vaishali Kulkarni\silentfunc{eth_rx_queue_start}
83514b24e2Vaishali Kulkarni\silentfunc{eth_tx_queue_start}
83614b24e2Vaishali Kulkarni\silentfunc{sp_vport_update}
83714b24e2Vaishali Kulkarni\begin{enumerate}
83814b24e2Vaishali Kulkarni	\item \myfunc{sp\_vport\_start}{sp_vport_start} -- this function initializes a vport in FW [ETH\_RAMROD\_VPORT\_START will be sent]. The handle for this function is a \texttt{vport\_id} which is passed and the most 'interesting' argument is the MTU for that VPORT.
83914b24e2Vaishali Kulkarni	This VPORT will be inactive after sending this ramrod, i.e., until enabling it via a vport update it will not actually perform Rx/Tx. \\	
84014b24e2Vaishali Kulkarni
84114b24e2Vaishali Kulkarni	\item \myfunc{eth\_rx\_queue\_start}{eth_rx_queue_start} -- initializes an rx queue on a given VPORT.
84214b24e2Vaishali Kulkarni	A pre-request is that the VPORT has already been initialized.
84314b24e2Vaishali Kulkarni	There are 2 identifier of the queue - the queue index to add and the VPORT index to add it to. The queue-index should be unique for the Rx-queue; No 2 Rx-queues of the same PF should use the same id. 
84414b24e2Vaishali Kulkarni	There are quite a few parameters that need to be supplied, e.g., status block, physical addresses of rings, etc.
84514b24e2Vaishali Kulkarni	
84614b24e2Vaishali Kulkarni	The function is expected to receive a pointer to a \texttt{p\_ret\_params} which it will fill with outputs [upon success]. The ecore would fill the address where producer-updates need to be written [in the storm's RAM]; The upper-driver will write producer updates to that address to replenish its Rx-rings.
84714b24e2Vaishali Kulkarni
84814b24e2Vaishali Kulkarni	\begin{NOTICE}
84914b24e2Vaishali Kulkarni	Address is mapped by GTT, so upper-driver can simply write to that address, using the necessary memory barriers.
85014b24e2Vaishali Kulkarni	\end{NOTICE}
85114b24e2Vaishali Kulkarni	In addition, ecore would also fill a \texttt{p\_handle}. This handle is opaque to the ecore-client, and should be passed to other Rx-queue APIs when doing configuration relating to that queue.
85214b24e2Vaishali Kulkarni	
85314b24e2Vaishali Kulkarni	After calling this function, upper-layer driver should initialize the Rx packets producers. \\
85414b24e2Vaishali Kulkarni
85514b24e2Vaishali Kulkarni	\item \myfunc{eth\_tx\_queue\_start}{eth_tx_queue_start} -- initializes a Tx queue on a given VPORT0.
85614b24e2Vaishali Kulkarni		Very similar to the Rx queue start method, with some slight differences in the parameters [BD ring address instead of Rx rings, etc.]. For Tx-queues, the same queue-id can be shared between 2 different queues. That would cause those queues to share the same coalescing configuration.
85714b24e2Vaishali Kulkarni		Just like for Rx-queues, the ecore would fill the \texttt{p\_ret\_params} with an opaque handler to be used for further calls relating to this queue. In addition, it will provide a \texttt{p\_doorbell} address, which is an address into which a doorbell needs to be written to activate firmware once a packet is placed on this Tx queue and the buffer descriptors are filled. 
85814b24e2Vaishali Kulkarni		\begin{NOTICE}
85914b24e2Vaishali Kulkarni		Doorbell addresses are on a different BAR than that of other memories/registers accessed by driver, and the PTT/GTT scheme does not apply to it; Thus the address can simply be accessed using the necessary memory barriers.
86014b24e2Vaishali Kulkarni		\end{NOTICE}
86114b24e2Vaishali Kulkarni
86214b24e2Vaishali Kulkarni	\item \myfunc{sp\_vport\_update}{sp_vport_update} -- This is required to enable the VPORT. It should be called after the Tx/Rx queues were already added, and this will enable the VPORT to send and receive packets\footnote{Notice that without classification configuration Rx won't actually work. Also notice this function can do a lot of things; Enabling the VPORT is only one of them.}.
86314b24e2Vaishali Kulkarni
86414b24e2Vaishali Kulkarni	In order to enable the VPORT for traffic, the upper-layer driver should set in \texttt{p\_params} the following:
86514b24e2Vaishali Kulkarni	\begin{enumerate}
86614b24e2Vaishali Kulkarni		\item \texttt{update\_vport\_active\_flg} to 1.
86714b24e2Vaishali Kulkarni		\item \texttt{vport\_active\_flg} to 1.
86814b24e2Vaishali Kulkarni	\end{enumerate}
86914b24e2Vaishali Kulkarni
87014b24e2Vaishali Kulkarni\end{enumerate}
87114b24e2Vaishali Kulkarni
87214b24e2Vaishali Kulkarni\section{Configuring Classifications}
87314b24e2Vaishali Kulkarni\label{sec:l2-class}
87414b24e2Vaishali KulkarniClassification configuration consists [mostly] of three thing:
87514b24e2Vaishali Kulkarni\begin{enumerate}
87614b24e2Vaishali Kulkarni	\item Configuration of the \myindex{Rx mode} -- This defines which datagrams [unicast, multicast, broadcast] should be accepted by the VPORT, and whether all such datagrams or only if a filter is configured for them.
87714b24e2Vaishali Kulkarni	\item Configuration of unicast / multicast filters -- defining filters for specific unicast / multicast addresses which should be matched, given that Rx mode  agrees.
87814b24e2Vaishali Kulkarni	\item Configuration of vlan filters -- by default, all vlans will be accepted. If at least one vlan [or vlan-mac] filter will be configured only traffic which matches one of the configured vlan filters will pass through.
87914b24e2Vaishali Kulkarni\end{enumerate}
88014b24e2Vaishali Kulkarni
88114b24e2Vaishali KulkarniThere are several ecore functions which are responsible for configuring classifications:
88214b24e2Vaishali Kulkarni
88314b24e2Vaishali Kulkarni\silentfunc{filter_accept_cmd}
88414b24e2Vaishali Kulkarni\silentfunc{sp_eth_filter_ucast}
88514b24e2Vaishali Kulkarni\silentfunc{filter_mcast_cmd}
88614b24e2Vaishali Kulkarni\begin{itemize}
88714b24e2Vaishali Kulkarni	\item \myfunc{filter\_accept\_cmd}{filter_accept_cmd} -- configures the Rx mode of the device.
88814b24e2Vaishali Kulkarni	\item \myfunc{sp\_vport\_update}{sp_vport_update} -- although not exactly a classification function, calling this will re-set the Rx mode [this calls \texttt{ecore\_filter\_accept\_cmd()} as part of its work].
88914b24e2Vaishali Kulkarni	\item \myfunc{sp\_eth\_filter\_ucast}{sp_eth_filter_ucast} -- configures either a unicast filter, vlan filter or a unicast/vlan filter pair.
89014b24e2Vaishali Kulkarni		An important parameter for the upper-layer driver\footnote{in the sense that it might affect design, since all fields are relevant.} is the `opcode' field:
89114b24e2Vaishali Kulkarni		\begin{itemize}
89214b24e2Vaishali Kulkarni			\item ECORE\_FILTER\_ADD -- adds a new filter.
89314b24e2Vaishali Kulkarni			\item ECORE\_FILTER\_REMOVE -- removes a filter.
89414b24e2Vaishali Kulkarni			\item ECORE\_FILTER\_MOVE -- removes a filter from one vport and adds it to another simultaneously\footnote{Needed by windows.}.
89514b24e2Vaishali Kulkarni			\item ECORE\_FILTER\_REPLACE -- adds a new filter after removing all previously configured filters.
89614b24e2Vaishali Kulkarni		\end{itemize}
89714b24e2Vaishali Kulkarni		
89814b24e2Vaishali Kulkarni	\item \myfunc{filter\_mcast\_cmd}{filter_mcast_cmd} -- configures a multicast filter.
89914b24e2Vaishali Kulkarni	\begin{warning}
90014b24e2Vaishali Kulkarni		This function exists in the ecore but at the moment it's not implemented.
90114b24e2Vaishali Kulkarni	\end{warning}
90214b24e2Vaishali Kulkarni
90314b24e2Vaishali Kulkarni\end{itemize}
90414b24e2Vaishali KulkarniThese functions expose the \texttt{ecore\_spq} implementation -- upper-driver layer can choose whether to wait for completion, supply a callback for completion or do-nothing upon completion (the last will usually be the chosen path).
90514b24e2Vaishali Kulkarni
90614b24e2Vaishali Kulkarni\section{Stopping an L2 device}
90714b24e2Vaishali KulkarniThis is pretty straight forward, and works in reverse-order to the initialization of the L2 device.
90814b24e2Vaishali KulkarniAfter upper-layer driver guarantees that no new Tx-packets will be generated and once Tx queues are all empty, it should do the following:
90914b24e2Vaishali Kulkarni\silentfunc{eth_tx_queue_stop}
91014b24e2Vaishali Kulkarni\silentfunc{eth_rx_queue_stop}
91114b24e2Vaishali Kulkarni\silentfunc{sp_vport_stop}
91214b24e2Vaishali Kulkarni\begin{enumerate}
91314b24e2Vaishali Kulkarni	\item Disable the VPORT by calling \texttt{ecore\_vport\_update()} after setting:
91414b24e2Vaishali Kulkarni	\begin{enumerate}
91514b24e2Vaishali Kulkarni		\item \texttt{update\_vport\_active\_flg} to 1.
91614b24e2Vaishali Kulkarni		\item \texttt{vport\_active\_flg} to 0.
91714b24e2Vaishali Kulkarni	\end{enumerate}
91814b24e2Vaishali Kulkarni	
91914b24e2Vaishali Kulkarni	\item Close all Tx queues\footnote{Actually, order does not matter between Tx and Rx queues}  by calling \myfunc{eth\_tx\_queue\_stop}{eth_tx_queue_stop}.
92014b24e2Vaishali Kulkarni
92114b24e2Vaishali Kulkarni	\item Close all Rx queues by \myfunc{eth\_rx\_queue\_stop}{eth_rx_queue_stop}.
92214b24e2Vaishali Kulkarni
92314b24e2Vaishali Kulkarni	\item Close the vport by calling \myfunc{sp\_vport\_stop}{sp_vport_stop}.
92414b24e2Vaishali Kulkarni\end{enumerate}
92514b24e2Vaishali Kulkarni
92614b24e2Vaishali KulkarniFollowing the completion of the \texttt{vport\_stop}, no further traffic should be working. Interrupts can be released, and resources can freed.
92714b24e2Vaishali KulkarniNotice this on its own doesn't return the device into a 'clean-slate' state; There are still several non-L2 things that needs to be done [e.g., cleaning the status blocks of the queues]
92814b24e2Vaishali Kulkarni\SpillChapterFuncs
92914b24e2Vaishali Kulkarni
93014b24e2Vaishali Kulkarni\chapterimage{pictures/qlogic-full-36.jpg}
93114b24e2Vaishali Kulkarni\chapter{100G support}
93214b24e2Vaishali Kulkarni\label{cha:100}
93314b24e2Vaishali Kulkarni
93414b24e2Vaishali KulkarniOur device supports \myindex{100G} link. However, the fastpath pipeline of each HW engine isn't fast enough for that line-rate. The Hardware function term is a catchphrase for the HW resource and identifications normally required by a single pci function. In 100G mode, the device will enumerate as a single pci function\footnote{Or more in multi-function mode; But we will stick with single-function mode for simplicity here.}, but the driver running over this pci function will utilize multiple HW functions.
93514b24e2Vaishali KulkarniFrom pci standpoint, the distinction between the HW functions (and thus the HW engines) is done via the bar address. Access to the first half of each of the pci function's bars will be translated into an access into a HW function on the first engine, while access to the second half will be translated into an access into a HW function on the second engine.
93614b24e2Vaishali KulkarniFrom the wire standpoint, both HW-functions are connected to a single physical port, i.e. transmitting traffic from either HW-function will lead to transmission on the same physical port. Incoming traffic from the port is routed to a hardware engine according to its protocol 4-tuple. The HW block responsible for this routing it the \myindex{OPTE}.
93714b24e2Vaishali Kulkarni
93814b24e2Vaishali KulkarniThis special configuration is also sometimes referred to as \myindex{Couple Mode Teaming} or \myindex{CMT}.
93914b24e2Vaishali Kulkarni
94014b24e2Vaishali KulkarniAfter the early initialization phase of the ecore (i.e., following ecore\_hw\_prepare()), the \textit{ecore\_dev} field \myindex{num\_hwfns} will be filled with the correct number of HW-functions under the PCI device. The ecore and its client should access only the first num\_hwfns entries in the \textit{hwfns} array.
94114b24e2Vaishali Kulkarni
94214b24e2Vaishali Kulkarni\section{Effects on MSI-X interrupts}
94314b24e2Vaishali Kulkarni\label{sec:100int}
94414b24e2Vaishali KulkarniEach path has its own IGU CAM, meaning it has its own set of available status block. But as both HW-functions share the same PCI function, there is a single MSI-X table for that device.
94514b24e2Vaishali KulkarniAs a result, when in CMT the MSI-X vectors are split between the two hw-func/Incomtions.
94614b24e2Vaishali Kulkarni
94714b24e2Vaishali Kulkarni\begin{exampleT}
94814b24e2Vaishali Kulkarni\label{ex:CMT1}
94914b24e2Vaishali KulkarniAssume a PCI function is in CMT mode. Let $\text{hwfn}_0$ stand for its HW-function under the first engine and $\text{hwfn}_1$ stand for its HW-function under the second engine.
95014b24e2Vaishali KulkarniLet $\text{MSIX}_i$ stand for the $i^{th}$ entry in the PCI function's MSI-X table.
95114b24e2Vaishali Kulkarni
95214b24e2Vaishali KulkarniThen for $\forall n \in \mathbb{N}_{+}$, $\text{MSIX}_{2n}$ is connected to $\text{hwfn}_0$'s status block of index $n$, and $\text{MSIX}_{2n+1}$ is connected to $hwfn_1$'s status block of index $n$.
95314b24e2Vaishali Kulkarni\end{exampleT}
95414b24e2Vaishali Kulkarni
95514b24e2Vaishali Kulkarni\section{Effects on device slowpath configuration}
95614b24e2Vaishali KulkarniEcore handles almost all the difference between CMT and regular mode on it's own, i.e., it reads the number of HW-functions under the devices and iterates when needed to configure both engines correctly (where as in the non-CMT mode it would have simply configured one).
95714b24e2Vaishali KulkarniWhat it does require is:
95814b24e2Vaishali Kulkarni\begin{itemize}
95914b24e2Vaishali Kulkarni	\item Implement OSAL\_BAR\_SIZE. Ecore uses it to determine where it needs to split the bars; Without it it's very likely things will fail very early during initialization.
96014b24e2Vaishali Kulkarni	
96114b24e2Vaishali Kulkarni	\item Set the HW-function's pf\_params for each HW-function before calling \textit{ecore\_resc\_alloc}.
96214b24e2Vaishali Kulkarni	
96314b24e2Vaishali Kulkarni	\item Enable slowpath interrupts -- the first 2 MSI-X vectors should be used for slowpath. Notice that the ecore itself will call OSAL\_DPC\_INIT for each HW-function.
96414b24e2Vaishali Kulkarni	\begin {exampleT}
96514b24e2Vaishali Kulkarni		following Example [\ref{ex:CMT1}], $\text{MSIX}_0$ should be enabled and connected to the DPC of $\text{hwfn}_0$ and $\text{MSIX}_1$ should be enabled and connected to the DPC of $\text{hwfn}_1$.
96614b24e2Vaishali Kulkarni	\end{exampleT}
96714b24e2Vaishali Kulkarni\end{itemize}
96814b24e2Vaishali Kulkarni
96914b24e2Vaishali KulkarniWhen disabling the slowpath, it's important to remember that there were 2 different DPCs allocated and 2 MSI-X vectors configured to support them, as it's the ecore-client responsibility for disabling the interrupts.
97014b24e2Vaishali Kulkarni
97114b24e2Vaishali Kulkarni\section{Effects on L2 fastpath configuration}
97214b24e2Vaishali KulkarniSince each HW-function is running on a different path and is an independent entity (as perceived by FW/HW), configuration should be almost symmetric for both HW-functions. E.g., Following the flow of section \ref{sec:l2-start}, ecore\_sp\_vport\_start() should be called separately for each HW-function, queues should be opened separately for each, etc..
97314b24e2Vaishali Kulkarni
97414b24e2Vaishali KulkarniNotice that in most cases you can even use the same indices, since FW-indices are per-path. E.g., you can use $\text{vport}_0$ on both HW-functions, since they are different on each path.
97514b24e2Vaishali Kulkarni
97614b24e2Vaishali Kulkarni\begin{warning}
97714b24e2Vaishali Kulkarni	When allocating the status blocks for your queues, do recall that the MSI-X table is shared between the engines, as explained in section [\ref{sec:100int}].
97814b24e2Vaishali Kulkarni\end{warning}
97914b24e2Vaishali Kulkarni
98014b24e2Vaishali Kulkarni\begin{NOTICE}
98114b24e2Vaishali Kulkarni	There is an issue between the user control of the number of queues and the actual configuration of queues - e.g., assume user wants $X$ queues. If we use a symmetric configuration what we actually do is open $X$ queues on each path, meaning we actually open $2X$ queues.
98214b24e2Vaishali Kulkarni
98314b24e2Vaishali Kulkarni	We can either only open $X/2$ queues on each engine, in which case we lose some abilities, e.g., control the keys of the RSS hash-function, or open $2X$ queues and try to hide this fact from user, but this most likely will either incur a performance penalty, hard-to-maintain code or both.
98414b24e2Vaishali Kulkarni\end{NOTICE}
98514b24e2Vaishali Kulkarni
98614b24e2Vaishali Kulkarni\chapterimage{qlogic-full-36}
98714b24e2Vaishali Kulkarni\chapter{iSCSI protocol}
98814b24e2Vaishali Kulkarni\label{cha:iscsi}
98914b24e2Vaishali Kulkarni
99014b24e2Vaishali KulkarniThis chapter describes the ecore interface for the upper-layer driver of the iSCSI protocol.
99114b24e2Vaishali Kulkarni
99214b24e2Vaishali Kulkarni\section{Start iSCSI PF}
99314b24e2Vaishali Kulkarni\silentfunc{sp_iscsi_func_start}
99414b24e2Vaishali Kulkarni\silentfunc{iscsi_get_global_cmdq_cons}
99514b24e2Vaishali Kulkarni\begin{itemize}
99614b24e2Vaishali Kulkarni	\item The basic initialization process is described in section \ref{sec:init-init} for all protocols. \\
99714b24e2Vaishali Kulkarni	Specifically for iSCSI, before calling \texttt{ecore\_resc\_alloc()}, the upper driver should determine the PF-global parameters, allocate all PF-global queues, and fill the \texttt{iscsi\_pf\_params} part in struct \texttt{ecore\_pf\_params}. \\
99814b24e2Vaishali Kulkarni	The following table describes the parameters that should be filled (the rest should be zero):
99914b24e2Vaishali Kulkarni	\begin{center}
100014b24e2Vaishali Kulkarni		\begin{tabular}{| l | p{10cm} |}
100114b24e2Vaishali Kulkarni		\hline
100214b24e2Vaishali Kulkarni		\textbf{Parameter} & \textbf{Description} \\ \hline
100314b24e2Vaishali Kulkarni		\texttt{num\_cons} & Up to 4K are supported, suggested default value 128 \\ \hline
100414b24e2Vaishali Kulkarni		\texttt{num\_tasks} & Up to 4K are supported, suggested default value 1K \\ \hline
100514b24e2Vaishali Kulkarni		\texttt{half\_way\_close\_timeout} & Timeout from sending FIN until abortive close, suggested default value 10sec \\ \hline
100614b24e2Vaishali Kulkarni		\texttt{num\_sq\_pages\_in\_ring} & Number of outstanding tasks on the connection * 8B / page-size. \newline Suggested default value for number of outstanding tasks on the connection 256 \\ \hline
100714b24e2Vaishali Kulkarni		\texttt{num\_r2tq\_pages\_in\_ring} & Same as \texttt{num\_sq\_pages\_in\_ring} \\ \hline
100814b24e2Vaishali Kulkarni		\texttt{num\_uhq\_pages\_in\_ring} & Number of outstanding un-ACKed PDUs, suggested default value -- same as \texttt{num\_sq\_pages\_in\_ring} \\ \hline
100914b24e2Vaishali Kulkarni		\texttt{num\_queues} & Number of global queues (CQ / CmdQ / RQ). \newline This should be $\leq$ number of available MSIX vectors for the PF \\ \hline
101014b24e2Vaishali Kulkarni		\texttt{log\_page\_size} & 12 for 4KB pages \\ \hline
101114b24e2Vaishali Kulkarni		\texttt{glbl\_q\_params\_addr} & The physical address of the list of pointers to the arrays of pointers to global queues pages. \newline The list is built as follows: CQ\#0 PBL pointer, RQ\#0 PBL pointer, CmdQ\#0 PBL pointer, CQ\#1 PBL pointer, RQ\#1 PBL pointer, CmdQ\#1 PBL pointer, etc. \newline Each PBL pointer points to the physical address which contains an array of pointers to the physical addresses of the specific queue pages. \\ \hline
101214b24e2Vaishali Kulkarni		\texttt{rqe\_log\_size} & 8 for 256B RQE \\ \hline