1*14b24e2bSVaishali Kulkarni%----------------------------------------------------------------------------------------
3*14b24e2bSVaishali Kulkarni%----------------------------------------------------------------------------------------
4*14b24e2bSVaishali Kulkarni
5*14b24e2bSVaishali Kulkarni\documentclass[11pt,fleqn,hidelinks,oneside]{book} % Default font size and left-justified equations
6*14b24e2bSVaishali Kulkarni\usepackage[nottoc,notlot,notlof]{tocbibind}
7*14b24e2bSVaishali Kulkarni\makeindex % Tells LaTeX to create the files required for indexing
8*14b24e2bSVaishali Kulkarni%----------------------------------------------------------------------------------------
9*14b24e2bSVaishali Kulkarni
10*14b24e2bSVaishali Kulkarni% Create a command to cleanly insert a snippet with the style above anywhere in the document
11*14b24e2bSVaishali Kulkarni\newcommand{\insertcode}[2]{\begin{itemize}\item[]\lstinputlisting[caption=#2,label=#1,style=Style1,float=h!]{#1}\end{itemize}} % The first argument is the script location/filename and the second is a caption for the listing
12*14b24e2bSVaishali Kulkarni
13*14b24e2bSVaishali Kulkarni\newcommand{\myref}[1]
14*14b24e2bSVaishali Kulkarni	{\textcolor{blue}{[\ref{#1}]}}
15*14b24e2bSVaishali Kulkarni
16*14b24e2bSVaishali Kulkarni\newcommand{\myindex}[1]
17*14b24e2bSVaishali Kulkarni	{\index{#1@\texttt{#1}}#1}
18*14b24e2bSVaishali Kulkarni
19*14b24e2bSVaishali Kulkarni\newcommand{\ChapterFuncs}{}
20*14b24e2bSVaishali Kulkarni
21*14b24e2bSVaishali Kulkarni%Fpr some reason, this doesn't work inside \item so we can't have this as part of \myfunc
22*14b24e2bSVaishali Kulkarni\newcommand{\silentfunc}[1]
23*14b24e2bSVaishali Kulkarni{\expandafter\def\expandafter\ChapterFuncs\expandafter{\ChapterFuncs { } \insertcode{snippets/#1_generated.h}{}}}
24*14b24e2bSVaishali Kulkarni
25*14b24e2bSVaishali Kulkarni\newcommand{\myfunc}[2]
26*14b24e2bSVaishali Kulkarni{\index{ZZZ@API Function!ecore\_#1@\texttt{ecore\_#1}}%
27*14b24e2bSVaishali Kulkarni\silentfunc{#2}\texttt{ecore\_#1()}}
28*14b24e2bSVaishali Kulkarni
29*14b24e2bSVaishali Kulkarni\newenvironment{bottompar}{\par\vspace*{\fill}}{\clearpage}
30*14b24e2bSVaishali Kulkarni
31*14b24e2bSVaishali Kulkarni\newcommand{\SpillChapterFuncs}%
32*14b24e2bSVaishali Kulkarni%{\begin{bottompar}
33*14b24e2bSVaishali Kulkarni{%
34*14b24e2bSVaishali Kulkarni%\texttt{\textbf{\\API functions in this chapter: \\}}%
35*14b24e2bSVaishali Kulkarni\section{API functions discussed in this chapter}
36*14b24e2bSVaishali Kulkarni\ChapterFuncs{}%
37*14b24e2bSVaishali Kulkarni%\end{bottompar}%
38*14b24e2bSVaishali Kulkarni\renewcommand{\ChapterFuncs}{}}
39*14b24e2bSVaishali Kulkarni
40*14b24e2bSVaishali Kulkarni%----------------------------------------------------------------------------------------
41*14b24e2bSVaishali Kulkarni
42*14b24e2bSVaishali Kulkarni\input{structure} % Insert the commands.tex file which contains the majority of the structure behind the template
43*14b24e2bSVaishali Kulkarni
44*14b24e2bSVaishali Kulkarni%\lstset{belowskip=-20pt plus 2pt}
45*14b24e2bSVaishali Kulkarni\lstset{belowskip=\smallskipamount,aboveskip=\smallskipamount,boxpos=h!,float=h!}
46*14b24e2bSVaishali Kulkarni\makeatletter
47*14b24e2bSVaishali Kulkarni\setlength{\@fptop}{5pt}
48*14b24e2bSVaishali Kulkarni\makeatother
49*14b24e2bSVaishali Kulkarni
50*14b24e2bSVaishali Kulkarni
51*14b24e2bSVaishali Kulkarni\usepackage{hyperref}
52*14b24e2bSVaishali Kulkarni\usepackage{verbatim}
53*14b24e2bSVaishali Kulkarni
54*14b24e2bSVaishali Kulkarni%Macros
55*14b24e2bSVaishali Kulkarni\newcommand{\mlist}[1]{\begin{itemize}{#1}\end{itemize}}
56*14b24e2bSVaishali Kulkarni\newcommand{\mlisti}[2]{\item {\textcolor{red}{#1} -- #2}}
57*14b24e2bSVaishali Kulkarni
58*14b24e2bSVaishali Kulkarni\long\def\greybox#1{%
59*14b24e2bSVaishali Kulkarni    \newbox\contentbox%
60*14b24e2bSVaishali Kulkarni    \newbox\bkgdbox%
61*14b24e2bSVaishali Kulkarni    \setbox\contentbox\hbox to \hsize{%
62*14b24e2bSVaishali Kulkarni        \vtop{
63*14b24e2bSVaishali Kulkarni            \kern\columnsep
64*14b24e2bSVaishali Kulkarni            \hbox to \hsize{%
65*14b24e2bSVaishali Kulkarni                \kern\columnsep%
66*14b24e2bSVaishali Kulkarni                \advance\hsize by -2\columnsep%
67*14b24e2bSVaishali Kulkarni                \setlength{\textwidth}{\hsize}%
68*14b24e2bSVaishali Kulkarni                \vbox{
69*14b24e2bSVaishali Kulkarni                    \parskip=\baselineskip
70*14b24e2bSVaishali Kulkarni                    \parindent=0bp
71*14b24e2bSVaishali Kulkarni                    #1
72*14b24e2bSVaishali Kulkarni                }%
73*14b24e2bSVaishali Kulkarni                \kern\columnsep%
74*14b24e2bSVaishali Kulkarni            }%
75*14b24e2bSVaishali Kulkarni            \kern\columnsep%
76*14b24e2bSVaishali Kulkarni        }%
77*14b24e2bSVaishali Kulkarni    }%
78*14b24e2bSVaishali Kulkarni    \setbox\bkgdbox\vbox{
79*14b24e2bSVaishali Kulkarni        \pdfliteral{0.75 0.75 0.75 rg}
80*14b24e2bSVaishali Kulkarni        \hrule width  \wd\contentbox %
81*14b24e2bSVaishali Kulkarni               height \ht\contentbox %
82*14b24e2bSVaishali Kulkarni               depth  \dp\contentbox
83*14b24e2bSVaishali Kulkarni        \pdfliteral{0 0 0 rg}
84*14b24e2bSVaishali Kulkarni    }%
85*14b24e2bSVaishali Kulkarni    \wd\bkgdbox=0bp%
86*14b24e2bSVaishali Kulkarni    \vbox{\hbox to \hsize{\box\bkgdbox\box\contentbox}}%
87*14b24e2bSVaishali Kulkarni    \vskip\baselineskip%
88*14b24e2bSVaishali Kulkarni}
89*14b24e2bSVaishali Kulkarni
90*14b24e2bSVaishali Kulkarni\newcommand{\greycom}[2]{\greybox{\textcolor{red}{#1} -- #2}}
91*14b24e2bSVaishali Kulkarni
92*14b24e2bSVaishali Kulkarni
93*14b24e2bSVaishali Kulkarni\global \mdfdefinestyle{MyMdStyle}{%
94*14b24e2bSVaishali Kulkarni	linecolor=black, linewidth=1,%
95*14b24e2bSVaishali Kulkarni	outerlinecolor=red,outerlinewidth=2pt,%
96*14b24e2bSVaishali Kulkarni	roundcorner=5pt,backgroundcolor=brown!10,nobreak=true}
97*14b24e2bSVaishali Kulkarni
98*14b24e2bSVaishali Kulkarni\newenvironment{warning}
99*14b24e2bSVaishali Kulkarni	{\par\begin{mdframed}[style=MyMdStyle] \begin{Warning}}
100*14b24e2bSVaishali Kulkarni	{\end{Warning}\end{mdframed}\vspace{5pt}\par}
101*14b24e2bSVaishali Kulkarni
102*14b24e2bSVaishali Kulkarni\newcommand{\HRule}{\rule{\linewidth}{0.5mm}}
103*14b24e2bSVaishali Kulkarni\newenvironment{TBD}
104*14b24e2bSVaishali Kulkarni	{\par\vspace{3pt}\begin{mdframed}[style=MyMdStyle,outerlinecolor=blue,%
105*14b24e2bSVaishali Kulkarni									  backgroundcolor=blue!10]%
106*14b24e2bSVaishali Kulkarni		\begin{question}}
107*14b24e2bSVaishali Kulkarni  {\end{question}\end{mdframed}\par}
108*14b24e2bSVaishali Kulkarni
109*14b24e2bSVaishali Kulkarni\newenvironment{NOTICE}
110*14b24e2bSVaishali Kulkarni  {\par\begin{mdframed}[style=MyMdStyle,outerlinecolor=black,%
111*14b24e2bSVaishali Kulkarni  						linecolor=black, outerlinewidth=1.5pt]%
112*14b24e2bSVaishali Kulkarni    \begin{itemize}{}{\leftmargin=1cm
113*14b24e2bSVaishali Kulkarni                   \labelwidth=\leftmargin}\item[\Large\Info]}
114*14b24e2bSVaishali Kulkarni  {\end{itemize}\end{mdframed}\par}
115*14b24e2bSVaishali Kulkarni
116*14b24e2bSVaishali Kulkarni\newenvironment{REMINDER}
117*14b24e2bSVaishali Kulkarni	{\par\begin{mdframed}[style=MyMdStyle,outerlinecolor=blue,%
118*14b24e2bSVaishali Kulkarni  						  linecolor=blue, outerlinewidth=2pt]%
119*14b24e2bSVaishali Kulkarni		\begin{reminder}}
120*14b24e2bSVaishali Kulkarni  {\end{reminder}\end{mdframed}\par}
121*14b24e2bSVaishali Kulkarni
122*14b24e2bSVaishali Kulkarni\bibliographystyle{plain}
123*14b24e2bSVaishali Kulkarni
124*14b24e2bSVaishali Kulkarni\begin{document}
125*14b24e2bSVaishali Kulkarni
126*14b24e2bSVaishali Kulkarni\begin{titlepage}
127*14b24e2bSVaishali Kulkarni\begin{center}
128*14b24e2bSVaishali Kulkarni
129*14b24e2bSVaishali Kulkarni% Upper part of the page. The '~' is needed because \\
130*14b24e2bSVaishali Kulkarni% only works if a paragraph has started.
131*14b24e2bSVaishali Kulkarni\includegraphics[width=0.5\textwidth]{./qlogic-logo}~\\[3cm]
132*14b24e2bSVaishali Kulkarni
133*14b24e2bSVaishali Kulkarni% Title
134*14b24e2bSVaishali Kulkarni\HRule \\[0.4cm]
135*14b24e2bSVaishali Kulkarni{ \huge \bfseries E4 ecore \\[0.4cm] }
136*14b24e2bSVaishali Kulkarni
137*14b24e2bSVaishali Kulkarni\HRule \\[1.5cm]
138*14b24e2bSVaishali Kulkarni
139*14b24e2bSVaishali Kulkarni\begin{minipage}{0.4\textwidth}
140*14b24e2bSVaishali Kulkarni\begin{flushleft} \large
141*14b24e2bSVaishali Kulkarni\emph{Authors:}\\
142*14b24e2bSVaishali KulkarniAriel \textsc{Elior} \\
143*14b24e2bSVaishali KulkarniMichal \textsc{Kalderon} \\
144*14b24e2bSVaishali KulkarniYuval \textsc{Mintz} \\
145*14b24e2bSVaishali KulkarniMerav \textsc{Sicron} \\
146*14b24e2bSVaishali KulkarniTomer \textsc{Tayar} \\
147*14b24e2bSVaishali KulkarniSudarsana Reddy \textsc{Kalluru} \\
148*14b24e2bSVaishali Kulkarni\end{flushleft}
149*14b24e2bSVaishali Kulkarni\end{minipage}
150*14b24e2bSVaishali Kulkarni\begin{minipage}{0.4\textwidth}
151*14b24e2bSVaishali Kulkarni\begin{flushright} \large
152*14b24e2bSVaishali Kulkarni\emph{Version:} \\
153*14b24e2bSVaishali Kulkarni0.0.10
154*14b24e2bSVaishali Kulkarni\end{flushright}
155*14b24e2bSVaishali Kulkarni\end{minipage}
156*14b24e2bSVaishali Kulkarni
157*14b24e2bSVaishali Kulkarni\vfill
158*14b24e2bSVaishali Kulkarni
159*14b24e2bSVaishali Kulkarni% Bottom of the page
160*14b24e2bSVaishali Kulkarni{\large \today}
161*14b24e2bSVaishali Kulkarni
162*14b24e2bSVaishali Kulkarni\end{center}
163*14b24e2bSVaishali Kulkarni\end{titlepage}
164*14b24e2bSVaishali Kulkarni
165*14b24e2bSVaishali Kulkarni\pagestyle{empty} % No headers
166*14b24e2bSVaishali Kulkarni\chapterimage{qlogic-full-36}
167*14b24e2bSVaishali Kulkarni\tableofcontents % Print the table of contents itself
168*14b24e2bSVaishali Kulkarni
169*14b24e2bSVaishali Kulkarni\cleardoublepage % Forces the first chapter to start on an odd page so it's on the right
170*14b24e2bSVaishali Kulkarni
171*14b24e2bSVaishali Kulkarni\pagestyle{fancy} % Print headers again
172*14b24e2bSVaishali Kulkarni
173*14b24e2bSVaishali Kulkarni
174*14b24e2bSVaishali Kulkarni%----------------------------------------------------------------------------------------
175*14b24e2bSVaishali Kulkarni%	Real Content
176*14b24e2bSVaishali Kulkarni%----------------------------------------------------------------------------------------
177*14b24e2bSVaishali Kulkarni\chapterimage{pictures/qlogic-full-36.jpg}
178*14b24e2bSVaishali Kulkarni\chapter{Introduction}
179*14b24e2bSVaishali KulkarniBy definition, a driver is the entity which allows an OS to drive a hardware device.
180*14b24e2bSVaishali KulkarniAs such the driver contains both device-specific parts and OS-specific parts.
181*14b24e2bSVaishali KulkarniThe Everest architecture, with programmable fastpath processors (Storms), host-based device-dedicated memory (ILT), and minimal on-chip management presents a device which requires a driver with significant portions of device-specific code.
182*14b24e2bSVaishali Kulkarni
183*14b24e2bSVaishali KulkarniDrivers will be implemented for Everest 4 devices in many OSs (linux, windows, freebsd, solaris, esx, aix, hpux…).
184*14b24e2bSVaishali KulkarniImplementing the device-specific code again and again in each OS is both wasteful and difficult to maintain.
185*14b24e2bSVaishali KulkarniFor this purpose the ecore was conceived.
186*14b24e2bSVaishali KulkarniA large mass of code for operating and interacting with the Everest 4 device, to be incorporated into and used by OS drivers.
187*14b24e2bSVaishali Kulkarni
188*14b24e2bSVaishali KulkarniIn the abstract, the ecore is a layer between the HW/FW and the OS.
189*14b24e2bSVaishali KulkarniIt is device-specific and OS-agnostic. When ecore code requires OS services (e.g. memory allocation, pci configuration space access, etc.) it calls an abstract OS function for that purpose. These are implemented in OS-specific layers.
190*14b24e2bSVaishali KulkarniEcore flows may be driven by the HW (e.g. by an interrupt) or by the OS specific portion of the driver (e.g. driver load/unload).
191*14b24e2bSVaishali Kulkarni
192*14b24e2bSVaishali Kulkarni\begin{itemize}
193*14b24e2bSVaishali Kulkarni
194*14b24e2bSVaishali Kulkarni	\item Slowpath flows tend to reside largely in ecore and less so in OS specific layers. As much of the functionality as possible is placed in the ecore to leverage it across multiple platforms. \\
195*14b24e2bSVaishali Kulkarni
196*14b24e2bSVaishali Kulkarni	\item Fastpath flows tend to be in the OS specific layer as too much layering and abstraction is out of place in fastpath.
197*14b24e2bSVaishali KulkarniHowever, the fastpath would usually be set up by ecore flows, for example the address where transmission flow should write a doorbell to the BAR is determined by the ecore at init phase and this address is supplied by ecore to the OS specific layer. \\
198*14b24e2bSVaishali Kulkarni
199*14b24e2bSVaishali Kulkarni\end{itemize}
200*14b24e2bSVaishali Kulkarni
201*14b24e2bSVaishali KulkarniDifferent drivers in the same OS may have the ecore within them, and may use it for similar or different purposes:
202*14b24e2bSVaishali Kulkarni
203*14b24e2bSVaishali Kulkarni\begin{exampleT}
204*14b24e2bSVaishali Kulkarni	In linux there will be an ethernet driver, an fcoe driver, an iscsi driver, a roce driver and also a slim driver for the diag utility.
205*14b24e2bSVaishali Kulkarni	All of these may exists in the same system.
206*14b24e2bSVaishali Kulkarni	All of these will have an ecore instance incorporated in them.
207*14b24e2bSVaishali Kulkarni	Either one of the drivers might use the ecore to initialize the device, or the sections of the device pertaining to that driver’s operation.
208*14b24e2bSVaishali Kulkarni	A storage driver may use the ecore for storage specific purposes, such as the initialization and allocation of task context.
209*14b24e2bSVaishali Kulkarni\end{exampleT}
210*14b24e2bSVaishali Kulkarni
211*14b24e2bSVaishali KulkarniThe ecore is not a driver in its own capacity, but only code which is used by other drivers. Thus, separate drivers, including separate instances of the same driver within an OS, have separate instances of the ecore within them, which are concurrently active.
212*14b24e2bSVaishali Kulkarni
213*14b24e2bSVaishali Kulkarni\section{scope}
214*14b24e2bSVaishali KulkarniThis document strives to define and detail what is the ecore.
215*14b24e2bSVaishali KulkarniThe first parts of the document deal with the concept of the ecore, and its place in the software layers between the device and the OS.
216*14b24e2bSVaishali KulkarniThe rest of the document deals with the content of the ecore.
217*14b24e2bSVaishali KulkarniThis document does not deal with the needs and use cases of any specific OS or tool, but only with the common ground which is the ecore.
218*14b24e2bSVaishali Kulkarni
219*14b24e2bSVaishali KulkarniThe document sometimes delves in-depth into the inner-workings of the ecore; Since the programmer coming to utilize the ecore might not need [or want] to know those inner workings, such a person should look into specific sections in each chapter, specifically:
220*14b24e2bSVaishali Kulkarni\begin{enumerate}
221*14b24e2bSVaishali Kulkarni	\item Chapter \ref{cha:overview}'s introduction and section \ref{sec:overview-api} for a listing of the ecore API files and their locations.
222*14b24e2bSVaishali Kulkarni
223*14b24e2bSVaishali Kulkarni	\item OS abstraction layer [\ref{sec:osal}] for functions needed to be implemented by upper-layer driver in order to support the ecore.
224*14b24e2bSVaishali Kulkarni
225*14b24e2bSVaishali Kulkarni	\item Register-access [\ref{cha:reg}], mainly for learning about PTTs which are required by various ecore API functions.
226*14b24e2bSVaishali Kulkarni
227*14b24e2bSVaishali Kulkarni	\item Initialization and De-initialization of the HW [ \ref{sec:init-init}, \ref{sec:init-de-init}].
228*14b24e2bSVaishali Kulkarni
229*14b24e2bSVaishali Kulkarni	\item Status block initialization [\ref{ssec:sb-init}] and Interrupt handling flow [\ref{sec:sb-flow}].
230*14b24e2bSVaishali Kulkarni
231*14b24e2bSVaishali Kulkarni	\item Link interface [\ref{sec:mfw-link}].
232*14b24e2bSVaishali Kulkarni
233*14b24e2bSVaishali Kulkarni	\item Protocol related initialization/de-initialization:
234*14b24e2bSVaishali Kulkarni	\begin{enumerate}
235*14b24e2bSVaishali Kulkarni		\item L2-related, see Chapter [\ref{cha:l2}].
236*14b24e2bSVaishali Kulkarni	\end{enumerate}
237*14b24e2bSVaishali Kulkarni\end{enumerate}
238*14b24e2bSVaishali Kulkarni
239*14b24e2bSVaishali KulkarniIn addition, each chapter which includes ecore API functions that can be called by the upper-layer driver lists those functions' prototypes at its end.
240*14b24e2bSVaishali Kulkarni
241*14b24e2bSVaishali Kulkarni%\bibliography{ecore}
242*14b24e2bSVaishali Kulkarni
243*14b24e2bSVaishali Kulkarni\chapterimage{qlogic-full-36}
244*14b24e2bSVaishali Kulkarni\chapter{Ecore interface overview}
245*14b24e2bSVaishali Kulkarni\label{cha:overview}
246*14b24e2bSVaishali KulkarniThe ecore can be found at the perforce servers under:
247*14b24e2bSVaishali Kulkarni\begin{center}
248*14b24e2bSVaishali Kulkarni	//servers/main/nx2/579xx/drivers/ecore
249*14b24e2bSVaishali Kulkarni\end{center}
250*14b24e2bSVaishali Kulkarni
251*14b24e2bSVaishali KulkarniMost of the ecore consists of the \textit{inner} parts, i.e., HW-oriented implementation to which the upper-layer driver writer is oblivious.
252*14b24e2bSVaishali KulkarniAbove that is a concise API layer, through which the upper-layer driver should manipulate the ecore code.
253*14b24e2bSVaishali Kulkarni
254*14b24e2bSVaishali Kulkarni\section{Ecore API}
255*14b24e2bSVaishali Kulkarni\label{sec:overview-api}
256*14b24e2bSVaishali KulkarniThe Ecore API contains two types of files:
257*14b24e2bSVaishali Kulkarni\begin{enumerate}
258*14b24e2bSVaishali Kulkarni	\item Files of the format \texttt{ecore\_<module>\_api.h} -- these files are the SW API between the ecore and the upper-layer driver:
259*14b24e2bSVaishali Kulkarni	\begin{enumerate}
260*14b24e2bSVaishali Kulkarni		\item \texttt{ecore\_cxt\_api.h}.
261*14b24e2bSVaishali Kulkarni		\item \texttt{ecore\_dev\_api.h}.
262*14b24e2bSVaishali Kulkarni		\item \texttt{ecore\_fcoe\_api.h}.
263*14b24e2bSVaishali Kulkarni		\item \texttt{ecore\_int\_api.h}.
264*14b24e2bSVaishali Kulkarni		\item \texttt{ecore\_iov\_api.h}.
265*14b24e2bSVaishali Kulkarni		\item \texttt{ecore\_iscsi\_api.h}.
266*14b24e2bSVaishali Kulkarni		\item \texttt{ecore\_ll2\_api.h}.
267*14b24e2bSVaishali Kulkarni		\item \texttt{ecore\_roce\_api.h}.
268*14b24e2bSVaishali Kulkarni		\item \texttt{ecore\_sp\_api.h}.
269*14b24e2bSVaishali Kulkarni		\item \texttt{ecore\_vf\_api.h}.
270*14b24e2bSVaishali Kulkarni		\item \texttt{ecore\_mcp\_api.h}.
271*14b24e2bSVaishali Kulkarni	\end{enumerate}
272*14b24e2bSVaishali Kulkarni	\item Files of the format \texttt{ecore\_hsi\_<protocol>.h} -- these files contain the API between FW/HW and the the ecore/upper-layer driver:
273*14b24e2bSVaishali Kulkarni	\begin{enumerate}
274*14b24e2bSVaishali Kulkarni		\item \texttt{ecore\_hsi\_common.h}.
275*14b24e2bSVaishali Kulkarni		\item \texttt{ecore\_hsi\_eth.h}.
276*14b24e2bSVaishali Kulkarni		\item \texttt{ecore\_hsi\_fcoe.h}.
277*14b24e2bSVaishali Kulkarni		\item \texttt{ecore\_hsi\_iscsi.h}.
278*14b24e2bSVaishali Kulkarni		\item \texttt{ecore\_hsi\_roce.h}.
279*14b24e2bSVaishali Kulkarni		\item \texttt{ecore\_hsi\_tcp.h}.
280*14b24e2bSVaishali Kulkarni		\item \texttt{ecore\_hsi\_toe.h}.
281*14b24e2bSVaishali Kulkarni	\end{enumerate}
282*14b24e2bSVaishali Kulkarni\end{enumerate}
283*14b24e2bSVaishali KulkarniUpper-layer driver should not include any other ecore header file, as the rest of the header files are internal, with the following exceptions:
284*14b24e2bSVaishali Kulkarni\begin{itemize}
285*14b24e2bSVaishali Kulkarni	\item \texttt{ecore\_chain.h} -- Networking drivers will probably want to include this to benefit from the already-implemented chain.
286*14b24e2bSVaishali Kulkarni	\item \texttt{ecore\_utils.h} -- Useful macros which can be used by upper-layer driver.
287*14b24e2bSVaishali Kulkarni	\item \texttt{ecore\_status.h} -- contains \texttt{enum \_ecore\_status\_t}. Many of the ecore return values are of this type.
288*14b24e2bSVaishali Kulkarni\end{itemize}
289*14b24e2bSVaishali Kulkarni
290*14b24e2bSVaishali Kulkarni\begin{warning}
291*14b24e2bSVaishali KulkarniCurrently \texttt{ecore.h, ecore\_proto\_if.h} should also be included by upper-layer driver; This will (hopefully) be fixed shortly.
292*14b24e2bSVaishali Kulkarni\end{warning}
293*14b24e2bSVaishali Kulkarni
294*14b24e2bSVaishali Kulkarni
295*14b24e2bSVaishali Kulkarni\section{Ecore Internal files}
296*14b24e2bSVaishali KulkarniThis lists the ecore files, giving each a short description:
297*14b24e2bSVaishali Kulkarni
298*14b24e2bSVaishali Kulkarni\begin{itemize}
299*14b24e2bSVaishali Kulkarni	\item \texttt{ecore\_attn\_values.h}
300*14b24e2bSVaishali Kulkarni
301*14b24e2bSVaishali Kulkarni	\item \texttt{ecore\_chain.h} -- Implements a cyclic chain; Used for various interfaces with the FW [Buffer-Descriptoss, Event Queues, etc.].
302*14b24e2bSVaishali Kulkarni
303*14b24e2bSVaishali Kulkarni	\item \texttt{ecore\_cxt\_api.[ch]} -- Handles the allocation, configuration and distribution of contexts to the various clients.
304*14b24e2bSVaishali Kulkarni
305*14b24e2bSVaishali Kulkarni	\item \texttt{ecore\_dbg\_fw\_funcs.[ch], ecore\_dbg\_values.h, ecore\_fw\_defs} -- Files which contain code related for various debug features ecore can provide [e.g., grcDump].
306*14b24e2bSVaishali Kulkarni
307*14b24e2bSVaishali Kulkarni	\item \texttt{ecore\_fcoe.[ch], ecore\_iscsi.[ch], ecore\_ll2.[ch], ecore\_roce.[ch]} -- files containing specific ecore code for the storage protocols.
308*14b24e2bSVaishali Kulkarni
309*14b24e2bSVaishali Kulkarni	\item \texttt{ecore\_dev.[ch]} -- Contains much of the functionality of starting/stopping the hardware. See chapter \ref{cha:hwinit}.
310*14b24e2bSVaishali Kulkarni
311*14b24e2bSVaishali Kulkarni	\item \texttt{ecore\_hw.[ch], ecore\_gtt\_reg\_addr.h, ecore\_gtt\_values.h} -- contains the functionality for register access and DMAE. See chapter \ref{cha:reg}.
312*14b24e2bSVaishali Kulkarni
313*14b24e2bSVaishali Kulkarni	\item \texttt{ecore.h} -- contains the defintion of the most \textit{elementary} structures in the ecore, the \texttt{ecore\_dev} and the \texttt{ecore\_hwfn}.
314*14b24e2bSVaishali Kulkarni
315*14b24e2bSVaishali Kulkarni	\item \texttt{ecore\_init\_defs.h, ecore\_init\_fw\_funcs.[ch], ecore\_init\_ops.[ch], \\ ecore\_init\_values.h, ecore\_rt\_defs} -- Code responsible for initialization and configuration of the HW and loading of the FW, mostly in relation with the init-tool. See chapter \ref{cha:hwinit}.
316*14b24e2bSVaishali Kulkarni	\begin{REMINDER}
317*14b24e2bSVaishali Kulkarni			Chapter \ref{cha:hwinit} doesn't really give a thorough explanation of the init tool - at most it mentions it. Do we want a section/chapter of it somewhere?
318*14b24e2bSVaishali Kulkarni	\end{REMINDER}
319*14b24e2bSVaishali Kulkarni
320*14b24e2bSVaishali Kulkarni	\item \texttt{ecore\_int.[ch]} -- Handles interrupts and attentions. See chapter \ref{cha:int}.
321*14b24e2bSVaishali Kulkarni
322*14b24e2bSVaishali Kulkarni	\item \texttt{ecore\_iro.h, ecore\_iro\_values.h} -- Generated FW files. Enables ecore to access [or supply to upper-layer] addresses inside the \texttt{storm}'s RAM.
323*14b24e2bSVaishali Kulkarni
324*14b24e2bSVaishali Kulkarni	\item \texttt{ecore\_mcp.[ch]} -- Contains the interface between the ecore and the MFW. See chapter \ref{cha:mfw}.
325*14b24e2bSVaishali Kulkarni
326*14b24e2bSVaishali Kulkarni	\item \texttt{ecore\_sp\_commands.[ch], ecore\_spq.[ch]} -- Contained the slowpath logic required for sending ramrods and configuring \& handling the various slowpath events.
327*14b24e2bSVaishali Kulkarni
328*14b24e2bSVaishali Kulkarni	\item \texttt{ecore\_sriov.[ch], ecore\_vf.[ch], ecore\_vfpf\_if.h} -- Contains the SRIOV implementation both from the PF and VF sides.
329*14b24e2bSVaishali Kulkarni\end{itemize}
330*14b24e2bSVaishali Kulkarni
331*14b24e2bSVaishali Kulkarni\section{OS abstraction Layer}
332*14b24e2bSVaishali Kulkarni\label{sec:osal}
333*14b24e2bSVaishali Kulkarni
334*14b24e2bSVaishali Kulkarni%\section{Driver Core}
335*14b24e2bSVaishali Kulkarni%As the ecore contains most of the lowlevel code operating the non-fastpath parts of the working with the HW and FW, it can be thought of as some sort of library – it contains bits of code meant to be operated from an outside source. Each OS needs to implement its own driver, calling the various functions in the ecore API in a place fitting for that OS driver flows.
336*14b24e2bSVaishali Kulkarni%Each OS will independently need to create a driver that incorporates the ecore, both filling the OS dependent callbacks required by the ecore to perform and supply an upper level of abstraction which best suits that OS. Notice this upper layer is sometimes also, mistakenly, referred to as ecore [e.g., bnx2c for linux drivers] but there’s an important distinction:
337*14b24e2bSVaishali Kulkarni%\begin{itemize}
338*14b24e2bSVaishali Kulkarni%	\item Ecore – shared code between ALL operating systems.
339*14b24e2bSVaishali Kulkarni%	\item Upper-Layer – shared code by all drivers on a single operating system.
340*14b24e2bSVaishali Kulkarni%\end{itemize}
341*14b24e2bSVaishali Kulkarni
342*14b24e2bSVaishali Kulkarni%It’s possible [and likely] that an operating system will break the various protocols into different sub-drivers, where each sub-driver will be designated for a specific protocol. Notice that if such separation is made, the preferred implementation is that the OS will implement a ‘core’ driver consisting of the Ecore and an upper-layer, and define an API through which the various protocol drivers communicate with the OS core driver\footnote{Although notice there should be no inter-dependencies between HW-functions in the ecore, so the alternative method where each contains the ecore is also feasible}.
343*14b24e2bSVaishali Kulkarni
344*14b24e2bSVaishali KulkarniThe ecore utilizes various functions which should be implemented by the upper layer. There are two main ‘types’ of functions:
345*14b24e2bSVaishali Kulkarni\begin{enumerate}
346*14b24e2bSVaishali Kulkarni	\item Basic OS-specific operations that the ecore needs in order to perform it’s work; e.g., memory allocations – the ecore needs to allocate memory for various reasons, and it needs the upper layer to supply the method by which it can do so.
347*14b24e2bSVaishali Kulkarni	\item Hooks by which the upper-layer can run additional OS specific code, or make decisions affecting the work of the ecore. E.g., in the SRIOV flows, the mechanism for passing messages from VF to PF is implemented in the ecore but the decision whether a request is valid or not might be OS specific – as in the case of unicast filters.
348*14b24e2bSVaishali Kulkarni\end{enumerate}
349*14b24e2bSVaishali Kulkarni
350*14b24e2bSVaishali KulkarniThe various functions that need to be implemented by the upper-layer can be found in Appendix \ref{app:osal} -- OSAL Documentation.
351*14b24e2bSVaishali Kulkarni
352*14b24e2bSVaishali Kulkarni
353*14b24e2bSVaishali Kulkarni\section{Ecore print scheme}
354*14b24e2bSVaishali KulkarniThe ecore utilizes several printing methods to print messages to the system logs; It requires some functions to be implemented by the upper-layer for this to work – the required documentation can be found in Appendix \ref{app:osal} -- OSAL Documentation.
355*14b24e2bSVaishali KulkarniIn order to support this, the verbosity mechanism contains two distinct values \myindex{\texttt{DP\_LEVEL}} and \myindex{\texttt{DP\_MODULE}} [both can be found in \texttt{ecore.h}]. Since the printing scheme in the ecore was defined with the linux limitations in mind – that is, the API [via ethtool] allowing the setting of the debug message level is only 32-bit long, both \texttt{DP\_MODULE} and \texttt{DP\_LEVEL} together contain only 32-bits.
356*14b24e2bSVaishali KulkarniThe \texttt{DP\_LEVEL} determines which prints will actually reach the logs based on the message urgency, defining 4 levels – verbose, info, notice and error. When level is set, all prints which are at least as urgent will be printed. Notice this means there’s a single level – e.g., you can’t have a configuration in which you’ll get all the `info’ level prints, but not the `notice’ level.
357*14b24e2bSVaishali KulkarniThe \texttt{DP\_MODULE} is relevant only when level is set to verbose, and it defines which of the verbose prints should reach system logs, based mostly on component/flow. When setting the module level, a bit mask of the requested components/flows is set.
358*14b24e2bSVaishali KulkarniIn order to set which prints should reach system logs, the upper layer should utilize the ecore function \myfunc{init\_dp}{init_dp} defined in \texttt{ecore\_dev.c}.
359*14b24e2bSVaishali Kulkarni
360*14b24e2bSVaishali Kulkarni\section{Compilation flags}
361*14b24e2bSVaishali KulkarniThe ecore project contains several optional compilation flags that if passed would affect the content compiled. A few notable flags:
362*14b24e2bSVaishali Kulkarni\begin{itemize}
363*14b24e2bSVaishali Kulkarni	\item ASIC\_ONLY -- By default, this is `off'. Setting this would remove content that is relevant only for simulations of the hardware, I.e., emulations and FPGAs.
364*14b24e2bSVaishali Kulkarni
365*14b24e2bSVaishali Kulkarni	\item REAL\_ASIC\_ONLY -- By default, this is `off'. Setting this would remove content that is relevant for non-productized hardware, E.g., workarounds for BigBear A0.
366*14b24e2bSVaishali Kulkarni
367*14b24e2bSVaishali Kulkarni	\item REMOVE\_DBG -- By default, this is `off'. There are several structures and field in ecore which aren't functional; there sole purpose is to store interesting data for memory dumps in case of failures. Setting this would remove all such data items.
368*14b24e2bSVaishali Kulkarni\end{itemize}
369*14b24e2bSVaishali Kulkarni
370*14b24e2bSVaishali Kulkarni\SpillChapterFuncs
371*14b24e2bSVaishali Kulkarni
372*14b24e2bSVaishali Kulkarni
373*14b24e2bSVaishali Kulkarni\chapterimage{qlogic-full-36}
374*14b24e2bSVaishali Kulkarni\chapter{Register Access}
375*14b24e2bSVaishali Kulkarni\label{cha:reg}
376*14b24e2bSVaishali KulkarniThis section describes the ecore API for accessing registers.
377*14b24e2bSVaishali KulkarniThe E4 bar is a reduced BAR, i.e., it does not map the entire register address range.
378*14b24e2bSVaishali KulkarniTo access the entire range, windows are defined that can be configured to point to a certain address within the device and allow reading and writing of registers / memory from that address.
379*14b24e2bSVaishali KulkarniThere are two types of windows, \textbf{PTT} (per PF Translation Table) and \textbf{GTT} (Global Translation Table).
380*14b24e2bSVaishali Kulkarni
381*14b24e2bSVaishali KulkarniThe \textit{external BAR} is the BAR accessed by the ecore. It is divided into configurable windows which point to different areas within the device (Image \ref{fig:bars}, Internal BAR vs. External BAR, demonstrates this).
382*14b24e2bSVaishali Kulkarni
383*14b24e2bSVaishali Kulkarni\begin{figure}[ht]
384*14b24e2bSVaishali Kulkarni	\caption{Internal BAR vs. External BAR}
385*14b24e2bSVaishali Kulkarni	\centering
386*14b24e2bSVaishali Kulkarni	\includegraphics[width=0.8\paperwidth]{reg_access}
387*14b24e2bSVaishali Kulkarni	\label{fig:bars}
388*14b24e2bSVaishali Kulkarni\end{figure}
389*14b24e2bSVaishali Kulkarni
390*14b24e2bSVaishali KulkarniFor more details on the E4 BAR access scheme the reader is referred to the “Reduced PF BAR0 size” section of \cite{doc:PXP}. \\
391*14b24e2bSVaishali Kulkarni
392*14b24e2bSVaishali Kulkarni
393*14b24e2bSVaishali KulkarniAll register access should be done within the ecore layer and it is not expected for the upper layers to access registers at all.
394*14b24e2bSVaishali KulkarniFor this reason, there is no description here on how to find the register address and how to distinguish whether the address is mapped into a \myindex{GTT} or a \myindex{PTT}.
395*14b24e2bSVaishali KulkarniHowever, in case a need does rise in the future, API for reading/writing is detailed below as well.
396*14b24e2bSVaishali Kulkarni
397*14b24e2bSVaishali KulkarniEcore requires an OSAL implementation of the macros:
398*14b24e2bSVaishali Kulkarni\begin{enumerate}
399*14b24e2bSVaishali Kulkarni	\item \myindex{REG\_RD}
400*14b24e2bSVaishali Kulkarni	\item \myindex{REG\_WR}
401*14b24e2bSVaishali Kulkarni\end{enumerate}
402*14b24e2bSVaishali KulkarniThese macros are a direct read / write from the BAR with the absolute address offset given.
403*14b24e2bSVaishali KulkarniImplementation should add the offset to the mapped BAR address and call the appropriate OS specific API.
404*14b24e2bSVaishali Kulkarni
405*14b24e2bSVaishali KulkarniSeveral ecore interface functions require a PTT. There is a pool of PTTs maintained by ecore.
406*14b24e2bSVaishali KulkarniThe reason there are several PTTs is to enable simultaneous access to device registers from different flows.
407*14b24e2bSVaishali KulkarniThe PTT is reserved per flow, and it is the responsibility of the upper layer to make sure it does not use the same PTT in flows that can run concurrently. Upper-layer requests for a PTT entry using \myfunc{ptt\_acquire}{ptt_acquire}.
408*14b24e2bSVaishali KulkarniHowever, to avoid running out of this resource, it is also the responsibility of the upper layer not to acquire too many PTTs without releasing them. Returning a PTT entry back to the pool is done via \myfunc{ptt\_release}{ptt_release}.
409*14b24e2bSVaishali Kulkarni
410*14b24e2bSVaishali KulkarniUsing a PTT, ecore [and upper-driver] can access registers/memories using inner BAR addresses; The ecore is responsible for configuring the memory windows, and translates the inner address into an external address [i.e., one which resides on the actual BAR as seen by the host]. The register access is then made by calling \texttt{ecore\_wr} and \texttt{ecore\_rd}.
411*14b24e2bSVaishali Kulkarni\SpillChapterFuncs
412*14b24e2bSVaishali Kulkarni
413*14b24e2bSVaishali Kulkarni
414*14b24e2bSVaishali Kulkarni\chapterimage{qlogic-full-36}
415*14b24e2bSVaishali Kulkarni\chapter{Hardware/Firmware initialization}
416*14b24e2bSVaishali Kulkarni\label{cha:hwinit}
417*14b24e2bSVaishali Kulkarni
418*14b24e2bSVaishali Kulkarni\section{Basic concepts -- inner-working of the ecore}
419*14b24e2bSVaishali Kulkarni\begin{itemize}
420*14b24e2bSVaishali Kulkarni	\item \myindex{ILT} – one of the features of our device is that the memories used by various HW blocks are allocated on the host memory [as opposed to large embedded memory segment on chip]. The driver is responsible for allocating the memory needed for those HW blocks [DMA-coherent memory] and configure both the HW blocks themselves and a communal sub-block known as ILT. The ecore contains complicated code that decides exactly how much memory each such block needs, allocates it in an ‘ilt\_shadow’, and then uses that shadow to configure the ILT itself with all the allocated chunks of memory.
421*14b24e2bSVaishali Kulkarni
422*14b24e2bSVaishali KulkarniAdditional ILT documentation is at \cite{doc:ILT}.
423*14b24e2bSVaishali Kulkarni
424*14b24e2bSVaishali Kulkarni	\item \myindex{RT array} – when the ecore initializes the HW, it utilizes a common, tool-generated code known as the init-tool. Since there are quite a few values which depend upon actual setup configuration and thus must receive feedback during the initialization from the ecore, instead of adding many such hooks there’s the concept of the RunTime array – an array of values filled by the ecore prior to the init-tool run based on the complex ecore logic. The init-tool will then utilize the values in that array to configure the HW according to the correct order of configuration [i.e., writing the values set by ecore in the array in the correct place in the initialization flow where they’re required/the block that contains them is configured].
425*14b24e2bSVaishali Kulkarni\end{itemize}
426*14b24e2bSVaishali Kulkarni
427*14b24e2bSVaishali Kulkarni\section{Initialization}
428*14b24e2bSVaishali Kulkarni\label{sec:init-init}
429*14b24e2bSVaishali KulkarniThe functions required for initializing the HW/FW mostly reside in \texttt{ecore\_dev.[ch]}; More accurately, most of the outside API [toward the upper-layer] is in \texttt{ecore\_dev.h} – the functions themselves utilize many other ecore files.
430*14b24e2bSVaishali KulkarniThis section gives a brief description of the functions that need to be called, what they do, requirements, etc., in order to successfully initialize the ecore structs and load the HW/FW.
431*14b24e2bSVaishali Kulkarni
432*14b24e2bSVaishali Kulkarni\silentfunc{init_struct}
433*14b24e2bSVaishali Kulkarni\silentfunc{hw_prepare}
434*14b24e2bSVaishali Kulkarni\silentfunc{resc_alloc}
435*14b24e2bSVaishali Kulkarni\silentfunc{resc_setup}
436*14b24e2bSVaishali Kulkarni\silentfunc{hw_init}
437*14b24e2bSVaishali Kulkarni\begin{itemize}
438*14b24e2bSVaishali Kulkarni	\item \myfunc{init\_struct}{init\_struct} – After allocating and setting of zeroes of the ecore\_dev [the upper-layer responsibility], a pointer to it should be passed to this function for some early initialization of the data structure. \\
439*14b24e2bSVaishali Kulkarni
440*14b24e2bSVaishali Kulkarni	\item \myfunc{hw\_prepare}{hw_prepare} – This function serves two purposes [plus some additional inner ecore workings]:
441*14b24e2bSVaishali Kulkarni	\begin{enumerate}
442*14b24e2bSVaishali Kulkarni		\item It enables the ecore to access its BAR, doing things such as enabling the PTT pool and opening the access in the PGLUE\_B block.
443*14b24e2bSVaishali Kulkarni		Notice this doesn’t actually do anything to the PCI BAR itself – the upper-layer should have initialized those before calling this function, and must guarantee that its REG\_WR/RD functions actually point to valid, accessible addresses.
444*14b24e2bSVaishali Kulkarni		\item It learns as much as it can about system configuration from HW and SHMEM.
445*14b24e2bSVaishali Kulkarni	\end{enumerate}
446*14b24e2bSVaishali Kulkarni
447*14b24e2bSVaishali KulkarniTrying to access registers except for pci-related ones prior to calling this function will fail. \\
448*14b24e2bSVaishali Kulkarni
449*14b24e2bSVaishali Kulkarni	\item \myfunc{resc\_alloc}{resc_alloc} – Allocates the various ecore-related memory, e.g., contexts, slowpath queue, SRIOV information, etc. Notice that before calling this function, each HW-function of the \texttt{ecore\_dev} should have its `pf\_params’ set, as the function depends upon the protocol-specific resources for its calculations. \\
450*14b24e2bSVaishali Kulkarni
451*14b24e2bSVaishali Kulkarni	\item \myfunc{resc\_setup}{resc_setup} – Configures the various slowpath elements. Notice that since there’s no guarantee chip is alive at this point [i.e., it’s very likely the chip is reset at this point], it fills the configuration in the runtime array instead of actually writing it to chip. \\
452*14b24e2bSVaishali Kulkarni
453*14b24e2bSVaishali Kulkarni	\item \myfunc{hw\_init}{hw_init} – This function actually initializes the chip, using the init-tool and the runtime array to make the correct configuration.
454*14b24e2bSVaishali Kulkarni	 As part of the slowpath interrupt enablement, ecore invokes OSAL\_SLOWPATH\_IRQ\_REQ() callback for each HW function. The client implementation should setup the IRQ handlers for slowpath interrupt handling.
455*14b24e2bSVaishali Kulkarni	 This is required since as part of the flow the \texttt{function\_start} ramrod will be sent to FW; Once FW finishes handling it, an \myindex{EQE} [Event Queue Element] will be placed in the slowpath event queue and an interrupt will be fired. The flow is dependent on the EQE being processed.
456*14b24e2bSVaishali Kulkarni
457*14b24e2bSVaishali Kulkarni	Some interesting sub-functions of the \texttt{ecore\_hw\_init()} method, at least for debugging purposes as many possible errors can be caught there:
458*14b24e2bSVaishali Kulkarni	\begin{itemize}
459*14b24e2bSVaishali Kulkarni		\item \texttt{ecore\_get\_init\_mode()} – this creates a bitmask which will be later passed to the init-tool which describes the configured mode – Multi function vs. Single function, 40G vs. 100G etc. A wrong configuration here could explain many peculiar events later on. \\
460*14b24e2bSVaishali Kulkarni
461*14b24e2bSVaishali Kulkarni%		\item ecore\_mcp\_load\_req() – the MFW [assuming it is present] will answer with one of 3 possible answers: ENGINE, PORT or FUNCTION.
462*14b24e2bSVaishali Kulkarni%		The MFW is responsible for initializing the common blocks [i.e., the HW blocks shared between the 2 engines], but the driver is responsible for the rest.
463*14b24e2bSVaishali Kulkarni%		Each function needs to perform different initialization based on whether it’s the first to load on its engine [ENGINE], the first to load on its port [PORT] or if it’s being loaded on an already initialized port [FUNCTION]\footnote{Initialization which is common for both engines will be performed by the MFW.}.
464*14b24e2bSVaishali Kulkarni%	Some very basic errors can be detected here, if the function receives an unexpected answer from MFW.
465*14b24e2bSVaishali Kulkarni	\end{itemize}
466*14b24e2bSVaishali Kulkarni
467*14b24e2bSVaishali KulkarniOnce this function returns, the chip is initialized, FW is functional and slowpath event queues are operational.
468*14b24e2bSVaishali Kulkarni
469*14b24e2bSVaishali Kulkarni\end{itemize}
470*14b24e2bSVaishali Kulkarni
471*14b24e2bSVaishali Kulkarni\section{Zipped and Binary firmware}
472*14b24e2bSVaishali Kulkarni\label{sec:init-Zipped and Binary firmware}
473*14b24e2bSVaishali Kulkarni\begin{itemize}
474*14b24e2bSVaishali Kulkarni	\item \myindex{Zipped Firmware} - There are two types of firmware files generated in ecore.\\
475*14b24e2bSVaishali KulkarniNon-zipped firmware [ecore\_init\_values.h and ecore\_init\_values.bin] and Zipped firmware [ecore\_init\_values\_zipped.h and
476*14b24e2bSVaishali Kulkarniecore\_init\_values\_zipped.bin] files. Each type of file is generated in two formats that is a C header file and binary file,
477*14b24e2bSVaishali Kulkarniwhere each has all relevant data needed to initialize the firmware. Either of these file types can be used for firmware initialization.
478*14b24e2bSVaishali KulkarniThe difference is that Zipped firmware files has lot of dmae firmware data zipped which is beneficiary in reducing the code size.\\
479*14b24e2bSVaishali Kulkarni
480*14b24e2bSVaishali KulkarniBy default, the non-zipped variant is used. If ecore clients want to use zipped version of firmware then they need to have
481*14b24e2bSVaishali KulkarniCONFIG\_ECORE\_ZIPPED\_FW defined/enabled by their operating system drivers to make feature operational. For unzipping the
482*14b24e2bSVaishali Kulkarnizipped firmware data ecore clients need to implement OSAL\_UNZIP\_DATA() as well. This OSAL is meant for unzipping the
483*14b24e2bSVaishali Kulkarnizipped firmware data in order to do firmware initialization.\\
484*14b24e2bSVaishali Kulkarni
485*14b24e2bSVaishali Kulkarni	\item \myindex{Binary Firmware} - As explained above there are two formats of firmware files
486*14b24e2bSVaishali Kulkarnigenerated by ecore, C header files [ecore\_init\_values.h and ecore\_init\_values\_zipped.h] and
487*14b24e2bSVaishali Kulkarnibinary firmware files [ecore\_init\_values.bin and ecore\_init\_values\_zipped.bin]. Either of those files formats
488*14b24e2bSVaishali Kulkarnican be used by ecore clients to utilize firmware data. By default, ecore uses the .h files which are compiled as part of the ecore,
489*14b24e2bSVaishali Kulkarnibut using binary firmware files has the advantage where the code size is reduced and the FW can be loaded from a file imported by
490*14b24e2bSVaishali Kulkarnithe system.\\
491*14b24e2bSVaishali Kulkarni
492*14b24e2bSVaishali KulkarniIf ecore clients want to use firmware data from binary files then they need to have CONFIG\_ECORE\_BINARY\_FW defined/enabled by their
493*14b24e2bSVaishali Kulkarnioperating system drivers to make feature operational. Ecore clients must store all binary firmware data from the
494*14b24e2bSVaishali Kulkarnifile in to a void* pointer and pass that firmware data buffer pointer in ecore\_hw\_init() as an argument.
495*14b24e2bSVaishali KulkarniIf ecore client is not using binary firmware file or instead using firmware from regular header files then they
496*14b24e2bSVaishali Kulkarnishould pass NULL as an argument for binary firmware data buffer in ecore\_hw\_init().
497*14b24e2bSVaishali Kulkarni
498*14b24e2bSVaishali Kulkarni
499*14b24e2bSVaishali Kulkarni\end{itemize}
500*14b24e2bSVaishali Kulkarni
501*14b24e2bSVaishali Kulkarni\section{De-Initialization}
502*14b24e2bSVaishali Kulkarni\label{sec:init-de-init}
503*14b24e2bSVaishali Kulkarni\silentfunc{hw_stop}
504*14b24e2bSVaishali Kulkarni\silentfunc{resc_free}
505*14b24e2bSVaishali Kulkarni\silentfunc{hw_remove}
506*14b24e2bSVaishali Kulkarni\begin{itemize}
507*14b24e2bSVaishali Kulkarni	\item \myfunc{hw\_stop}{hw_stop} – this function notifies the MFW that the HW-functions unload, stops the FW/HW for all HW-functions in the \texttt{ecore\_dev} including sending the common PF\_STOP ramrod for each HW-function, and disables the HW-functions in various HW blocks.
508*14b24e2bSVaishali Kulkarni	Notice that before calling this, all the protocol specifics done after initializing the HW should have already been reversed by the upper-layer [e.g., L2 VPORTs which were started by the upper layer should be stopped before calling this].
509*14b24e2bSVaishali Kulkarni	Following this function, it is guaranteed HW will not generate any more slowpath interrupts, so the interrupt handler can be released [and slowpath DPC context can be stopped]. \\
510*14b24e2bSVaishali Kulkarni
511*14b24e2bSVaishali Kulkarni	\item \myfunc{int\_disable\_post\_isr\_release}{ecore_int_disable_post_isr_release} – this function performs the required IRQ related cleanup post the ISR release. The function need to be called after releasing all slowpath IRQs of the device.
512*14b24e2bSVaishali Kulkarni
513*14b24e2bSVaishali Kulkarni	\item \myfunc{resc\_free}{resc_free} – Releases the memory allocated by the ecore during \texttt{ecore\_resc\_alloc()}. \\
514*14b24e2bSVaishali Kulkarni
515*14b24e2bSVaishali Kulkarni	\item \myfunc{hw\_remove}{hw_remove} – Release the memory allocated early by the ecore during \texttt{ecore\_hw\_prepare()}.
516*14b24e2bSVaishali Kulkarni	Following this, REG\_RD/REG\_WR are no longer operational - upper layer can disable the PCI BAR.
517*14b24e2bSVaishali Kulkarni\end{itemize}
518*14b24e2bSVaishali Kulkarni\SpillChapterFuncs
519*14b24e2bSVaishali Kulkarni
520*14b24e2bSVaishali Kulkarni%\chapterimage{qlogic-full-36}
521*14b24e2bSVaishali Kulkarni%\chapter{Firmware hsi}
522*14b24e2bSVaishali Kulkarni%\begin{NOTICE}
523*14b24e2bSVaishali Kulkarni%Placeholder - owner Michal
524*14b24e2bSVaishali Kulkarni%\end{NOTICE}
525*14b24e2bSVaishali Kulkarni
526*14b24e2bSVaishali Kulkarni
527*14b24e2bSVaishali Kulkarni\chapterimage{qlogic-full-36}
528*14b24e2bSVaishali Kulkarni\chapter{Interrupts}
529*14b24e2bSVaishali Kulkarni\label{cha:int}
530*14b24e2bSVaishali KulkarniThis chapter describes how the device notifies the driver about operations -
531*14b24e2bSVaishali Kulkarniit describes how firmware status is reflected on host memory via status blocks, and how the firmware initiates an interrupt toward the driver.
532*14b24e2bSVaishali Kulkarni
533*14b24e2bSVaishali KulkarniA reference document that fully describes status blocks can be found at \cite{doc:SB}.
534*14b24e2bSVaishali Kulkarni
535*14b24e2bSVaishali Kulkarni
536*14b24e2bSVaishali Kulkarni\section{Status blocks - host point of view}
537*14b24e2bSVaishali KulkarniThe \myindex{status block} structures are allocated on host memory. The status block is an array of indices which are updated by firmware (mainly ring consumer values).
538*14b24e2bSVaishali KulkarniThere are 288 status blocks per path in Big Bear and 368 in K2.
539*14b24e2bSVaishali Kulkarni
540*14b24e2bSVaishali KulkarniWhen one of the indices on a status block is updated (because some event occurred at the device), the status block is copied from internal device memory to host memory, and an interrupt is generated.
541*14b24e2bSVaishali KulkarniThe CAU unit may aggregate several events and generate a single update of the status block and a single interrupt, in order to lower the number of interrupts sent to host CPU.
542*14b24e2bSVaishali Kulkarni
543*14b24e2bSVaishali KulkarniThe indices of the status blocks are referred to as \myindex{protocol indices} (abbreviated to \textit{pi}).
544*14b24e2bSVaishali KulkarniOriginally, the motivation behind multiple status blocks was to enable multiple protocols to work with the same status block, giving each protocol a different index.
545*14b24e2bSVaishali KulkarniHowever, with single personality this is no longer the case.
546*14b24e2bSVaishali KulkarniMultiple indices are used for L2 to differentiate between RX / TX and different class of service operations.
547*14b24e2bSVaishali Kulkarni
548*14b24e2bSVaishali Kulkarni\subsection{Initialization}
549*14b24e2bSVaishali Kulkarni\label{ssec:sb-init}
550*14b24e2bSVaishali KulkarniThere is a dedicated status block for ecore usage which is allocated and maintained by ecore.
551*14b24e2bSVaishali KulkarniThe fastpath status blocks used for traffic need to be allocated by the protocol driver.
552*14b24e2bSVaishali KulkarniThis memory must be DMA-coherent memory.
553*14b24e2bSVaishali KulkarniThe ecore defines a structure called \texttt{ecore\_sb\_info} which should be allocated by the protocol driver and initialized using the function \myfunc{int\_sb\_init}{int_sb_init}
554*14b24e2bSVaishali Kulkarni%[code snippet \ref{snippets/ecore_int_sb_init.h}].
555*14b24e2bSVaishali KulkarniThis structure is later used for calling the functions \texttt{ecore\_sb\_update\_sb\_idx()} and \texttt{ecore\_sb\_ack()}.
556*14b24e2bSVaishali Kulkarni
557*14b24e2bSVaishali Kulkarni%\insertcode{snippets/ecore_int_sb_init.h}{Initialize status blocks}
558*14b24e2bSVaishali Kulkarni
559*14b24e2bSVaishali Kulkarni\begin{NOTICE}
560*14b24e2bSVaishali Kulkarni	Status blocks need to be allocated and initialized before queues are created.
561*14b24e2bSVaishali Kulkarni\end{NOTICE}
562*14b24e2bSVaishali Kulkarni
563*14b24e2bSVaishali Kulkarni\section{Mode and configuration}
564*14b24e2bSVaishali KulkarniThe device can work in one of the following interrupt modes:
565*14b24e2bSVaishali Kulkarni\begin{enumerate}
566*14b24e2bSVaishali Kulkarni	\item INTA – Physical interrupt line.
567*14b24e2bSVaishali Kulkarni	\item MSI –  Message signaled interrupts. Device is programmed with one address to write to, and 16-bit data to identify the interrupt.
568*14b24e2bSVaishali Kulkarni	\item MSIX – Large number of interrupts (up to 2048) and each one gets a separate target address, making it possible to designate different interrupts to different processors.
569*14b24e2bSVaishali Kulkarni	This is the preferred mode for performance.
570*14b24e2bSVaishali Kulkarni	\item POLL – HW increments producers on status blocks in case of interrupts but it doesn't generate any message nor does it assert any physical line. It's the upper-layer responsibility to periodically poll on those changes to identify interrupts. \\
571*14b24e2bSVaishali Kulkarni\end{enumerate}
572*14b24e2bSVaishali Kulkarni
573*14b24e2bSVaishali KulkarniEnabling and disabling interrupts is OS specific and done differently by the OS specific layer of the driver.
574*14b24e2bSVaishali KulkarniHowever, the device needs to be configured differently according to the selected interrupt mode; This initialization is done by the ecore.
575*14b24e2bSVaishali Kulkarni
576*14b24e2bSVaishali KulkarniIn order to so, the proper interrupt mode using an \myindex{ecore\_int\_mode} enum [can be seen in code snippet [\ref{snippets/ecore_int_mode.h}]] needs to be passed when calling \texttt{ecore\_hw\_init}.
577*14b24e2bSVaishali Kulkarni
578*14b24e2bSVaishali Kulkarni\insertcode{snippets/ecore_int_mode.h}{Enum for the interrupt mode}
579*14b24e2bSVaishali Kulkarni
580*14b24e2bSVaishali KulkarniIf upper-layer driver would later wish to change the interrupt mode, it can do so by calling \myfunc{int\_igu\_enable\_int}{int_igu_enable_int},
581*14b24e2bSVaishali Kulkarnior to \myfunc{int\_igu\_disable\_int}{int_igu_disable_int} when wishing to disable interrupt generation altogether.
582*14b24e2bSVaishali Kulkarni
583*14b24e2bSVaishali Kulkarni%\insertcode{snippets/ecore_int_endis.h}{Functions for enabling/disabling interrupts}
584*14b24e2bSVaishali Kulkarni
585*14b24e2bSVaishali KulkarniIn MSIX mode, each status block should generate it's own interrupt message, meaning in reasonable OSes it should be possible to connect each interrupt with the specific handler of that interrupt's source.
586*14b24e2bSVaishali KulkarniThe \textit{sb\_id} passed as value to \textit{ecore\_int\_sb\_init()} will indicate the index of the vector in the MSI-X table that would be used to generate interrupts for this specific SB.
587*14b24e2bSVaishali KulkarniI.e., if the value passed is $X$, then the $X^{th}$ MSI-X vector will generate interrupts for this SB.
588*14b24e2bSVaishali Kulkarni
589*14b24e2bSVaishali KulkarniWhen working in INTA / MSI we work in single-ISR multiple-DPC mode; The same interrupt line can signify interrupts from many possible status blocks. In this case the information of which status block generated an interrupt needs to be read from a register in the IGU. Use \myfunc{int\_igu\_read\_sisr\_reg}{int_igu_read_sisr_reg} to get the information [returned value is a bitmask of status blocks which asserted the interrupt].
590*14b24e2bSVaishali Kulkarni
591*14b24e2bSVaishali Kulkarni%\insertcode{snippets/ecore_int_sisr.h}{INTA mechanism for reading interrupt source}
592*14b24e2bSVaishali Kulkarni
593*14b24e2bSVaishali Kulkarni\section{IGU block operation}
594*14b24e2bSVaishali KulkarniThe IGU block has a mapping of status blocks to interrupts.
595*14b24e2bSVaishali KulkarniThe mapping is done inside the IGU CAM and maps a (function, vector) pair to an MSI-X message.
596*14b24e2bSVaishali KulkarniIn case of INTA / MSI, each function has a register in the IGU stating which status block gave the interrupt.
597*14b24e2bSVaishali KulkarniThe IGU block is responsible for generating the interrupt. It receives the command to generate an interrupt from the CAU block.
598*14b24e2bSVaishali KulkarniThe IGU block maintains producer-consumer pairs per status block.
599*14b24e2bSVaishali KulkarniThe CAU updates the producer after it wrote the status block to host memory.
600*14b24e2bSVaishali KulkarniThe driver updates the consumer after it finished processing the status block.
601*14b24e2bSVaishali KulkarniThe IGU block generates an interrupt when there is a prod-cons difference on the status block.
602*14b24e2bSVaishali Kulkarni
603*14b24e2bSVaishali KulkarniCAU also handles coalescing of status block writes and interrupt generation.
604*14b24e2bSVaishali KulkarniThe CAU unit may aggregate several events and generate a single update of the status block and a single interrupt, in order to lower the number of interrupts sent to host CPU.
605*14b24e2bSVaishali Kulkarni
606*14b24e2bSVaishali Kulkarni\section{Interrupt handling flow}
607*14b24e2bSVaishali Kulkarni\label{sec:sb-flow}
608*14b24e2bSVaishali KulkarniThe flow of handling an interrupt in the device and driver is as follows:
609*14b24e2bSVaishali Kulkarni\silentfunc{sb_update_sb_idx}
610*14b24e2bSVaishali Kulkarni\silentfunc{sb_ack}
611*14b24e2bSVaishali Kulkarni\begin{enumerate}
612*14b24e2bSVaishali Kulkarni	\item The device (Firmware/CAU) updates a status block index.
613*14b24e2bSVaishali Kulkarni
614*14b24e2bSVaishali Kulkarni	\item The device copies the status block to host memory and generates an interrupt.
615*14b24e2bSVaishali Kulkarni
616*14b24e2bSVaishali Kulkarni	\item OS is triggered, calling the driver's Interrupt Service Routine [ISR].
617*14b24e2bSVaishali Kulkarni
618*14b24e2bSVaishali Kulkarni	\item (Possible upper-half handling and bottom-half scheduling, or other OS-specifics which are outside the scope of this document).
619*14b24e2bSVaishali Kulkarni
620*14b24e2bSVaishali Kulkarni	\item Driver identifies a producer update on the status block (as the producer is written as part of the status block on host memory) using \myfunc{sb\_update\_sb\_idx}{sb_update_sb_idx}.
621*14b24e2bSVaishali Kulkarni
622*14b24e2bSVaishali Kulkarni	\item Driver scans the protocol indices in the status block to determine the interrupt source.
623*14b24e2bSVaishali Kulkarni	\begin{NOTICE}
624*14b24e2bSVaishali Kulkarni		It's likely the upper-layer doesn't really need to scan the status block, but rather compare values in some previous-supplied addresses against a shadow copy. E.g., In L2 the ecore callbacks configuring the queues will return the addresses which upper-layer should test for producer updates. See section [\ref{sec:l2-start}].
625*14b24e2bSVaishali Kulkarni	\end{NOTICE}
626*14b24e2bSVaishali Kulkarni
627*14b24e2bSVaishali Kulkarni	\item When Driver completes processing all the indices on the status block, it writes the producer value from the status block into the IGU consumer address, using \myfunc{sb\_ack}{sb_ack}.
628*14b24e2bSVaishali Kulkarni
629*14b24e2bSVaishali Kulkarni	\item The IGU compares the producer and consumer -- if they differ it will generate an additional interrupt.
630*14b24e2bSVaishali Kulkarni
631*14b24e2bSVaishali Kulkarni\end{enumerate}
632*14b24e2bSVaishali Kulkarni
633*14b24e2bSVaishali Kulkarni\begin{exampleT}
634*14b24e2bSVaishali Kulkarni	Assume an Rx packet is received by device. After FW places the packet in the Rx rings, it updates the status block of that Rx ring; This in turn is copied into host memory and an MSI-X interrupt for the appropriate Rx queue's status block is triggered.
635*14b24e2bSVaishali Kulkarni	Driver reads the status blocks, scanning the indicies and identifies the interrupt is an Rx CQE consumer and handles the incoming packet. Assuming this is the only interrupt source [and there was also a single packet] driver than acks the status block.
636*14b24e2bSVaishali Kulkarni\end{exampleT}
637*14b24e2bSVaishali Kulkarni\SpillChapterFuncs
638*14b24e2bSVaishali Kulkarni
639*14b24e2bSVaishali Kulkarni
640*14b24e2bSVaishali Kulkarni\chapterimage{qlogic-full-36}
641*14b24e2bSVaishali Kulkarni\chapter{Management firmware [MFW] interface}
642*14b24e2bSVaishali Kulkarni\label{cha:mfw}
643*14b24e2bSVaishali Kulkarni
644*14b24e2bSVaishali KulkarniThe management firmware runs on its own processor on the chip [\myindex{MCP}] and has many responsibilities – it serves as the entity initially configuring the chip [during bios phase], answering the various management protocols, synchronizing between PFs, configuring the physical link, etc.
645*14b24e2bSVaishali KulkarniHW functions and the \myindex{MFW} may interact with each other in both ways – driver may send messages to the MFW in the form of commands on a buffer, while the MFW generates attentions for the driver and posts messages in a designated mailbox in the SHMEM. The implementation of the interface resides in \texttt{ecore\_mcp.[ch]}, with the addition of \texttt{.h} files generated by the MFW owners, e.g., \texttt{mcp\_public.h} which contains the SHMEM structure and the list of commands.
646*14b24e2bSVaishali KulkarniThe API that should be included by upper-layer driver is defined in \texttt{ecore\_mcp\_api.h}.
647*14b24e2bSVaishali Kulkarni
648*14b24e2bSVaishali KulkarniThe interface between driver and MFW is initialized as early as possible in the initial initialization flow [specifically as part of \texttt{ecore\_hw\_prepare()}],  as this initializes the Driver access to SHMEM which is used later during initialization to learn about the chip configuration [which was read from NVRAM by MFW and written into SHMEM].
649*14b24e2bSVaishali KulkarniThe upper layer doesn’t need to take care of allocating/releasing of this interface – it’s part of the greater initialization/de-initialization of the ecore.
650*14b24e2bSVaishali Kulkarni
651*14b24e2bSVaishali Kulkarni\section{Shared Memory [SHMEM]}
652*14b24e2bSVaishali KulkarniThe \myindex{shared memory} is a segment of memory accessible to all functions as well as the MFW. The memory is used for various purposes:
653*14b24e2bSVaishali Kulkarni\begin{enumerate}
654*14b24e2bSVaishali Kulkarni	\item MFW fills it with current HW configuration, either based on the default found in the NVRAM or based on some management-protocol [e.g., it’s possible vlans configuration is determined by switch and communicated to the MFW]. Driver reads those values and decides upon its logical state/configures HW appropriately. \\
655*14b24e2bSVaishali Kulkarni
656*14b24e2bSVaishali Kulkarni	\item The driver--MFW interface is based on mailboxes in well-known addresses in the SHMEM. \\
657*14b24e2bSVaishali Kulkarni
658*14b24e2bSVaishali Kulkarni	\item It’s possible [as in E3] that there will be driver-held information that will be requested by some management-protocol, and the driver will have to fill it in some well-known address in the SHMEM.
659*14b24e2bSVaishali Kulkarni\end{enumerate}
660*14b24e2bSVaishali Kulkarni
661*14b24e2bSVaishali KulkarniAn upper-layer driver is not supposed to access the SHMEM directly; It should only do so by using ecore functions and accessing ecore structs. The ecore \textit{mcp\_info} struct contains as one of its fields \textit{func\_info} which is filled by the ecore during early device initialization with all the function-specific static\footnote{i.e., data that shouldn't change while driver is running} data. Upper-layer driver can read those values for its own usage.
662*14b24e2bSVaishali Kulkarni
663*14b24e2bSVaishali Kulkarni\section{Ecore - MFW interface}
664*14b24e2bSVaishali Kulkarni\begin{itemize}
665*14b24e2bSVaishali Kulkarni	\item Sending messages from driver to MFW -- Each HW-function has an address in the SHMEM in which the MFW will poll for messages from that HW-function.
666*14b24e2bSVaishali Kulkarni	A message is a u32 consisting of a command bit-mask which indicates of the message the HW-functions sends and a cyclic sequential number.
667*14b24e2bSVaishali Kulkarni	In addition there’s another u32 field which might contain additional parameters [command-specific].
668*14b24e2bSVaishali Kulkarni	The driver increases the sequence number and writes the message and then polls until the MFW writes its response [with the correct sequence number] to another known address in SHMEM\footnote{Obviously, this is a one-pending mechanism.}
669*14b24e2bSVaishali Kulkarni	The MFW can also send an additional parameter [command-specific]. \\
670*14b24e2bSVaishali Kulkarni
671*14b24e2bSVaishali Kulkarni	\item Messages from MFW to driver -- MFW will trigger a general HW attention which will be handled by the specific HW-function [there’s a different general HW attention per HW-function].
672*14b24e2bSVaishali Kulkarni	Per-HW-function there’s an array of message producers in SHMEM,  of which the ecore maintains a copy.
673*14b24e2bSVaishali Kulkarni	Before sending the attention, the MFW will increment the producer of the message it wishes to inform the driver and the driver will recognize the message by noticing the difference in producers.
674*14b24e2bSVaishali Kulkarni	After handling said message, the driver will ack the message by writing the new producer back to SHMEM and disabling the general HW attention.
675*14b24e2bSVaishali Kulkarni	Notice it's [at least theoretically] possible for the ecore to encounter multiple MFW messages following a single attention from HW. \\
676*14b24e2bSVaishali Kulkarni\end{itemize}
677*14b24e2bSVaishali Kulkarni
678*14b24e2bSVaishali KulkarniNotice the commands’ content vary -- some of the commands will require additional parameters to be filled in specific fields in the SHMEM before the commands are passed.
679*14b24e2bSVaishali Kulkarni
680*14b24e2bSVaishali Kulkarni\section{API between ecore's MCP interface and upper-layer driver}
681*14b24e2bSVaishali Kulkarni\myfunc{mcp\_cmd}{mcp_cmd} --  this is the very core of message-passing from driver to MFW. Upper-layer driver should pass the command (FW\_MSG\_CODE\_* from \texttt{mcp\_public.h}) and a parameter, as well as pointers for the MFW response and additional possible parameter. The function will pass the command for MFW and await [sleep] for its reply. \\
682*14b24e2bSVaishali Kulkarni
683*14b24e2bSVaishali KulkarniA ‘special’ instance of this function is \texttt{ecore\_mcp\_load\_req()} [which isn’t an API function] - that function sends an indication to the MCP that the HW-function is being loaded.
684*14b24e2bSVaishali KulkarniThe MFW is used as both a book-keeper and synchronization mechanism for the loading of PFs, as there are communal resources. The response will be (FW\_MSG\_CODE\_DRV\_LOAD\_<X>), where X can be either ENGINE, PORT or FUNCTION:
685*14b24e2bSVaishali Kulkarni\begin{itemize}
686*14b24e2bSVaishali Kulkarni	\item Engine – HW-function is the first being loaded on its engine.
687*14b24e2bSVaishali Kulkarni	\item Port – Another HW-function has already initialized the engine, but this HW-function is first on its port.
688*14b24e2bSVaishali Kulkarni	\item Function – Another HW-function has already initialized the port.
689*14b24e2bSVaishali Kulkarni\end{itemize}
690*14b24e2bSVaishali KulkarniAccording to the MFW response the ecore knows what need to be initialized. \\
691*14b24e2bSVaishali Kulkarni
692*14b24e2bSVaishali Kulkarni\texttt{ecore\_handle\_mcp\_events()} – This function is called from the slowpath interrupt context [sleepless] upon MFW attention to the driver.
693*14b24e2bSVaishali KulkarniDependent on the exact message received from the MFW, it’s possible that this will eventually will call some OSAL which will need to be implemented by the upper-layer driver, e.g., in case of link change indication [The upper-layer needs to be notified and should decide on its own what to do with that information].
694*14b24e2bSVaishali Kulkarni
695*14b24e2bSVaishali Kulkarni\section{Link Interface}
696*14b24e2bSVaishali Kulkarni\label{sec:mfw-link}
697*14b24e2bSVaishali KulkarniThe MFW is responsible for configuring the physical link [i.e., MAC, PHY, etc.]. The ecore encapsulates the entire interface with MFW for configuring the link, leaving a relatively narrow API with the upper-layer driver.
698*14b24e2bSVaishali KulkarniThe ecore HW-function contains 2 related strctures –
699*14b24e2bSVaishali Kulkarni\silentfunc{mcp_get_link_params}
700*14b24e2bSVaishali Kulkarni\silentfunc{mcp_get_link_state}
701*14b24e2bSVaishali Kulkarni\begin{itemize}
702*14b24e2bSVaishali Kulkarni	\item Link\_params – The ecore uses this as inputs for configuring the link; According to the values in this struct, the ecore will later configure shmem in the appropriate places so that once the MFW receives the command to set the link it will use this configuratio.
703*14b24e2bSVaishali Kulkarni	During ecore initialization, the ecore will fill this structure with the default values from SHMEM [values set by MFW according to NVRAM configuration]
704*14b24e2bSVaishali KulkarniWhen upper-layer driver wishes to update link configuration, it should change this struct.
705*14b24e2bSVaishali KulkarniIt can access it by calling \myfunc{mcp\_get\_link\_params}{mcp_get_link_params} \\
706*14b24e2bSVaishali Kulkarni
707*14b24e2bSVaishali Kulkarni	\item Link\_output – The ecore fills the structure from attention handling context whenever the MFW indicates that a link change has occurred. Upper layer driver can read this to get information about the current state of the physical link. It can access this struct by calling \myfunc{mcp\_get\_link\_state}{mcp_get_link_state}.\\
708*14b24e2bSVaishali Kulkarni\end{itemize}
709*14b24e2bSVaishali Kulkarni
710*14b24e2bSVaishali KulkarniIn order to work with the ecore link interface, upper driver needs to implement an OSAL [\texttt{osal\_link\_update()}] which will be called whenever the link state has changed – this will notify the upper driver that the link has changed and that it should probably read link\_output and act upon it. \\
711*14b24e2bSVaishali Kulkarni
712*14b24e2bSVaishali KulkarniIn order to set/reset the link, the upper driver should call \myfunc{mcp\_set\_link}{mcp_set_link} after overriding the link\_params fields with its required link configured [optional, as without doing anything the structure will contain the default link configuration found in SHMEM].
713*14b24e2bSVaishali KulkarniPassing true will cause MFW to try setting the link [either by force or via auto-negotiation, based on the configuration], while passing false will cause the MFW to reset the link.
714*14b24e2bSVaishali Kulkarni
715*14b24e2bSVaishali KulkarniNotice the logic for link-flap-avoidance should be contained in MFW, e.g., in multi-function mode there’s no need for the upper-layer driver to count the number of functions loaded in order to decide whether during unload it should request a link reset; It should do it regardless.
716*14b24e2bSVaishali KulkarniIt’s the MFW's duty to decide whether the unloading function is actually the last loaded function on its port and thus whether to actually reset the link.
717*14b24e2bSVaishali Kulkarni
718*14b24e2bSVaishali Kulkarni\subsection{Energy Efficient Ethernet (EEE)}
719*14b24e2bSVaishali KulkarniEEE feature enables the device to put its transistors in sleep mode when there is no data activity on the wire. Hence achieves the significant reduction in the power consumption of the device. It's a Base-T feature, more details of which are captured under IEEE 802.3az standard. MFW negotiates the EEE parameters with the peer device and the results will be shared to the ecore as part of link notification. Following are the negotiated parameters which will be encapsulated in the struct \texttt{ecore\_mcp\_link\_state}.
720*14b24e2bSVaishali Kulkarni\begin{itemize}
721*14b24e2bSVaishali Kulkarni	\item eee\_active – EEE is negotiated and is currently operational.
722*14b24e2bSVaishali Kulkarni	\item eee\_adv\_caps – Device advertized capabilities.
723*14b24e2bSVaishali Kulkarni	\item eee\_lpi\_adv\_caps – Peer device advertized capabilities.
724*14b24e2bSVaishali Kulkarni\end{itemize}
725*14b24e2bSVaishali KulkarniFollowing are the EEE link parameters which can be queried by upper layer driver using \myfunc{mcp\_get\_link\_params}{mcp_get_link_params} API.
726*14b24e2bSVaishali Kulkarni\begin{itemize}
727*14b24e2bSVaishali Kulkarni	\item eee\_enable – EEE is enabled.
728*14b24e2bSVaishali Kulkarni	\item eee\_supported – Device supports EEE.
729*14b24e2bSVaishali Kulkarni	\item eee\_tx\_lpi\_enable – Determines whether the device should assert its Tx LPI.
730*14b24e2bSVaishali Kulkarni	\item eee\_tx\_lpi\_timer – EEE delay timer value, i.e., amount of time device should stay in idle mode prior to asserting its Tx LPI  (in  microseconds).
731*14b24e2bSVaishali Kulkarni\end{itemize}
732*14b24e2bSVaishali KulkarniUpper layer driver can configure the one or more of the EEE following parameters.
733*14b24e2bSVaishali Kulkarni\begin{itemize}
734*14b24e2bSVaishali Kulkarni	\item eee\_enable
735*14b24e2bSVaishali Kulkarni	\item eee\_adv\_caps
736*14b24e2bSVaishali Kulkarni	\item eee\_tx\_lpi\_enable
737*14b24e2bSVaishali Kulkarni	\item eee\_tx\_lpi\_timer
738*14b24e2bSVaishali Kulkarni\end{itemize}
739*14b24e2bSVaishali Kulkarni
740*14b24e2bSVaishali Kulkarni\section{Dcbx Interface}
741*14b24e2bSVaishali Kulkarni\label{sec:mfw-dcbx}
742*14b24e2bSVaishali KulkarniThe MFW is responsible for negotiating the dcbx parameters [e.g., per priority flow control (PFC)] with peer device. During initialization, MFW reads the dcbx parameters from NVRAM (called local parameters) and negotiates these with the peer. The negotiated/agreed parameters are called operational dcbx parameters. MFW provides driver interfaces for querying and configuring the dcbx parameters. The ecore dcbx implementation provides three APIs, one for querying the dcbx paramters and the other two for updating the dcbx configuration.
743*14b24e2bSVaishali Kulkarni\silentfunc{dcbx_query_params}
744*14b24e2bSVaishali Kulkarni\silentfunc{dcbx_get_config_params}
745*14b24e2bSVaishali Kulkarni\silentfunc{dcbx_config_params}
746*14b24e2bSVaishali Kulkarni\begin{itemize}
747*14b24e2bSVaishali Kulkarni	\item \myfunc{dcbx\_query\_params}{dcbx\_query\_params} – The API returns the current dcbx configuration. It expects type (i.e., local/remote/operational) and the buffer for storing the dcbx parameters of that type.\\
748*14b24e2bSVaishali Kulkarni
749*14b24e2bSVaishali Kulkarni	\item \myfunc{dcbx\_get\_config\_params}{dcbx\_get\_config\_params} - The API returns the currently cached dcbx parameter set that can be modified for making the dcbx update requests. \\
750*14b24e2bSVaishali Kulkarni
751*14b24e2bSVaishali Kulkarni	\item \myfunc{dcbx\_config\_params}{dcbx\_config\_params} – The API is used for sending the dcbx parameters update request. The API expects dcbx parameters to be configured and the flag specifying whether the parameters need to be sent to hardware or just cache at the ecore. When driver sends dcbx config to the hardware, device initiates the dcbx negotiation with the peer using lldp protocol. The negotiation takes few seconds to complete, and also the lldp requests are rate limited (using a predefined credit value). The dcbx API option “hw\_commit” specifies whether the dcbx parameters need to be committed to the hardware or just cache at the driver. When client requests the commit, all the cached parameters are sent to the device and the parameter negotiation will be initiated with the peer. \\
752*14b24e2bSVaishali Kulkarni\end{itemize}
753*14b24e2bSVaishali KulkarniThe steps for configuring the dcbx parameters are, upper layer driver invokes ecore\_dcbx\_get\_config\_params() API to get the current config parameter set, and update the required parameters, and then invoke ecore\_dcbx\_config\_params() API.
754*14b24e2bSVaishali Kulkarni
755*14b24e2bSVaishali KulkarniIf there is any change in the dcbx configuration at the host (for example due to a negotiation with the peer), then MFW notifies the same to ecore. OSAL\_DCBX\_AEN() would be called after such notification, ecore client would need to provide the implementation for this OSAL.
756*14b24e2bSVaishali Kulkarni
757*14b24e2bSVaishali Kulkarni\section{Management protocol APIs}
758*14b24e2bSVaishali Kulkarni\label{sec:mfw-protocols}
759*14b24e2bSVaishali KulkarniMFW needs various bits of information from the driver, and it gathers those in one of two methods:
760*14b24e2bSVaishali Kulkarni\begin{itemize}
761*14b24e2bSVaishali Kulkarni	\item Pulling – if ecore can’t provide information on its own, ecore-client would be required to implement an OSAL.\\
762*14b24e2bSVaishali Kulkarni	\item Pushing – it’s the ecore and ecore-client’s responsibility to push the data.\\
763*14b24e2bSVaishali Kulkarni\end{itemize}
764*14b24e2bSVaishali KulkarniIn some cases, ‘Push’ is done without involvement of the ecore-client. If that’s not possible, it becomes more risky as the responsibility of doing things correctly passes to the ecore-client. Ecore-client shouldn’t presume to do ‘push’ only for calls which match the configured management mode. Instead it should always do them and let the ecore be the arbiter of whether those are needed by MFW or not. Ecore provides the following APIs for updating the configuration attributes, it is the client's responsibility to invoke these APIs at the appropriate time.
765*14b24e2bSVaishali Kulkarni\silentfunc{mcp_ov_update_current_config}
766*14b24e2bSVaishali Kulkarni\silentfunc{mcp_ov_update_mtu}
767*14b24e2bSVaishali Kulkarni\silentfunc{mcp_ov_update_mac}
768*14b24e2bSVaishali Kulkarni\silentfunc{mcp_ov_update_wol}
769*14b24e2bSVaishali Kulkarni\silentfunc{mcp_ov_update_driver_state}
770*14b24e2bSVaishali Kulkarni\silentfunc{mcp_update_fcoe_cvid}
771*14b24e2bSVaishali Kulkarni\silentfunc{mcp_update_fcoe_fabric_name}
772*14b24e2bSVaishali Kulkarni\begin{itemize}
773*14b24e2bSVaishali Kulkarni	\item \myfunc{mcp\_ov\_update\_current\_config}{mcp\_ov\_update\_current\_config} – Drivers need to call this API when user updates one (or more) of the following: mtu, primary mac or Wake on LAN settings (to a non-default value). In addition, it also needs to call a unique API per each:
774*14b24e2bSVaishali Kulkarni	\begin{itemize}
775*14b24e2bSVaishali Kulkarni		\item \myfunc{mcp\_ov\_update\_mtu}{mcp\_ov\_update\_mtu} – called when user sets the mtu to a value other than the default provided by the ecore.\\
776*14b24e2bSVaishali Kulkarni
777*14b24e2bSVaishali Kulkarni		\item \myfunc{mcp\_ov\_update\_mac}{mcp\_ov\_update\_mac} – called when user updates the primary mac address.\\
778*14b24e2bSVaishali Kulkarni
779*14b24e2bSVaishali Kulkarni		\item \myfunc{mcp\_ov\_update\_wol}{mcp\_ov\_update\_wol} – called when Wake-on-LAN settings are updated.\\
780*14b24e2bSVaishali Kulkarni	\end{itemize}
781*14b24e2bSVaishali Kulkarni	\item \myfunc{mcp\_ov\_update\_driver\_state}{mcp\_ov\_update\_driver\_state} – notify about a change in the driver state. Following are the possible driver states,
782*14b24e2bSVaishali Kulkarni	\begin{itemize}
783*14b24e2bSVaishali Kulkarni		\item ECORE\_OV\_DRIVER\_STATE\_NOT\_LOADED - Firmware is not loaded.\\
784*14b24e2bSVaishali Kulkarni
785*14b24e2bSVaishali Kulkarni		\item ECORE\_OV\_DRIVER\_STATE\_DISABLED - Driver is not ready yet.\\
786*14b24e2bSVaishali Kulkarni
787*14b24e2bSVaishali Kulkarni		\item ECORE\_OV\_DRIVER\_STATE\_ACTIVE - Driver is operational.\\
788*14b24e2bSVaishali Kulkarni	\end{itemize}
789*14b24e2bSVaishali Kulkarni	Ecore sets the following driver states,
790*14b24e2bSVaishali Kulkarni	\begin{itemize}
791*14b24e2bSVaishali Kulkarni		\item DISABLED - After firmware is successfully loaded on the device, ecore updates the driver state as DISABLED (as part of ecore\_hw\_init() implementation). \\
792*14b24e2bSVaishali Kulkarni		\item NOT\_LOADED - Ecore sets this state when the protocol driver is unloaded (as part of ecore\_hw\_remove()).\\
793*14b24e2bSVaishali Kulkarni	\end{itemize}
794*14b24e2bSVaishali Kulkarni	It's the protocol driver's responsibility to alternate between the states,
795*14b24e2bSVaishali Kulkarni	\begin{itemize}
796*14b24e2bSVaishali Kulkarni		\item ACTIVE - Set when the required initialization is done from the driver side and the device is ready for traffic switching.\\
797*14b24e2bSVaishali Kulkarni		\item DISABLED - Set when device is not operational (e.g., fastpath queues are released or not configured).\\
798*14b24e2bSVaishali Kulkarni	\end{itemize}
799*14b24e2bSVaishali Kulkarni	\item \myfunc{mcp\_update\_fcoe\_cvid}{mcp_update_fcoe_cvid} - Update MFW with the 802.1q fcoe vlan id assigned for the PF.\\
800*14b24e2bSVaishali Kulkarni	\item \myfunc{mcp\_update\_fcoe\_fabric\_name}{mcp_update_fcoe_fabric_name} - Update fabric name value to the MFW. Fabric name is the value returned by the fabric domain controller in response to a GS-FC “Get Fabric Name” command from the adapter.\\
801*14b24e2bSVaishali Kulkarni\end{itemize}
802*14b24e2bSVaishali Kulkarni
803*14b24e2bSVaishali KulkarniEcore also provides the TLV request interface for MFW for querying the driver/device attributes. MFW uses mailbox interface to notify ecore on the required TLV information. Ecore parses the request, populates the required information with the help of ecore clients and sends it to the MFW. Ecore client need to provide necessary infrastructure and the OSALs for implementing this interface.
804*14b24e2bSVaishali Kulkarni\begin{itemize}
805*14b24e2bSVaishali Kulkarni	\item OSAL\_MFW\_TLV\_REQ - The call indicates that ecore has received a TLV request notification from the MFW. The execution context in interrupt mode, hence ecore client need to schedule a thread/bottom-half context to handle this task, and return the control immediately. The bottom-half thread will need to invoke \myfunc{mfw\_process\_tlv\_req}{mfw_process_tlv_req} for further processing of the TLV request.\\
806*14b24e2bSVaishali Kulkarni	\item OSAL\_MFW\_FILL\_TLV\_DATA - Ecore invokes this callback to get the TLV values of a given type. Ecore client need to fill in the values for all the fields that it's aware of, and also need to set the flags associated with the respective fields. For instance,  if client sets value for 'npiv\_enabled' field, it needs to set the flag 'npiv\_enabled\_set' to true.\\
807*14b24e2bSVaishali Kulkarni\end{itemize}
808*14b24e2bSVaishali Kulkarni
809*14b24e2bSVaishali Kulkarni\SpillChapterFuncs
810*14b24e2bSVaishali Kulkarni
811*14b24e2bSVaishali Kulkarni
812*14b24e2bSVaishali Kulkarni\chapterimage{qlogic-full-36}
813*14b24e2bSVaishali Kulkarni\chapter{L2 protocol}
814*14b24e2bSVaishali Kulkarni\label{cha:l2}
815*14b24e2bSVaishali Kulkarni
816*14b24e2bSVaishali Kulkarni\section{L2-related terminology}
817*14b24e2bSVaishali KulkarniThis section describes in a very highlevel manner several FW objects which are related to L2. Developers implementing L2 support over the ecore should be familiar with these\footnote{Probably even more than is in the scope of this document.}.
818*14b24e2bSVaishali Kulkarni\begin{itemize}
819*14b24e2bSVaishali Kulkarni	\item Virtual port [\myindex{VPORT}] -- Can simply be seen as a collection of queues, each HW-function will have at least one VPORT configured\footnote{And in most scenarios one will suffice.}. Classifications are configured per-VPORT. \\
820*14b24e2bSVaishali Kulkarni
821*14b24e2bSVaishali Kulkarni	\item Queues -- Either Rx/Tx, queues are attached to a VPORT. There can multiple queues per-VPORT [e.g., if RSS/TSS is supported]. Usually, each Rx queue will use it's own status block for interrupts upon Rx packets but Tx queues can utilize the same status blocks, using different protocol indices. \\
822*14b24e2bSVaishali Kulkarni\end{itemize}
823*14b24e2bSVaishali Kulkarni
824*14b24e2bSVaishali Kulkarni\section{Starting an L2 device}
825*14b24e2bSVaishali Kulkarni\label{sec:l2-start}
826*14b24e2bSVaishali KulkarniThis section begins after section \ref{sec:init-init}, I.e., assuming the HW-function has already been initialized by the init tool and the PF\_START ramrod has already been sent.
827*14b24e2bSVaishali Kulkarni
828*14b24e2bSVaishali Kulkarni	\begin{NOTICE}
829*14b24e2bSVaishali Kulkarni	Although VPORTs' and queues' indices are shared between all HW-function on the same engine, the resource allocation scheme determines a range of VPORTs per-HW-function to use for configuration [i.e., developer can assume starting index is always 0 per-HW-function].
830*14b24e2bSVaishali Kulkarni	\end{NOTICE}
831*14b24e2bSVaishali Kulkarni
832*14b24e2bSVaishali Kulkarni
833*14b24e2bSVaishali Kulkarni\silentfunc{sp_vport_start}
834*14b24e2bSVaishali Kulkarni\silentfunc{eth_rx_queue_start}
835*14b24e2bSVaishali Kulkarni\silentfunc{eth_tx_queue_start}
836*14b24e2bSVaishali Kulkarni\silentfunc{sp_vport_update}
837*14b24e2bSVaishali Kulkarni\begin{enumerate}
838*14b24e2bSVaishali Kulkarni	\item \myfunc{sp\_vport\_start}{sp_vport_start} -- this function initializes a vport in FW [ETH\_RAMROD\_VPORT\_START will be sent]. The handle for this function is a \texttt{vport\_id} which is passed and the most 'interesting' argument is the MTU for that VPORT.
839*14b24e2bSVaishali Kulkarni	This VPORT will be inactive after sending this ramrod, i.e., until enabling it via a vport update it will not actually perform Rx/Tx. \\
840*14b24e2bSVaishali Kulkarni
841*14b24e2bSVaishali Kulkarni	\item \myfunc{eth\_rx\_queue\_start}{eth_rx_queue_start} -- initializes an rx queue on a given VPORT.
842*14b24e2bSVaishali Kulkarni	A pre-request is that the VPORT has already been initialized.
843*14b24e2bSVaishali Kulkarni	There are 2 identifier of the queue - the queue index to add and the VPORT index to add it to. The queue-index should be unique for the Rx-queue; No 2 Rx-queues of the same PF should use the same id.
844*14b24e2bSVaishali Kulkarni	There are quite a few parameters that need to be supplied, e.g., status block, physical addresses of rings, etc.
845*14b24e2bSVaishali Kulkarni
846*14b24e2bSVaishali Kulkarni	The function is expected to receive a pointer to a \texttt{p\_ret\_params} which it will fill with outputs [upon success]. The ecore would fill the address where producer-updates need to be written [in the storm's RAM]; The upper-driver will write producer updates to that address to replenish its Rx-rings.
847*14b24e2bSVaishali Kulkarni
848*14b24e2bSVaishali Kulkarni	\begin{NOTICE}
849*14b24e2bSVaishali Kulkarni	Address is mapped by GTT, so upper-driver can simply write to that address, using the necessary memory barriers.
850*14b24e2bSVaishali Kulkarni	\end{NOTICE}
851*14b24e2bSVaishali Kulkarni	In addition, ecore would also fill a \texttt{p\_handle}. This handle is opaque to the ecore-client, and should be passed to other Rx-queue APIs when doing configuration relating to that queue.
852*14b24e2bSVaishali Kulkarni
853*14b24e2bSVaishali Kulkarni	After calling this function, upper-layer driver should initialize the Rx packets producers. \\
854*14b24e2bSVaishali Kulkarni
855*14b24e2bSVaishali Kulkarni	\item \myfunc{eth\_tx\_queue\_start}{eth_tx_queue_start} -- initializes a Tx queue on a given VPORT0.
856*14b24e2bSVaishali Kulkarni		Very similar to the Rx queue start method, with some slight differences in the parameters [BD ring address instead of Rx rings, etc.]. For Tx-queues, the same queue-id can be shared between 2 different queues. That would cause those queues to share the same coalescing configuration.
857*14b24e2bSVaishali Kulkarni		Just like for Rx-queues, the ecore would fill the \texttt{p\_ret\_params} with an opaque handler to be used for further calls relating to this queue. In addition, it will provide a \texttt{p\_doorbell} address, which is an address into which a doorbell needs to be written to activate firmware once a packet is placed on this Tx queue and the buffer descriptors are filled.
858*14b24e2bSVaishali Kulkarni		\begin{NOTICE}
859*14b24e2bSVaishali Kulkarni		Doorbell addresses are on a different BAR than that of other memories/registers accessed by driver, and the PTT/GTT scheme does not apply to it; Thus the address can simply be accessed using the necessary memory barriers.
860*14b24e2bSVaishali Kulkarni		\end{NOTICE}
861*14b24e2bSVaishali Kulkarni
862*14b24e2bSVaishali Kulkarni	\item \myfunc{sp\_vport\_update}{sp_vport_update} -- This is required to enable the VPORT. It should be called after the Tx/Rx queues were already added, and this will enable the VPORT to send and receive packets\footnote{Notice that without classification configuration Rx won't actually work. Also notice this function can do a lot of things; Enabling the VPORT is only one of them.}.
863*14b24e2bSVaishali Kulkarni
864*14b24e2bSVaishali Kulkarni	In order to enable the VPORT for traffic, the upper-layer driver should set in \texttt{p\_params} the following:
865*14b24e2bSVaishali Kulkarni	\begin{enumerate}
866*14b24e2bSVaishali Kulkarni		\item \texttt{update\_vport\_active\_flg} to 1.
867*14b24e2bSVaishali Kulkarni		\item \texttt{vport\_active\_flg} to 1.
868*14b24e2bSVaishali Kulkarni	\end{enumerate}
869*14b24e2bSVaishali Kulkarni
870*14b24e2bSVaishali Kulkarni\end{enumerate}
871*14b24e2bSVaishali Kulkarni
872*14b24e2bSVaishali Kulkarni\section{Configuring Classifications}
873*14b24e2bSVaishali Kulkarni\label{sec:l2-class}
874*14b24e2bSVaishali KulkarniClassification configuration consists [mostly] of three thing:
875*14b24e2bSVaishali Kulkarni\begin{enumerate}
876*14b24e2bSVaishali Kulkarni	\item Configuration of the \myindex{Rx mode} -- This defines which datagrams [unicast, multicast, broadcast] should be accepted by the VPORT, and whether all such datagrams or only if a filter is configured for them.
877*14b24e2bSVaishali Kulkarni	\item Configuration of unicast / multicast filters -- defining filters for specific unicast / multicast addresses which should be matched, given that Rx mode  agrees.
878*14b24e2bSVaishali Kulkarni	\item Configuration of vlan filters -- by default, all vlans will be accepted. If at least one vlan [or vlan-mac] filter will be configured only traffic which matches one of the configured vlan filters will pass through.
879*14b24e2bSVaishali Kulkarni\end{enumerate}