xref: /illumos-gate/usr/src/uts/common/io/ena/ena_stats.c (revision 6f443ebc)
1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * Copyright 2021 Oxide Computer Company
14  */
15 #include "ena.h"
16 
17 /*
18  * The ENA device provides the following hardware stats. It appears
19  * that all stats are available at both a device-level and
20  * queue-level. However, Linux and FreeBSD don't implement queue
21  * scope. It's not clear how one would implement queue scope because
22  * there is nothing in the common code describing how to determine the
23  * queue index number. Both the SQ and CQ have device index values,
24  * but for a given logical queue they don't always match and so it's
25  * not clear what value to use for querying the stats. Therefore,
26  * device-wide basic and extended stats come from the device, while
27  * queue/ring stats come from driver.
28  *
29  * From empirical testing, these statistics appear to be cumulative.
30  * However, this guarantee is not explicitly documented anywhere in
31  * the common code that the author could find.
32  *
33  * BASIC (ENAHW_GET_STATS_TYPE_BASIC)
34  *
35  *     - Rx packets/bytes
36  *     - Rx drops
37  *     - Tx packets/bytes
38  *     - Tx drops
39  *
40  * EXTENDED (ENAHW_GET_STATS_TYPE_EXTENDED)
41  *
42  *     There is no structure defined for these stats in the Linux
43  *     driver. Based on the FreeBSD driver, it looks like extended
44  *     stats are simply a buffer of C strings? Come back to this
45  *     later.
46  *
47  * ENI (ENAHW_GET_STATS_TYPE_ENI)
48  *
49  *     - Rx Bandwidth Allowance Exceeded
50  *     - Tx Bandwidth Allowance Exceeded
51  *     - PPS Allowance Exceeded (presumably for combined Rx/Tx)
52  *     - Connection Tracking PPS Allowance Exceeded
53  *     - Link-local PPS Alloance Exceeded
54  */
55 
56 static int
ena_stat_device_basic_update(kstat_t * ksp,int rw)57 ena_stat_device_basic_update(kstat_t *ksp, int rw)
58 {
59 	ena_t *ena = ksp->ks_private;
60 	ena_basic_stat_t *ebs = ksp->ks_data;
61 	enahw_resp_desc_t resp;
62 	enahw_resp_basic_stats_t *stats = &resp.erd_resp.erd_basic_stats;
63 	int ret = 0;
64 
65 	if (rw == KSTAT_WRITE) {
66 		return (EACCES);
67 	}
68 
69 	if ((ret = ena_admin_get_basic_stats(ena, &resp)) != 0) {
70 		return (ret);
71 	}
72 
73 	mutex_enter(&ena->ena_lock);
74 
75 	ebs->ebs_tx_bytes.value.ui64 =
76 	    ((uint64_t)stats->erbs_tx_bytes_high << 32) |
77 	    (uint64_t)stats->erbs_tx_bytes_low;
78 	ebs->ebs_tx_pkts.value.ui64 =
79 	    ((uint64_t)stats->erbs_tx_pkts_high << 32) |
80 	    (uint64_t)stats->erbs_tx_pkts_low;
81 	ebs->ebs_tx_drops.value.ui64 =
82 	    ((uint64_t)stats->erbs_tx_drops_high << 32) |
83 	    (uint64_t)stats->erbs_tx_drops_low;
84 
85 	ebs->ebs_rx_bytes.value.ui64 =
86 	    ((uint64_t)stats->erbs_rx_bytes_high << 32) |
87 	    (uint64_t)stats->erbs_rx_bytes_low;
88 	ebs->ebs_rx_pkts.value.ui64 =
89 	    ((uint64_t)stats->erbs_rx_pkts_high << 32) |
90 	    (uint64_t)stats->erbs_rx_pkts_low;
91 	ebs->ebs_rx_drops.value.ui64 =
92 	    ((uint64_t)stats->erbs_rx_drops_high << 32) |
93 	    (uint64_t)stats->erbs_rx_drops_low;
94 
95 	mutex_exit(&ena->ena_lock);
96 
97 	return (0);
98 }
99 
100 void
ena_stat_device_basic_cleanup(ena_t * ena)101 ena_stat_device_basic_cleanup(ena_t *ena)
102 {
103 	if (ena->ena_device_basic_kstat != NULL) {
104 		kstat_delete(ena->ena_device_basic_kstat);
105 		ena->ena_device_basic_kstat = NULL;
106 	}
107 }
108 
109 boolean_t
ena_stat_device_basic_init(ena_t * ena)110 ena_stat_device_basic_init(ena_t *ena)
111 {
112 	kstat_t *ksp = kstat_create(ENA_MODULE_NAME,
113 	    ddi_get_instance(ena->ena_dip), "device_basic", "net",
114 	    KSTAT_TYPE_NAMED,
115 	    sizeof (ena_basic_stat_t) / sizeof (kstat_named_t), 0);
116 	ena_basic_stat_t *ebs = NULL;
117 
118 	if (ksp == NULL) {
119 		ena_err(ena, "!failed to create device_basic kstats");
120 		return (B_FALSE);
121 	}
122 
123 	ena->ena_device_basic_kstat = ksp;
124 	ebs = ksp->ks_data;
125 	ksp->ks_update = ena_stat_device_basic_update;
126 	ksp->ks_private = ena;
127 
128 	kstat_named_init(&ebs->ebs_tx_bytes, "tx_bytes", KSTAT_DATA_UINT64);
129 	ebs->ebs_tx_bytes.value.ui64 = 0;
130 	kstat_named_init(&ebs->ebs_tx_pkts, "tx_packets", KSTAT_DATA_UINT64);
131 	ebs->ebs_tx_pkts.value.ui64 = 0;
132 	kstat_named_init(&ebs->ebs_tx_drops, "tx_drops", KSTAT_DATA_UINT64);
133 	ebs->ebs_tx_drops.value.ui64 = 0;
134 
135 	kstat_named_init(&ebs->ebs_rx_bytes, "rx_bytes", KSTAT_DATA_UINT64);
136 	ebs->ebs_rx_bytes.value.ui64 = 0;
137 	kstat_named_init(&ebs->ebs_rx_pkts, "rx_packets", KSTAT_DATA_UINT64);
138 	ebs->ebs_rx_pkts.value.ui64 = 0;
139 	kstat_named_init(&ebs->ebs_rx_drops, "rx_drops", KSTAT_DATA_UINT64);
140 	ebs->ebs_rx_drops.value.ui64 = 0;
141 
142 	kstat_install(ena->ena_device_basic_kstat);
143 	return (B_TRUE);
144 }
145 
146 int
ena_stat_device_extended_update(kstat_t * ksp,int rw)147 ena_stat_device_extended_update(kstat_t *ksp, int rw)
148 {
149 	ena_t *ena = ksp->ks_private;
150 	ena_extended_stat_t *ees = ksp->ks_data;
151 	enahw_resp_desc_t resp;
152 	enahw_resp_eni_stats_t *stats = &resp.erd_resp.erd_eni_stats;
153 	int ret = 0;
154 
155 	if (rw == KSTAT_WRITE) {
156 		return (EACCES);
157 	}
158 
159 	if ((ret = ena_admin_get_eni_stats(ena, &resp)) != 0) {
160 		return (ret);
161 	}
162 
163 	mutex_enter(&ena->ena_lock);
164 
165 	ees->ees_bw_in_exceeded.value.ui64 = stats->eres_bw_in_exceeded;
166 	ees->ees_bw_out_exceeded.value.ui64 = stats->eres_bw_out_exceeded;
167 	ees->ees_pps_exceeded.value.ui64 = stats->eres_pps_exceeded;
168 	ees->ees_conns_exceeded.value.ui64 = stats->eres_conns_exceeded;
169 	ees->ees_linklocal_exceeded.value.ui64 = stats->eres_linklocal_exceeded;
170 
171 	mutex_exit(&ena->ena_lock);
172 
173 	return (0);
174 }
175 
176 void
ena_stat_device_extended_cleanup(ena_t * ena)177 ena_stat_device_extended_cleanup(ena_t *ena)
178 {
179 	if (ena->ena_device_extended_kstat != NULL) {
180 		kstat_delete(ena->ena_device_extended_kstat);
181 		ena->ena_device_extended_kstat = NULL;
182 	}
183 }
184 
185 boolean_t
ena_stat_device_extended_init(ena_t * ena)186 ena_stat_device_extended_init(ena_t *ena)
187 {
188 	kstat_t *ksp = kstat_create(ENA_MODULE_NAME,
189 	    ddi_get_instance(ena->ena_dip), "device_ext", "net",
190 	    KSTAT_TYPE_NAMED,
191 	    sizeof (ena_extended_stat_t) / sizeof (kstat_named_t), 0);
192 	ena_extended_stat_t *ees;
193 
194 	if (ksp == NULL) {
195 		ena_err(ena, "!failed to create device_ext kstats");
196 		return (B_FALSE);
197 	}
198 
199 	ena->ena_device_extended_kstat = ksp;
200 	ees = ksp->ks_data;
201 	ksp->ks_update = ena_stat_device_extended_update;
202 	ksp->ks_private = ena;
203 
204 	kstat_named_init(&ees->ees_bw_in_exceeded, "bw_in_exceeded",
205 	    KSTAT_DATA_UINT64);
206 	ees->ees_bw_in_exceeded.value.ui64 = 0;
207 
208 	kstat_named_init(&ees->ees_bw_out_exceeded, "bw_out_exceeded",
209 	    KSTAT_DATA_UINT64);
210 	ees->ees_bw_out_exceeded.value.ui64 = 0;
211 
212 	kstat_named_init(&ees->ees_pps_exceeded, "pps_exceeded",
213 	    KSTAT_DATA_UINT64);
214 	ees->ees_pps_exceeded.value.ui64 = 0;
215 
216 	kstat_named_init(&ees->ees_conns_exceeded, "conns_exceeded",
217 	    KSTAT_DATA_UINT64);
218 	ees->ees_conns_exceeded.value.ui64 = 0;
219 
220 	kstat_named_init(&ees->ees_linklocal_exceeded, "linklocal_exceeded",
221 	    KSTAT_DATA_UINT64);
222 	ees->ees_linklocal_exceeded.value.ui64 = 0;
223 
224 	kstat_install(ena->ena_device_extended_kstat);
225 	return (B_TRUE);
226 }
227 
228 void
ena_stat_aenq_cleanup(ena_t * ena)229 ena_stat_aenq_cleanup(ena_t *ena)
230 {
231 	if (ena->ena_aenq_kstat != NULL) {
232 		kstat_delete(ena->ena_aenq_kstat);
233 		ena->ena_aenq_kstat = NULL;
234 	}
235 }
236 
237 boolean_t
ena_stat_aenq_init(ena_t * ena)238 ena_stat_aenq_init(ena_t *ena)
239 {
240 	kstat_t *ksp = kstat_create(ENA_MODULE_NAME,
241 	    ddi_get_instance(ena->ena_dip), "aenq", "net", KSTAT_TYPE_NAMED,
242 	    sizeof (ena_aenq_stat_t) / sizeof (kstat_named_t),
243 	    KSTAT_FLAG_VIRTUAL);
244 	ena_aenq_stat_t *eas = &ena->ena_aenq_stat;
245 
246 	if (ksp == NULL) {
247 		ena_err(ena, "!failed to create aenq kstats");
248 		return (B_FALSE);
249 	}
250 
251 	ena->ena_aenq_kstat = ksp;
252 	ksp->ks_data = eas;
253 
254 	kstat_named_init(&eas->eaes_default, "default", KSTAT_DATA_UINT64);
255 	eas->eaes_default.value.ui64 = 0;
256 
257 	kstat_named_init(&eas->eaes_link_change, "link_change",
258 	    KSTAT_DATA_UINT64);
259 	eas->eaes_link_change.value.ui64 = 0;
260 
261 	kstat_install(ena->ena_aenq_kstat);
262 	return (B_TRUE);
263 }
264 
265 void
ena_stat_txq_cleanup(ena_txq_t * txq)266 ena_stat_txq_cleanup(ena_txq_t *txq)
267 {
268 	if (txq->et_kstat != NULL) {
269 		kstat_delete(txq->et_kstat);
270 		txq->et_kstat = NULL;
271 	}
272 }
273 
274 boolean_t
ena_stat_txq_init(ena_txq_t * txq)275 ena_stat_txq_init(ena_txq_t *txq)
276 {
277 	ena_t *ena = txq->et_ena;
278 	kstat_t *ksp;
279 	char buf[128];
280 	ena_txq_stat_t *ets = &txq->et_stat;
281 
282 	(void) snprintf(buf, sizeof (buf), "txq_%d", txq->et_txqs_idx);
283 
284 	ksp = kstat_create(ENA_MODULE_NAME, ddi_get_instance(ena->ena_dip), buf,
285 	    "net", KSTAT_TYPE_NAMED,
286 	    sizeof (ena_txq_stat_t) / sizeof (kstat_named_t),
287 	    KSTAT_FLAG_VIRTUAL);
288 
289 	if (ksp == NULL) {
290 		ena_err(ena, "!failed to create %s kstats", buf);
291 		return (B_FALSE);
292 	}
293 
294 	txq->et_kstat = ksp;
295 	ksp->ks_data = ets;
296 
297 	kstat_named_init(&ets->ets_hck_meoifail, "meoi_fail",
298 	    KSTAT_DATA_UINT64);
299 	ets->ets_hck_meoifail.value.ui64 = 0;
300 
301 	kstat_named_init(&ets->ets_blocked, "blocked", KSTAT_DATA_UINT64);
302 	ets->ets_blocked.value.ui64 = 0;
303 
304 	kstat_named_init(&ets->ets_unblocked, "unblocked", KSTAT_DATA_UINT64);
305 	ets->ets_unblocked.value.ui64 = 0;
306 
307 	kstat_named_init(&ets->ets_recycled, "recycled", KSTAT_DATA_UINT64);
308 	ets->ets_recycled.value.ui64 = 0;
309 
310 	kstat_named_init(&ets->ets_bytes, "bytes", KSTAT_DATA_UINT64);
311 	ets->ets_bytes.value.ui64 = 0;
312 
313 	kstat_named_init(&ets->ets_packets, "packets", KSTAT_DATA_UINT64);
314 	ets->ets_packets.value.ui64 = 0;
315 
316 	kstat_install(txq->et_kstat);
317 	return (B_TRUE);
318 }
319 
320 void
ena_stat_rxq_cleanup(ena_rxq_t * rxq)321 ena_stat_rxq_cleanup(ena_rxq_t *rxq)
322 {
323 	if (rxq->er_kstat != NULL) {
324 		kstat_delete(rxq->er_kstat);
325 		rxq->er_kstat = NULL;
326 	}
327 }
328 
329 boolean_t
ena_stat_rxq_init(ena_rxq_t * rxq)330 ena_stat_rxq_init(ena_rxq_t *rxq)
331 {
332 	ena_t *ena = rxq->er_ena;
333 	kstat_t *ksp;
334 	char buf[128];
335 	ena_rxq_stat_t *ers = &rxq->er_stat;
336 
337 	(void) snprintf(buf, sizeof (buf), "rxq_%d", rxq->er_rxqs_idx);
338 
339 	ksp = kstat_create(ENA_MODULE_NAME, ddi_get_instance(ena->ena_dip), buf,
340 	    "net", KSTAT_TYPE_NAMED,
341 	    sizeof (ena_rxq_stat_t) / sizeof (kstat_named_t),
342 	    KSTAT_FLAG_VIRTUAL);
343 
344 	if (ksp == NULL) {
345 		ena_err(ena, "!failed to create %s kstats", buf);
346 		return (B_FALSE);
347 	}
348 
349 	rxq->er_kstat = ksp;
350 	ksp->ks_data = ers;
351 
352 	kstat_named_init(&ers->ers_packets, "packets", KSTAT_DATA_UINT64);
353 	ers->ers_packets.value.ui64 = 0;
354 
355 	kstat_named_init(&ers->ers_bytes, "bytes", KSTAT_DATA_UINT64);
356 	ers->ers_bytes.value.ui64 = 0;
357 
358 	kstat_named_init(&ers->ers_multi_desc, "multi_desc", KSTAT_DATA_UINT64);
359 	ers->ers_multi_desc.value.ui64 = 0;
360 
361 	kstat_named_init(&ers->ers_allocb_fail, "allocb_fail",
362 	    KSTAT_DATA_UINT64);
363 	ers->ers_allocb_fail.value.ui64 = 0;
364 
365 	kstat_named_init(&ers->ers_intr_limit, "intr_limit", KSTAT_DATA_UINT64);
366 	ers->ers_intr_limit.value.ui64 = 0;
367 
368 	kstat_named_init(&ers->ers_hck_ipv4_err, "hck_ipv4_err",
369 	    KSTAT_DATA_UINT64);
370 	ers->ers_hck_ipv4_err.value.ui64 = 0;
371 
372 	kstat_named_init(&ers->ers_hck_l4_err, "hck_l4_err", KSTAT_DATA_UINT64);
373 	ers->ers_hck_l4_err.value.ui64 = 0;
374 
375 	kstat_install(rxq->er_kstat);
376 	return (B_TRUE);
377 }
378 
379 int
ena_ring_rx_stat(mac_ring_driver_t rh,uint_t stat,uint64_t * val)380 ena_ring_rx_stat(mac_ring_driver_t rh, uint_t stat, uint64_t *val)
381 {
382 	int ret = 0;
383 	ena_rxq_t *rxq = (ena_rxq_t *)rh;
384 
385 	mutex_enter(&rxq->er_stat_lock);
386 
387 	switch (stat) {
388 	case MAC_STAT_RBYTES:
389 		*val = rxq->er_stat.ers_bytes.value.ui64;
390 		break;
391 	case MAC_STAT_IPACKETS:
392 		*val = rxq->er_stat.ers_packets.value.ui64;
393 		break;
394 	default:
395 		*val = 0;
396 		ret = ENOTSUP;
397 	}
398 
399 	mutex_exit(&rxq->er_stat_lock);
400 	return (ret);
401 }
402 
403 int
ena_ring_tx_stat(mac_ring_driver_t rh,uint_t stat,uint64_t * val)404 ena_ring_tx_stat(mac_ring_driver_t rh, uint_t stat, uint64_t *val)
405 {
406 	int ret = 0;
407 	ena_txq_t *txq = (ena_txq_t *)rh;
408 
409 	mutex_enter(&txq->et_stat_lock);
410 
411 	switch (stat) {
412 	case MAC_STAT_OBYTES:
413 		*val = txq->et_stat.ets_bytes.value.ui64;
414 		break;
415 	case MAC_STAT_OPACKETS:
416 		*val = txq->et_stat.ets_packets.value.ui64;
417 		break;
418 	default:
419 		*val = 0;
420 		ret = ENOTSUP;
421 	}
422 
423 	mutex_exit(&txq->et_stat_lock);
424 	return (ret);
425 }
426 
427 int
ena_m_stat(void * arg,uint_t stat,uint64_t * val)428 ena_m_stat(void *arg, uint_t stat, uint64_t *val)
429 {
430 	ena_t *ena = arg;
431 	ena_basic_stat_t *ebs = ena->ena_device_basic_kstat->ks_data;
432 	int ret = 0;
433 
434 	ret = ena_stat_device_basic_update(ena->ena_device_basic_kstat,
435 	    KSTAT_READ);
436 
437 	if (ret != 0) {
438 		return (ret);
439 	}
440 
441 	mutex_enter(&ena->ena_lock);
442 
443 	/*
444 	 * The ENA device does not provide a lot of the stats that a
445 	 * traditional NIC device would.
446 	 */
447 	switch (stat) {
448 	case MAC_STAT_NORCVBUF:
449 		*val = ebs->ebs_rx_drops.value.ui64;
450 		break;
451 
452 	case MAC_STAT_RBYTES:
453 		*val = ebs->ebs_rx_bytes.value.ui64;
454 		break;
455 
456 	case MAC_STAT_IPACKETS:
457 		*val = ebs->ebs_rx_pkts.value.ui64;
458 		break;
459 
460 	case MAC_STAT_OBYTES:
461 		*val = ebs->ebs_tx_bytes.value.ui64;
462 		break;
463 
464 	case MAC_STAT_OPACKETS:
465 		*val = ebs->ebs_tx_pkts.value.ui64;
466 		break;
467 
468 	default:
469 		ret = ENOTSUP;
470 		break;
471 	}
472 
473 	mutex_exit(&ena->ena_lock);
474 	return (ret);
475 }
476