1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * hermon_stats.c 29 * Hermon IB Performance Statistics routines 30 * 31 * Implements all the routines necessary for setting up, querying, and 32 * (later) tearing down all the kstats necessary for implementing to 33 * the interfaces necessary to provide busstat(1M) access. 34 */ 35 36 #include <sys/types.h> 37 #include <sys/conf.h> 38 #include <sys/ddi.h> 39 #include <sys/sunddi.h> 40 #include <sys/modctl.h> 41 42 #include <sys/ib/adapters/hermon/hermon.h> 43 44 static kstat_t *hermon_kstat_picN_create(hermon_state_t *state, int num_pic, 45 int num_evt, hermon_ks_mask_t *ev_array); 46 static kstat_t *hermon_kstat_cntr_create(hermon_state_t *state, int num_pic, 47 int (*update)(kstat_t *, int)); 48 static int hermon_kstat_cntr_update(kstat_t *ksp, int rw); 49 50 /* 51 * Hermon IB Performance Events structure 52 * This structure is read-only and is used to setup the individual kstats 53 * and to initialize the tki_ib_perfcnt[] array for each Hermon instance. 54 */ 55 hermon_ks_mask_t hermon_ib_perfcnt_list[HERMON_CNTR_NUMENTRIES] = { 56 {"port_xmit_data", 0, 0}, 57 {"port_recv_data", 0, 0}, 58 {"port_xmit_pkts", 0, 0}, 59 {"port_recv_pkts", 0, 0}, 60 {"port_recv_err", 0, 0}, 61 {"port_xmit_discards", 0, 0}, 62 {"vl15_dropped", 0, 0}, 63 {"port_xmit_wait", 0, 0}, 64 {"port_recv_remote_phys_err", 0, 0}, 65 {"port_xmit_constraint_err", 0, 0}, 66 {"port_recv_constraint_err", 0, 0}, 67 {"symbol_err_counter", 0, 0}, 68 {"link_err_recovery_cnt", 0, 0}, 69 {"link_downed_cnt", 0, 0}, 70 {"excessive_buffer_overruns", 0, 0}, 71 {"local_link_integrity_err", 0, 0}, 72 {"clear_pic", 0, 0} 73 }; 74 75 76 /* 77 * hermon_kstat_init() 78 * Context: Only called from attach() path context 79 */ 80 int 81 hermon_kstat_init(hermon_state_t *state) 82 { 83 hermon_ks_info_t *ksi; 84 uint_t numports; 85 int i; 86 87 /* Allocate a kstat info structure */ 88 ksi = (hermon_ks_info_t *)kmem_zalloc(sizeof (hermon_ks_info_t), 89 KM_SLEEP); 90 if (ksi == NULL) { 91 return (DDI_FAILURE); 92 } 93 state->hs_ks_info = ksi; 94 95 /* 96 * Create as many "pic" kstats as we have IB ports. Enable all 97 * of the events specified in the "hermon_ib_perfcnt_list" structure. 98 */ 99 numports = state->hs_cfg_profile->cp_num_ports; 100 for (i = 0; i < numports; i++) { 101 ksi->hki_picN_ksp[i] = hermon_kstat_picN_create(state, i, 102 HERMON_CNTR_NUMENTRIES, hermon_ib_perfcnt_list); 103 if (ksi->hki_picN_ksp[i] == NULL) { 104 goto kstat_init_fail; 105 } 106 } 107 108 /* Create the "counters" kstat too */ 109 ksi->hki_cntr_ksp = hermon_kstat_cntr_create(state, numports, 110 hermon_kstat_cntr_update); 111 if (ksi->hki_cntr_ksp == NULL) { 112 goto kstat_init_fail; 113 } 114 115 /* Initialize the control register and initial counter values */ 116 ksi->hki_pcr = 0; 117 ksi->hki_pic0 = 0; 118 ksi->hki_pic1 = 0; 119 120 /* 121 * Initialize the Hermon hki_ib_perfcnt[] array values using the 122 * default values in hermon_ib_perfcnt_list[] 123 */ 124 for (i = 0; i < HERMON_CNTR_NUMENTRIES; i++) { 125 ksi->hki_ib_perfcnt[i] = hermon_ib_perfcnt_list[i]; 126 } 127 128 return (DDI_SUCCESS); 129 130 131 kstat_init_fail: 132 133 /* Delete all the previously created kstats */ 134 if (ksi->hki_cntr_ksp != NULL) { 135 kstat_delete(ksi->hki_cntr_ksp); 136 } 137 for (i = 0; i < numports; i++) { 138 if (ksi->hki_picN_ksp[i] != NULL) { 139 kstat_delete(ksi->hki_picN_ksp[i]); 140 } 141 } 142 143 /* Free the kstat info structure */ 144 kmem_free(ksi, sizeof (hermon_ks_info_t)); 145 146 return (DDI_FAILURE); 147 } 148 149 150 /* 151 * hermon_kstat_init() 152 * Context: Only called from attach() and/or detach() path contexts 153 */ 154 void 155 hermon_kstat_fini(hermon_state_t *state) 156 { 157 hermon_ks_info_t *ksi; 158 uint_t numports; 159 int i; 160 161 /* Get pointer to kstat info */ 162 ksi = state->hs_ks_info; 163 164 /* Delete all the "pic" kstats (one per port) */ 165 numports = state->hs_cfg_profile->cp_num_ports; 166 for (i = 0; i < numports; i++) { 167 if (ksi->hki_picN_ksp[i] != NULL) { 168 kstat_delete(ksi->hki_picN_ksp[i]); 169 } 170 } 171 172 /* Delete the "counter" kstats (one per port) */ 173 kstat_delete(ksi->hki_cntr_ksp); 174 175 /* Free the kstat info structure */ 176 kmem_free(ksi, sizeof (hermon_ks_info_t)); 177 } 178 179 180 /* 181 * hermon_kstat_picN_create() 182 * Context: Only called from attach() path context 183 */ 184 static kstat_t * 185 hermon_kstat_picN_create(hermon_state_t *state, int num_pic, int num_evt, 186 hermon_ks_mask_t *ev_array) 187 { 188 kstat_t *picN_ksp; 189 struct kstat_named *pic_named_data; 190 int drv_instance, i; 191 char *drv_name; 192 char pic_name[16]; 193 194 /* 195 * Create the "picN" kstat. In the steps, below we will attach 196 * all of our named event types to it. 197 */ 198 drv_name = (char *)ddi_driver_name(state->hs_dip); 199 drv_instance = ddi_get_instance(state->hs_dip); 200 (void) sprintf(pic_name, "pic%d", num_pic); 201 picN_ksp = kstat_create(drv_name, drv_instance, pic_name, "bus", 202 KSTAT_TYPE_NAMED, num_evt, NULL); 203 if (picN_ksp == NULL) { 204 return (NULL); 205 } 206 pic_named_data = (struct kstat_named *)(picN_ksp->ks_data); 207 208 /* 209 * Write event names and their associated pcr masks. The last entry 210 * in the array (clear_pic) is added separately below (as its pic 211 * value must be inverted). 212 */ 213 for (i = 0; i < num_evt - 1; i++) { 214 pic_named_data[i].value.ui64 = 215 ((uint64_t)i << (num_pic * HERMON_CNTR_SIZE)); 216 kstat_named_init(&pic_named_data[i], ev_array[i].ks_evt_name, 217 KSTAT_DATA_UINT64); 218 } 219 220 /* Add the "clear_pic" entry */ 221 pic_named_data[i].value.ui64 = 222 ~((uint64_t)HERMON_CNTR_MASK << (num_pic * HERMON_CNTR_SIZE)); 223 kstat_named_init(&pic_named_data[i], ev_array[i].ks_evt_name, 224 KSTAT_DATA_UINT64); 225 226 /* Install the kstat */ 227 kstat_install(picN_ksp); 228 229 return (picN_ksp); 230 } 231 232 233 /* 234 * hermon_kstat_cntr_create() 235 * Context: Only called from attach() path context 236 */ 237 static kstat_t * 238 hermon_kstat_cntr_create(hermon_state_t *state, int num_pic, 239 int (*update)(kstat_t *, int)) 240 { 241 struct kstat *cntr_ksp; 242 struct kstat_named *cntr_named_data; 243 int drv_instance, i; 244 char *drv_name; 245 char pic_str[16]; 246 247 /* 248 * Create the "counters" kstat. In the steps, below we will attach 249 * all of our "pic" to it. Note: The size of this kstat is 250 * num_pic + 1 because it also contains the "%pcr". 251 */ 252 drv_name = (char *)ddi_driver_name(state->hs_dip); 253 drv_instance = ddi_get_instance(state->hs_dip); 254 cntr_ksp = kstat_create(drv_name, drv_instance, "counters", "bus", 255 KSTAT_TYPE_NAMED, num_pic + 1, KSTAT_FLAG_WRITABLE); 256 if (cntr_ksp == NULL) { 257 return (NULL); 258 } 259 cntr_named_data = (struct kstat_named *)(cntr_ksp->ks_data); 260 261 /* 262 * Initialize the named kstats (for the "pcr" and for the 263 * individual "pic" kstats) 264 */ 265 kstat_named_init(&cntr_named_data[0], "pcr", KSTAT_DATA_UINT64); 266 for (i = 0; i < num_pic; i++) { 267 (void) sprintf(pic_str, "pic%d", i); 268 kstat_named_init(&cntr_named_data[i+1], pic_str, 269 KSTAT_DATA_UINT64); 270 } 271 272 /* 273 * Store the Hermon softstate pointer in the kstat's private field so 274 * that it is available to the update function. 275 */ 276 cntr_ksp->ks_private = (void *)state; 277 cntr_ksp->ks_update = update; 278 279 /* Install the kstat */ 280 kstat_install(cntr_ksp); 281 282 return (cntr_ksp); 283 } 284 285 286 /* 287 * hermon_kstat_cntr_update() 288 * Context: Called from the kstat context 289 */ 290 static int 291 hermon_kstat_cntr_update(kstat_t *ksp, int rw) 292 { 293 hermon_state_t *state; 294 hermon_ks_mask_t *ib_perf; 295 hermon_ks_info_t *ksi; 296 struct kstat_named *data; 297 uint64_t pcr; 298 uint32_t tmp; 299 uint32_t oldval; 300 uint_t numports, indx; 301 int status; 302 hermon_hw_sm_perfcntr_t sm_perfcntr; 303 304 /* 305 * Extract the Hermon softstate pointer, kstat data, pointer to the 306 * kstat info structure, and pointer to the hki_ib_perfcnt[] array 307 * from the input parameters. Note: For warlock purposes, these 308 * parameters are all accessed only in this routine and are, 309 * therefore, protected by the lock used by the kstat framework. 310 */ 311 state = ksp->ks_private; 312 data = (struct kstat_named *)(ksp->ks_data); 313 ksi = state->hs_ks_info; 314 ib_perf = &ksi->hki_ib_perfcnt[0]; 315 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ksi)) 316 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data)) 317 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ib_perf)) 318 319 /* 320 * Depending on whether we are reading the "pic" counters or 321 * writing the "pcr" control register, we need to handle and 322 * fill in the kstat data appropriately. 323 * 324 * If this is a write to the "pcr", then extract the value from 325 * the kstat data and store it in the kstat info structure. 326 * 327 * Otherwise, if this is a read of the "pic" counter(s), then 328 * extract the register offset, size, and mask values from the 329 * ib_perf[] array. Then read the corresponding register and store 330 * it into the kstat data. Note: We only read/fill in pic1 if more 331 * than one port is configured. 332 */ 333 numports = state->hs_cfg_profile->cp_num_ports; 334 if (rw == KSTAT_WRITE) { 335 /* Update the stored "pcr" value */ 336 ksi->hki_pcr = data[0].value.ui64; 337 return (0); 338 } else { 339 /* 340 * Get the current "pcr" value and extract the lower 341 * portion (corresponding to the counters for "pic0") 342 */ 343 pcr = ksi->hki_pcr; 344 indx = pcr & HERMON_CNTR_MASK; 345 data[0].value.ui64 = pcr; 346 347 /* 348 * Fill in the "pic0" counter, corresponding to port 1. 349 * This involves reading in the current value in the register 350 * and calculating how many events have happened since this 351 * register was last polled. Then we save away the current 352 * value for the counter and increment the "pic0" total by 353 * the number of new events. 354 */ 355 oldval = ib_perf[indx].ks_old_pic0; 356 357 status = hermon_getperfcntr_cmd_post(state, 1, 358 HERMON_CMD_NOSLEEP_SPIN, &sm_perfcntr); 359 if (status != HERMON_CMD_SUCCESS) { 360 return (-1); 361 } 362 switch (indx) { 363 case 0: /* port_xmit_data */ 364 tmp = sm_perfcntr.portxmdata; 365 break; 366 case 1: /* port_recv_data */ 367 tmp = sm_perfcntr.portrcdata; 368 break; 369 case 2: /* port_xmit_pkts */ 370 tmp = sm_perfcntr.portxmpkts; 371 break; 372 case 3: /* port_recv_pkts */ 373 tmp = sm_perfcntr.portrcpkts; 374 break; 375 case 4: /* port_recv_err */ 376 tmp = sm_perfcntr.portrcv; 377 break; 378 case 5: /* port_xmit_discards */ 379 tmp = sm_perfcntr.portxmdiscard; 380 break; 381 case 6: /* vl15_dropped */ 382 tmp = sm_perfcntr.vl15drop; 383 break; 384 case 7: /* port_xmit_wait */ 385 tmp = sm_perfcntr.portxmwait; 386 break; 387 case 8: /* port_recv_remote_phys_err */ 388 tmp = sm_perfcntr.portrcvrem; 389 break; 390 case 9: /* port_xmit_constraint_err */ 391 tmp = sm_perfcntr.portxmconstr; 392 break; 393 case 10: /* port_recv_constraint_err */ 394 tmp = sm_perfcntr.portrcconstr; 395 break; 396 case 11: /* symbol_err_counter */ 397 tmp = sm_perfcntr.symerr; 398 break; 399 case 12: /* link_err_recovery_cnt */ 400 tmp = sm_perfcntr.linkerrrec; 401 break; 402 case 13: /* link_downed_cnt */ 403 tmp = sm_perfcntr.linkdown; 404 break; 405 case 14: /* excessive_buffer_overruns */ 406 tmp = sm_perfcntr.xsbuffovrun; 407 break; 408 case 15: /* local_link_integrity_err */ 409 tmp = sm_perfcntr.locallinkint; 410 break; 411 case 16: /* clear_pic */ 412 tmp = 0; /* XXX */ 413 break; 414 default: 415 cmn_err(CE_CONT, "perf counter out of range\n"); 416 } 417 418 ib_perf[indx].ks_old_pic0 = tmp; 419 420 tmp = tmp - oldval; 421 ksi->hki_pic0 += tmp; 422 data[1].value.ui64 = ksi->hki_pic0; 423 424 /* 425 * If necessary, fill in the "pic1" counter for port 2. 426 * This works the same as above except that we extract the 427 * upper bits (corresponding to the counters for "pic1") 428 */ 429 if (numports == HERMON_MAX_PORTS) { 430 indx = pcr >> HERMON_CNTR_SIZE; 431 oldval = ib_perf[indx].ks_old_pic1; 432 433 status = hermon_getperfcntr_cmd_post(state, 2, 434 HERMON_CMD_NOSLEEP_SPIN, &sm_perfcntr); 435 if (status != HERMON_CMD_SUCCESS) { 436 return (-1); 437 } 438 switch (indx) { 439 case 0: /* port_xmit_data */ 440 tmp = sm_perfcntr.portxmdata; 441 break; 442 case 1: /* port_recv_data */ 443 tmp = sm_perfcntr.portrcdata; 444 break; 445 case 2: /* port_xmit_pkts */ 446 tmp = sm_perfcntr.portxmpkts; 447 break; 448 case 3: /* port_recv_pkts */ 449 tmp = sm_perfcntr.portrcpkts; 450 break; 451 case 4: /* port_recv_err */ 452 tmp = sm_perfcntr.portrcv; 453 break; 454 case 5: /* port_xmit_discards */ 455 tmp = sm_perfcntr.portxmdiscard; 456 break; 457 case 6: /* vl15_dropped */ 458 tmp = sm_perfcntr.vl15drop; 459 break; 460 case 7: /* port_xmit_wait */ 461 tmp = sm_perfcntr.portxmwait; 462 break; 463 case 8: /* port_recv_remote_phys_err */ 464 tmp = sm_perfcntr.portrcvrem; 465 break; 466 case 9: /* port_xmit_constraint_err */ 467 tmp = sm_perfcntr.portxmconstr; 468 break; 469 case 10: /* port_recv_constraint_err */ 470 tmp = sm_perfcntr.portrcconstr; 471 break; 472 case 11: /* symbol_err_counter */ 473 tmp = sm_perfcntr.symerr; 474 break; 475 case 12: /* link_err_recovery_cnt */ 476 tmp = sm_perfcntr.linkerrrec; 477 break; 478 case 13: /* link_downed_cnt */ 479 tmp = sm_perfcntr.linkdown; 480 break; 481 case 14: /* excessive_buffer_overruns */ 482 tmp = sm_perfcntr.xsbuffovrun; 483 break; 484 case 15: /* local_link_integrity_err */ 485 tmp = sm_perfcntr.locallinkint; 486 break; 487 case 16: /* clear_pic */ 488 tmp = 0; /* XXX */ 489 break; 490 default: 491 cmn_err(CE_CONT, "perf counter out of range\n"); 492 } 493 494 ib_perf[indx].ks_old_pic1 = tmp; 495 496 tmp = tmp - oldval; 497 ksi->hki_pic1 += tmp; 498 data[2].value.ui64 = ksi->hki_pic1; 499 } 500 501 return (0); 502 } 503 } 504