1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 #include <sys/conf.h>
27 #include <sys/ddi.h>
28 #include <sys/stat.h>
29 #include <sys/pci.h>
30 #include <sys/sunddi.h>
31 #include <sys/modctl.h>
32 #include <sys/file.h>
33 #include <sys/cred.h>
34 #include <sys/byteorder.h>
35 #include <sys/atomic.h>
36 #include <sys/scsi/scsi.h>
37 #include <sys/mac_client.h>
38 #include <sys/modhash.h>
39
40 /*
41 * leadville header files
42 */
43 #include <sys/fibre-channel/fc.h>
44 #include <sys/fibre-channel/impl/fc_fcaif.h>
45
46 /*
47 * fcoe header files
48 */
49 #include <sys/fcoe/fcoe_common.h>
50
51 /*
52 * fcoei header files
53 */
54 #include <fcoei.h>
55
56 /*
57 * forward declaration of stack functions
58 */
59 static uint32_t fcoei_xch_check(
60 mod_hash_key_t key, mod_hash_val_t *val, void *arg);
61 static int fcoei_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
62 static int fcoei_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
63 static int fcoei_open(dev_t *devp, int flag, int otype, cred_t *credp);
64 static int fcoei_close(dev_t dev, int flag, int otype, cred_t *credp);
65 static int fcoei_ioctl(
66 dev_t dev, int cmd, intptr_t data, int mode, cred_t *credp, int *rval);
67 static int fcoei_attach_init(fcoei_soft_state_t *ss);
68 static int fcoei_detach_uninit(fcoei_soft_state_t *ss);
69 static void fcoei_watchdog(void *arg);
70 static void fcoei_process_events(fcoei_soft_state_t *ss);
71 static void fcoei_trigger_fp_attach(void *arg);
72 static void fcoei_abts_exchange(fcoei_exchange_t *xch);
73 static void fcoei_clear_watchdog_jobs(fcoei_soft_state_t *ss);
74
75 /*
76 * Driver identificaton stuff
77 */
78 static struct cb_ops fcoei_cb_ops = {
79 fcoei_open,
80 fcoei_close,
81 nodev,
82 nodev,
83 nodev,
84 nodev,
85 nodev,
86 fcoei_ioctl,
87 nodev,
88 nodev,
89 nodev,
90 nochpoll,
91 ddi_prop_op,
92 0,
93 D_MP | D_NEW | D_HOTPLUG,
94 CB_REV,
95 nodev,
96 nodev
97 };
98
99 static struct dev_ops fcoei_ops = {
100 DEVO_REV,
101 0,
102 nodev,
103 nulldev,
104 nulldev,
105 fcoei_attach,
106 fcoei_detach,
107 nodev,
108 &fcoei_cb_ops,
109 NULL,
110 ddi_power,
111 ddi_quiesce_not_needed
112 };
113
114 static struct modldrv modldrv = {
115 &mod_driverops,
116 FCOEI_NAME_VERSION,
117 &fcoei_ops,
118 };
119
120 static struct modlinkage modlinkage = {
121 MODREV_1,
122 &modldrv,
123 NULL
124 };
125
126 /*
127 * Driver's global variables
128 */
129 void *fcoei_state = NULL;
130 int fcoei_use_ext_log = 0;
131
132 /*
133 * Common loadable module entry points _init, _fini, _info
134 */
135 int
_init(void)136 _init(void)
137 {
138 int ret;
139
140 ret = ddi_soft_state_init(&fcoei_state, sizeof (fcoei_soft_state_t), 0);
141 if (ret != DDI_SUCCESS) {
142 FCOEI_LOG(__FUNCTION__, "soft state init failed: %x", ret);
143 return (ret);
144 }
145
146 ret = mod_install(&modlinkage);
147 if (ret != 0) {
148 ddi_soft_state_fini(&fcoei_state);
149 FCOEI_LOG(__FUNCTION__, "fcoei mod_install failed: %x", ret);
150 return (ret);
151 }
152
153 /*
154 * Let FCTL initialize devo_bus_ops
155 */
156 fc_fca_init(&fcoei_ops);
157
158 FCOEI_LOG(__FUNCTION__, "fcoei _init succeeded");
159 return (ret);
160 }
161
162 int
_fini(void)163 _fini(void)
164 {
165 int ret;
166
167 ret = mod_remove(&modlinkage);
168 if (ret != 0) {
169 FCOEI_EXT_LOG(__FUNCTION__, "fcoei mod_remove failed: %x", ret);
170 return (ret);
171 }
172
173 ddi_soft_state_fini(&fcoei_state);
174 FCOEI_LOG(__FUNCTION__, "fcoei _fini succeeded");
175 return (ret);
176 }
177
178 int
_info(struct modinfo * modinfop)179 _info(struct modinfo *modinfop)
180 {
181 return (mod_info(&modlinkage, modinfop));
182 }
183
184 /*
185 * Autoconfiguration entry points: attach, detach, getinfo
186 */
187
188 static int
fcoei_attach(dev_info_t * dip,ddi_attach_cmd_t cmd)189 fcoei_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
190 {
191 int ret;
192 int fcoe_ret;
193 int instance;
194 fcoei_soft_state_t *ss;
195
196 instance = ddi_get_instance(dip);
197 FCOEI_LOG(__FUNCTION__, "instance is %d", instance);
198 switch (cmd) {
199 case DDI_ATTACH:
200 ret = ddi_soft_state_zalloc(fcoei_state, instance);
201 if (ret != DDI_SUCCESS) {
202 FCOEI_LOG(__FUNCTION__, "ss zalloc failed: %x", ret);
203 return (ret);
204 }
205
206 /*
207 * Get the soft state, and do basic initialization with dip
208 */
209 ss = ddi_get_soft_state(fcoei_state, instance);
210 ss->ss_dip = dip;
211
212 fcoe_ret = fcoei_attach_init(ss);
213 if (fcoe_ret != FCOE_SUCCESS) {
214 ddi_soft_state_free(fcoei_state, instance);
215 FCOEI_LOG(__FUNCTION__, "fcoei_attach_init failed: "
216 "%x", fcoe_ret);
217 return (DDI_FAILURE);
218 }
219
220 ss->ss_flags |= SS_FLAG_TRIGGER_FP_ATTACH;
221 (void) timeout(fcoei_trigger_fp_attach, ss, FCOE_SEC2TICK(1));
222 FCOEI_LOG(__FUNCTION__, "fcoei_attach succeeded: dip-%p, "
223 "cmd-%x", dip, cmd);
224 return (DDI_SUCCESS);
225
226 case DDI_RESUME:
227 return (DDI_SUCCESS);
228
229 default:
230 FCOEI_LOG(__FUNCTION__, "unsupported attach cmd-%X", cmd);
231 return (DDI_FAILURE);
232 }
233 }
234
235 static int
fcoei_detach(dev_info_t * dip,ddi_detach_cmd_t cmd)236 fcoei_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
237 {
238 int fcoe_ret;
239 int instance;
240 fcoei_soft_state_t *ss;
241
242 instance = ddi_get_instance(dip);
243 ss = ddi_get_soft_state(fcoei_state, instance);
244 if (ss == NULL) {
245 FCOEI_LOG(__FUNCTION__, "get ss failed: dip-%p", dip);
246 return (DDI_FAILURE);
247 }
248
249 switch (cmd) {
250 case DDI_DETACH:
251 if (ss->ss_flags & SS_FLAG_TRIGGER_FP_ATTACH) {
252 FCOEI_LOG(__FUNCTION__, "still await fp attach");
253 return (DDI_FAILURE);
254 }
255
256 if (ss->ss_flags & SS_FLAG_LV_BOUND) {
257 FCOEI_LOG(__FUNCTION__, "fp is not detached yet");
258 return (DDI_FAILURE);
259 }
260
261 fcoe_ret = fcoei_detach_uninit(ss);
262 if (fcoe_ret != FCOE_SUCCESS) {
263 FCOEI_LOG(__FUNCTION__, "fcoei_detach_uninit failed:"
264 " dip-%p, fcoe_ret-%d", dip, fcoe_ret);
265 return (DDI_FAILURE);
266 }
267
268 FCOEI_LOG(__FUNCTION__, "succeeded: dip-%p, cmd-%x", dip, cmd);
269 return (DDI_SUCCESS);
270
271 case DDI_SUSPEND:
272 return (DDI_SUCCESS);
273
274 default:
275 FCOEI_LOG(__FUNCTION__, "unspported detach cmd-%X", cmd);
276 return (DDI_FAILURE);
277 }
278 }
279
280 /*
281 * Device access entry points: open, close, ioctl
282 */
283
284 static int
fcoei_open(dev_t * devp,int flag,int otype,cred_t * credp)285 fcoei_open(dev_t *devp, int flag, int otype, cred_t *credp)
286 {
287 fcoei_soft_state_t *ss;
288
289 if (otype != OTYP_CHR) {
290 FCOEI_LOG(__FUNCTION__, "flag: %x", flag);
291 return (EINVAL);
292 }
293
294 if (drv_priv(credp)) {
295 return (EPERM);
296 }
297
298 /*
299 * First of all, get related soft state
300 */
301 ss = ddi_get_soft_state(fcoei_state, (int)getminor(*devp));
302 if (ss == NULL) {
303 return (ENXIO);
304 }
305
306 mutex_enter(&ss->ss_ioctl_mutex);
307 if (ss->ss_ioctl_flags & FCOEI_IOCTL_FLAG_OPEN) {
308 /*
309 * We don't support concurrent open
310 */
311 mutex_exit(&ss->ss_ioctl_mutex);
312 return (EBUSY);
313 }
314
315 ss->ss_ioctl_flags |= FCOEI_IOCTL_FLAG_OPEN;
316 mutex_exit(&ss->ss_ioctl_mutex);
317
318 return (0);
319 }
320
321 static int
fcoei_close(dev_t dev,int flag,int otype,cred_t * credp)322 fcoei_close(dev_t dev, int flag, int otype, cred_t *credp)
323 {
324 fcoei_soft_state_t *ss;
325
326 if (otype != OTYP_CHR) {
327 FCOEI_LOG(__FUNCTION__, "flag: %x, %p", flag, credp);
328 return (EINVAL);
329 }
330
331 /*
332 * First of all, get related soft state
333 */
334 ss = ddi_get_soft_state(fcoei_state, (int)getminor(dev));
335 if (ss == NULL) {
336 return (ENXIO);
337 }
338
339 mutex_enter(&ss->ss_ioctl_mutex);
340 if (!(ss->ss_ioctl_flags & FCOEI_IOCTL_FLAG_OPEN)) {
341 /*
342 * If it's not open, we can exit
343 */
344
345 mutex_exit(&ss->ss_ioctl_mutex);
346 return (ENODEV);
347 }
348
349 ss->ss_ioctl_flags &= ~FCOEI_IOCTL_FLAG_OPEN;
350 mutex_exit(&ss->ss_ioctl_mutex);
351
352 return (0);
353 }
354
355 static int
fcoei_ioctl(dev_t dev,int cmd,intptr_t data,int mode,cred_t * credp,int * rval)356 fcoei_ioctl(dev_t dev, int cmd, intptr_t data, int mode,
357 cred_t *credp, int *rval)
358 {
359 fcoei_soft_state_t *ss;
360 int ret = 0;
361
362 if (drv_priv(credp) != 0) {
363 FCOEI_LOG(__FUNCTION__, "data: %p, %x", data, mode);
364 return (EPERM);
365 }
366
367 /*
368 * Get related soft state
369 */
370 ss = ddi_get_soft_state(fcoei_state, (int32_t)getminor(dev));
371 if (!ss) {
372 return (ENXIO);
373 }
374
375 /*
376 * Process ioctl
377 */
378 switch (cmd) {
379
380 default:
381 FCOEI_LOG(__FUNCTION__, "ioctl-0x%02X", cmd);
382 ret = ENOTTY;
383 }
384
385 /*
386 * Set return value
387 */
388 *rval = ret;
389 return (ret);
390 }
391
392 /*
393 * fcoei_attach_init
394 * init related stuff of the soft state
395 *
396 * Input:
397 * ss = the soft state that will be processed
398 *
399 * Return:
400 * if it succeeded or not
401 *
402 * Comment:
403 * N/A
404 */
405 static int
fcoei_attach_init(fcoei_soft_state_t * ss)406 fcoei_attach_init(fcoei_soft_state_t *ss)
407 {
408 fcoe_port_t *eport;
409 fcoe_client_t client_fcoei;
410 char taskq_name[32];
411 int ret;
412 la_els_logi_t *els = &ss->ss_els_logi;
413 svc_param_t *class3_param;
414
415 /*
416 * Register fcoei to FCOE as its client
417 */
418 client_fcoei.ect_eport_flags = EPORT_FLAG_INI_MODE |
419 EPORT_FLAG_IS_DIRECT_P2P;
420 client_fcoei.ect_max_fc_frame_size = FCOE_MAX_FC_FRAME_SIZE;
421 client_fcoei.ect_private_frame_struct_size = sizeof (fcoei_frame_t);
422 fcoei_init_ect_vectors(&client_fcoei);
423 client_fcoei.ect_client_port_struct = ss;
424 client_fcoei.ect_fcoe_ver = FCOE_VER_NOW;
425 FCOEI_LOG(__FUNCTION__, "version: %x %x", FCOE_VER_NOW, fcoe_ver_now);
426 ret = ddi_prop_get_int(DDI_DEV_T_ANY, ss->ss_dip,
427 DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "mac_id", -1);
428 if (ret == -1) {
429 FCOEI_LOG(__FUNCTION__, "get mac_id failed");
430 return (DDI_FAILURE);
431 } else {
432 client_fcoei.ect_channelid = ret;
433 }
434
435 /*
436 * It's fcoe's responsiblity to initialize eport's all elements,
437 * so we needn't do eport initialization
438 */
439 eport = fcoe_register_client(&client_fcoei);
440 if (eport == NULL) {
441 goto fail_register_client;
442 } else {
443 ss->ss_eport = eport;
444 FCOE_SET_DEFAULT_FPORT_ADDR(eport->eport_efh_dst);
445 }
446
447 /*
448 * Now it's time to register fca_tran to FCTL
449 * Remember fc_local_port is transparent to FCA (fcoei)
450 */
451 ss->ss_fca_tran.fca_version = FCTL_FCA_MODREV_5;
452 ss->ss_fca_tran.fca_numports = 1;
453 ss->ss_fca_tran.fca_pkt_size = sizeof (fcoei_exchange_t);
454 ss->ss_fca_tran.fca_cmd_max = 2048;
455
456 /*
457 * scsi_tran_hba_setup could need these stuff
458 */
459 ss->ss_fca_tran.fca_dma_lim = NULL;
460 ss->ss_fca_tran.fca_iblock = NULL;
461 ss->ss_fca_tran.fca_dma_attr = NULL;
462 ss->ss_fca_tran.fca_acc_attr = NULL;
463
464 /*
465 * Initialize vectors
466 */
467 fcoei_init_fcatran_vectors(&ss->ss_fca_tran);
468
469 /*
470 * fc_fca_attach only sets driver's private, it has nothing to with
471 * common port object between fcoei and leadville.
472 * After this attach, fp_attach will be triggered, and it will call
473 * fca_bind_port to let fcoei to know about common port object.
474 */
475 if (fc_fca_attach(ss->ss_dip, &ss->ss_fca_tran) != DDI_SUCCESS) {
476 goto fail_fca_attach;
477 }
478
479 /*
480 * It's time to do ss initialization
481 */
482 ret = ddi_create_minor_node(ss->ss_dip, "admin",
483 S_IFCHR, ddi_get_instance(ss->ss_dip), DDI_NT_NEXUS, 0);
484 if (ret != DDI_SUCCESS) {
485 goto fail_minor_node;
486 }
487
488 ss->ss_flags = 0;
489 ss->ss_port = NULL;
490 /*
491 * ss->ss_eport has been initialized
492 */
493
494 ss->ss_sol_oxid_hash = mod_hash_create_idhash(
495 "fcoei_sol_oxid_hash", FCOEI_SOL_HASH_SIZE,
496 mod_hash_null_valdtor);
497 ss->ss_unsol_rxid_hash = mod_hash_create_idhash(
498 "fcoei_unsol_rxid_hash", FCOEI_UNSOL_HASH_SIZE,
499 mod_hash_null_valdtor);
500 list_create(&ss->ss_comp_xch_list, sizeof (fcoei_exchange_t),
501 offsetof(fcoei_exchange_t, xch_comp_node));
502 ss->ss_next_sol_oxid = 0xFFFF;
503 ss->ss_next_unsol_rxid = 0xFFFF;
504
505 mutex_init(&ss->ss_watchdog_mutex, 0, MUTEX_DRIVER, 0);
506 cv_init(&ss->ss_watchdog_cv, NULL, CV_DRIVER, NULL);
507 (void) snprintf(taskq_name, 32, "leadville_fcoei_%d_taskq",
508 ddi_get_instance(ss->ss_dip));
509 taskq_name[31] = 0;
510 ss->ss_taskq = ddi_taskq_create(ss->ss_dip,
511 taskq_name, 64, TASKQ_DEFAULTPRI, DDI_SLEEP);
512
513 ss->ss_link_state = FC_STATE_OFFLINE;
514 ss->ss_link_speed = 0;
515 ss->ss_port_event_counter = 0;
516
517 list_create(&ss->ss_event_list, sizeof (fcoei_event_t),
518 offsetof(fcoei_event_t, ae_node));
519
520 ss->ss_sol_cnt1 = 0;
521 ss->ss_sol_cnt2 = 0;
522 ss->ss_sol_cnt = &ss->ss_sol_cnt1;
523 ss->ss_unsol_cnt1 = 0;
524 ss->ss_unsol_cnt2 = 0;
525 ss->ss_unsol_cnt = &ss->ss_unsol_cnt1;
526 ss->ss_ioctl_flags = 0;
527
528 mutex_init(&ss->ss_ioctl_mutex, 0, MUTEX_DRIVER, 0);
529
530 bcopy(eport->eport_portwwn, els->nport_ww_name.raw_wwn, 8);
531 bcopy(eport->eport_nodewwn, els->node_ww_name.raw_wwn, 8);
532 els->common_service.fcph_version = 0x2008;
533 els->common_service.btob_credit = 3;
534 els->common_service.cmn_features = 0x8800;
535 els->common_service.conc_sequences = 0xff;
536 els->common_service.relative_offset = 3;
537 els->common_service.e_d_tov = 0x07d0;
538 class3_param = (svc_param_t *)&els->class_3;
539 class3_param->class_opt = 0x8800;
540 class3_param->rcv_size = els->common_service.rx_bufsize = 2048;
541 class3_param->conc_sequences = 0xff;
542 class3_param->open_seq_per_xchng = 1;
543
544 /*
545 * Fill out RNID Management Information
546 */
547 bcopy(ss->ss_eport->eport_portwwn, ss->ss_rnid.global_id, 8);
548 ss->ss_rnid.unit_type = FCOEI_RNID_HBA;
549 ss->ss_rnid.ip_version = FCOEI_RNID_IPV4;
550
551 /*
552 * Start our watchdog
553 */
554 (void) ddi_taskq_dispatch(ss->ss_taskq,
555 fcoei_watchdog, ss, DDI_SLEEP);
556 while (!(ss->ss_flags & SS_FLAG_WATCHDOG_RUNNING)) {
557 delay(50);
558 }
559
560 /*
561 * Report the device to the system
562 */
563 ddi_report_dev(ss->ss_dip);
564 return (DDI_SUCCESS);
565
566
567 fail_minor_node:
568 FCOEI_LOG(__FUNCTION__, "fail_minor_node");
569 (void) fc_fca_detach(ss->ss_dip);
570
571 fail_fca_attach:
572 eport->eport_deregister_client(eport);
573 FCOEI_LOG(__FUNCTION__, "fail_fca_attach");
574
575 fail_register_client:
576 FCOEI_LOG(__FUNCTION__, "fail_register_client");
577 return (DDI_FAILURE);
578 }
579
580 /*
581 * fcoei_detach_uninit
582 * uninit related stuff of the soft state
583 *
584 * Input:
585 * ss = the soft state that will be processed
586 *
587 * Return:
588 * if it succeeded or not
589 *
590 * Comment:
591 * N/A
592 */
593 int
fcoei_detach_uninit(fcoei_soft_state_t * ss)594 fcoei_detach_uninit(fcoei_soft_state_t *ss)
595 {
596 /*
597 * Stop watchdog first
598 */
599 if (ss->ss_flags & SS_FLAG_WATCHDOG_RUNNING) {
600 ss->ss_flags |= SS_FLAG_TERMINATE_WATCHDOG;
601 cv_broadcast(&ss->ss_watchdog_cv);
602 }
603
604 /*
605 * Destroy the taskq
606 */
607 ddi_taskq_wait(ss->ss_taskq);
608 ddi_taskq_destroy(ss->ss_taskq);
609
610 /*
611 * Release all allocated resources
612 */
613 mutex_destroy(&ss->ss_ioctl_mutex);
614 mutex_destroy(&ss->ss_watchdog_mutex);
615 cv_destroy(&ss->ss_watchdog_cv);
616 mod_hash_destroy_idhash(ss->ss_sol_oxid_hash);
617 mod_hash_destroy_idhash(ss->ss_unsol_rxid_hash);
618 list_destroy(&ss->ss_event_list);
619 ss->ss_eport->eport_deregister_client(ss->ss_eport);
620 ddi_remove_minor_node(ss->ss_dip, NULL);
621
622 /*
623 * Release itself
624 */
625 ddi_soft_state_free(fcoei_state, ddi_get_instance(ss->ss_dip));
626 return (FCOE_SUCCESS);
627 }
628
629 /*
630 * fcoei_watchdog
631 * Perform periodic checking and routine tasks
632 *
633 * Input:
634 * arg = the soft state that will be processed
635 *
636 * Return:
637 * N/A
638 *
639 * Comment:
640 * N/A
641 */
642 static void
fcoei_watchdog(void * arg)643 fcoei_watchdog(void *arg)
644 {
645 fcoei_soft_state_t *ss;
646 clock_t tmp_delay;
647 clock_t start_clock;
648 clock_t last_clock;
649
650 /*
651 * For debugging
652 */
653 ss = (fcoei_soft_state_t *)arg;
654 FCOEI_LOG(__FUNCTION__, "ss %p", ss);
655 FCOEI_LOG(__FUNCTION__, "sol_hash %p", ss->ss_sol_oxid_hash);
656 FCOEI_LOG(__FUNCTION__, "unsol_hash %p", ss->ss_unsol_rxid_hash);
657 ss->ss_flags |= SS_FLAG_WATCHDOG_RUNNING;
658 tmp_delay = FCOE_SEC2TICK(1) / 2;
659 last_clock = CURRENT_CLOCK;
660
661 /*
662 * If nobody reqeusts to terminate the watchdog, we will work forever
663 */
664 while (!(ss->ss_flags & SS_FLAG_TERMINATE_WATCHDOG)) {
665 /*
666 * We handle all asynchronous events serially
667 */
668 fcoei_process_events(ss);
669
670 /*
671 * To avoid to check timing too freqently, we check
672 * if we need skip timing stuff.
673 */
674 start_clock = CURRENT_CLOCK;
675 if ((start_clock - last_clock) < tmp_delay) {
676 goto end_timing;
677 } else {
678 last_clock = start_clock;
679 }
680
681 /*
682 * It's time to do timeout checking of solicited exchanges
683 */
684 if (ss->ss_sol_cnt == (&ss->ss_sol_cnt1)) {
685 if (ss->ss_sol_cnt2 == 0) {
686 ss->ss_sol_cnt = &ss->ss_sol_cnt2;
687 } else {
688 mod_hash_walk(ss->ss_sol_oxid_hash,
689 fcoei_xch_check, ss);
690 }
691 } else {
692 if (ss->ss_sol_cnt1 == 0) {
693 ss->ss_sol_cnt = &ss->ss_sol_cnt1;
694 } else {
695 mod_hash_walk(ss->ss_sol_oxid_hash,
696 fcoei_xch_check, ss);
697 }
698 }
699
700 /*
701 * It's time to do timeout checking of unsolicited exchange
702 */
703 if (ss->ss_unsol_cnt == (&ss->ss_unsol_cnt1)) {
704 if (ss->ss_unsol_cnt2 == 0) {
705 ss->ss_unsol_cnt = &ss->ss_unsol_cnt2;
706 } else {
707 mod_hash_walk(ss->ss_unsol_rxid_hash,
708 fcoei_xch_check, ss);
709 }
710 } else {
711 if (ss->ss_unsol_cnt1 == 0) {
712 ss->ss_unsol_cnt = &ss->ss_unsol_cnt1;
713 } else {
714 mod_hash_walk(ss->ss_unsol_rxid_hash,
715 fcoei_xch_check, ss);
716 }
717 }
718
719 /*
720 * Check if there are exchanges which are ready to complete
721 */
722 fcoei_handle_comp_xch_list(ss);
723
724 end_timing:
725 /*
726 * Wait for next cycle
727 */
728 mutex_enter(&ss->ss_watchdog_mutex);
729 ss->ss_flags |= SS_FLAG_WATCHDOG_IDLE;
730 if (!list_is_empty(&ss->ss_event_list)) {
731 goto skip_wait;
732 }
733
734 (void) cv_timedwait(&ss->ss_watchdog_cv,
735 &ss->ss_watchdog_mutex, CURRENT_CLOCK +
736 (clock_t)tmp_delay);
737 skip_wait:
738 ss->ss_flags &= ~SS_FLAG_WATCHDOG_IDLE;
739 mutex_exit(&ss->ss_watchdog_mutex);
740 }
741
742 /*
743 * Do clear work before exit
744 */
745 fcoei_clear_watchdog_jobs(ss);
746
747 /*
748 * Watchdog has stopped
749 */
750 ss->ss_flags &= ~SS_FLAG_WATCHDOG_RUNNING;
751 }
752
753 static void
fcoei_clear_watchdog_jobs(fcoei_soft_state_t * ss)754 fcoei_clear_watchdog_jobs(fcoei_soft_state_t *ss)
755 {
756 fcoei_event_t *ae;
757 fcoe_frame_t *frm;
758
759 mutex_enter(&ss->ss_watchdog_mutex);
760 while (!list_is_empty(&ss->ss_event_list)) {
761 ae = (fcoei_event_t *)list_head(&ss->ss_event_list);
762 list_remove(&ss->ss_event_list, ae);
763 switch (ae->ae_type) {
764 case AE_EVENT_SOL_FRAME:
765 frm = (fcoe_frame_t *)ae->ae_obj;
766 frm->frm_eport->eport_release_frame(frm);
767 break;
768
769 case AE_EVENT_UNSOL_FRAME:
770 frm = (fcoe_frame_t *)ae->ae_obj;
771 frm->frm_eport->eport_free_netb(frm->frm_netb);
772 frm->frm_eport->eport_release_frame(frm);
773 break;
774
775 case AE_EVENT_PORT:
776 atomic_dec_32(&ss->ss_port_event_counter);
777 /* FALLTHROUGH */
778
779 case AE_EVENT_RESET:
780 kmem_free(ae, sizeof (fcoei_event_t));
781 break;
782
783 case AE_EVENT_EXCHANGE:
784 /* FALLTHROUGH */
785
786 default:
787 break;
788 }
789 }
790
791 mod_hash_clear(ss->ss_unsol_rxid_hash);
792 mod_hash_clear(ss->ss_sol_oxid_hash);
793
794 while (!list_is_empty(&ss->ss_comp_xch_list)) {
795 (void) list_remove_head(&ss->ss_comp_xch_list);
796 }
797 mutex_exit(&ss->ss_watchdog_mutex);
798 }
799
800 /*
801 * fcoei_process_events
802 * Process the events one by one
803 *
804 * Input:
805 * ss = the soft state that will be processed
806 *
807 * Return:
808 * N/A
809 *
810 * Comment:
811 * N/A
812 */
813 static void
fcoei_process_events(fcoei_soft_state_t * ss)814 fcoei_process_events(fcoei_soft_state_t *ss)
815 {
816 fcoei_event_t *ae = NULL;
817
818 /*
819 * It's the only place to delete node from ss_event_list, so we needn't
820 * hold mutex to check if the list is empty.
821 */
822 ASSERT(!MUTEX_HELD(&ss->ss_watchdog_mutex));
823 while (list_is_empty(&ss->ss_event_list) == B_FALSE) {
824 mutex_enter(&ss->ss_watchdog_mutex);
825 ae = (fcoei_event_t *)list_remove_head(&ss->ss_event_list);
826 mutex_exit(&ss->ss_watchdog_mutex);
827
828 switch (ae->ae_type) {
829 case AE_EVENT_SOL_FRAME:
830 fcoei_handle_sol_frame_done((fcoe_frame_t *)ae->ae_obj);
831 break;
832
833 case AE_EVENT_UNSOL_FRAME:
834 fcoei_process_unsol_frame((fcoe_frame_t *)ae->ae_obj);
835 break;
836
837 case AE_EVENT_EXCHANGE:
838 fcoei_process_event_exchange(ae);
839 break;
840
841 case AE_EVENT_PORT:
842 fcoei_process_event_port(ae);
843 break;
844
845 case AE_EVENT_RESET:
846 fcoei_process_event_reset(ae);
847 break;
848
849 default:
850 FCOEI_LOG(__FUNCTION__, "unsupported events");
851 }
852
853 }
854 }
855
856 /*
857 * fcoei_handle_tmout_xch_list
858 * Complete every exchange in the timed-out xch list of the soft state
859 *
860 * Input:
861 * ss = the soft state that need be handled
862 *
863 * Return:
864 * N/A
865 *
866 * Comment:
867 * When mod_hash_walk is in progress, we can't change the hashtable.
868 * This is post-walk handling of exchange timing
869 */
870 void
fcoei_handle_comp_xch_list(fcoei_soft_state_t * ss)871 fcoei_handle_comp_xch_list(fcoei_soft_state_t *ss)
872 {
873 fcoei_exchange_t *xch = NULL;
874
875 while ((xch = list_remove_head(&ss->ss_comp_xch_list)) != NULL) {
876 fcoei_complete_xch(xch, NULL, xch->xch_fpkt->pkt_state,
877 xch->xch_fpkt->pkt_reason);
878 }
879 }
880
881 /*
882 * fcoei_xch_check
883 * Check if the exchange timed out or link is down
884 *
885 * Input:
886 * key = rxid of the unsolicited exchange
887 * val = the unsolicited exchange
888 * arg = the soft state
889 *
890 * Return:
891 * MH_WALK_CONTINUE = continue to walk
892 *
893 * Comment:
894 * We need send ABTS for timed-out for solicited exchange
895 * If it's solicited FLOGI, we need set SS_FLAG_FLOGI_FAILED
896 * If the link is down, we think it has timed out too.
897 */
898 /* ARGSUSED */
899 static uint32_t
fcoei_xch_check(mod_hash_key_t key,mod_hash_val_t * val,void * arg)900 fcoei_xch_check(mod_hash_key_t key, mod_hash_val_t *val, void *arg)
901 {
902 fcoei_exchange_t *xch = (fcoei_exchange_t *)val;
903
904 ASSERT(xch->xch_ss == arg);
905 if ((xch->xch_end_tick < CURRENT_CLOCK) &&
906 (xch->xch_ss->ss_link_state != FC_STATE_OFFLINE)) {
907 if (xch->xch_flags & XCH_FLAG_IN_SOL_HASH) {
908 ASSERT(xch->xch_oxid == CMHK(key));
909 /*
910 * It's solicited exchange
911 */
912 fcoei_abts_exchange(xch);
913 if (LA_ELS_FLOGI == ((ls_code_t *)(void *)
914 xch->xch_fpkt->pkt_cmd)->ls_code) {
915 /*
916 * It's solicited FLOGI
917 */
918 xch->xch_ss->ss_flags |= SS_FLAG_FLOGI_FAILED;
919 }
920 }
921
922 FCOEI_LOG(__FUNCTION__, "oxid-%x/rxid-%x timed out",
923 xch->xch_oxid, xch->xch_rxid);
924 xch->xch_flags |= XCH_FLAG_TMOUT;
925 xch->xch_fpkt->pkt_state = FC_PKT_TIMEOUT;
926 xch->xch_fpkt->pkt_reason = FC_REASON_ABORTED;
927 list_insert_tail(&xch->xch_ss->ss_comp_xch_list, xch);
928 } else if (xch->xch_ss->ss_link_state == FC_STATE_OFFLINE) {
929 FCOEI_LOG(__FUNCTION__, "oxid-%x/rxid-%x offline complete",
930 xch->xch_oxid, xch->xch_rxid);
931 xch->xch_flags |= XCH_FLAG_TMOUT;
932 xch->xch_fpkt->pkt_state = FC_PKT_PORT_OFFLINE;
933 xch->xch_fpkt->pkt_reason = FC_REASON_OFFLINE;
934 list_insert_tail(&xch->xch_ss->ss_comp_xch_list, xch);
935 }
936
937 return (MH_WALK_CONTINUE);
938 }
939
940 /*
941 * fcoei_init_ifm
942 * initialize fcoei_frame
943 *
944 * Input:
945 * frm = the frame that ifm need link to
946 * xch = the exchange that ifm need link to
947 *
948 * Return:
949 * N/A
950 *
951 * Comment:
952 * For solicited frames, it's called after FC frame header initialization
953 * For unsolicited frames, it's called just after the frame enters fcoei
954 */
955 void
fcoei_init_ifm(fcoe_frame_t * frm,fcoei_exchange_t * xch)956 fcoei_init_ifm(fcoe_frame_t *frm, fcoei_exchange_t *xch)
957 {
958 FRM2IFM(frm)->ifm_frm = frm;
959 FRM2IFM(frm)->ifm_xch = xch;
960 FRM2IFM(frm)->ifm_rctl = FRM_R_CTL(frm);
961 }
962
963 /*
964 * fcoei_trigger_fp_attach
965 * Trigger fp_attach for this fcoei port
966 *
967 * Input:
968 * arg = the soft state that fp will attach
969 *
970 * Return:
971 * N/A
972 *
973 * Comment:
974 * N/A
975 */
976 static void
fcoei_trigger_fp_attach(void * arg)977 fcoei_trigger_fp_attach(void * arg)
978 {
979 fcoei_soft_state_t *ss = (fcoei_soft_state_t *)arg;
980 dev_info_t *child = NULL;
981 int rval = NDI_FAILURE;
982
983 ndi_devi_alloc_sleep(ss->ss_dip, "fp", DEVI_PSEUDO_NODEID, &child);
984 if (child == NULL) {
985 FCOEI_LOG(__FUNCTION__, "can't alloc dev_info");
986 return;
987 }
988
989 /*
990 * fp/fctl need this property
991 */
992 if (ddi_prop_update_string(DDI_DEV_T_NONE, child,
993 "bus-addr", "0,0") != DDI_PROP_SUCCESS) {
994 FCOEI_LOG(__FUNCTION__, "update bus-addr failed");
995 (void) ndi_devi_free(child);
996 return;
997 }
998
999 /*
1000 * If it's physical HBA, fp.conf will register the property.
1001 * fcoei is one software HBA, so we need register it manually
1002 */
1003 if (ddi_prop_update_int(DDI_DEV_T_NONE, child,
1004 "port", 0) != DDI_PROP_SUCCESS) {
1005 FCOEI_LOG(__FUNCTION__, "update port failed");
1006 (void) ndi_devi_free(child);
1007 return;
1008 }
1009
1010 /*
1011 * It will call fp_attach eventually
1012 */
1013 rval = ndi_devi_online(child, NDI_ONLINE_ATTACH);
1014 ss->ss_flags &= ~SS_FLAG_TRIGGER_FP_ATTACH;
1015 if (rval != NDI_SUCCESS) {
1016 FCOEI_LOG(__FUNCTION__, "devi_online: %d", rval);
1017 } else {
1018 FCOEI_LOG(__FUNCTION__, "triggered successfully");
1019 }
1020 }
1021
1022 /*
1023 * fcoei_abts_exchange
1024 * Send ABTS to abort solicited exchange
1025 *
1026 * Input:
1027 * xch = the exchange that will be aborted
1028 *
1029 * Return:
1030 * N/A
1031 *
1032 * Comment:
1033 * ABTS frame uses the same oxid as the exchange
1034 */
1035 static void
fcoei_abts_exchange(fcoei_exchange_t * xch)1036 fcoei_abts_exchange(fcoei_exchange_t *xch)
1037 {
1038 fc_packet_t *fpkt = xch->xch_fpkt;
1039 fcoe_frame_t *frm = NULL;
1040
1041 /*
1042 * BLS_ABTS doesn't contain any other payload except FCFH
1043 */
1044 frm = xch->xch_ss->ss_eport->eport_alloc_frame(xch->xch_ss->ss_eport,
1045 FCFH_SIZE, NULL);
1046 if (frm == NULL) {
1047 FCOEI_LOG(__FUNCTION__, "can't alloc frame: %p", xch);
1048 return;
1049 }
1050
1051 FFM_R_CTL(0x81, frm);
1052 FFM_D_ID(fpkt->pkt_cmd_fhdr.d_id, frm);
1053 FFM_S_ID(fpkt->pkt_cmd_fhdr.s_id, frm);
1054 FFM_F_CTL(0x090000, frm);
1055 FFM_SEQ_ID(0x01, frm);
1056 FFM_OXID(xch->xch_oxid, frm);
1057 FFM_RXID(xch->xch_rxid, frm);
1058 fcoei_init_ifm(frm, xch);
1059 xch->xch_ss->ss_eport->eport_tx_frame(frm);
1060 }
1061
1062 /*
1063 * fcoei_complete_xch
1064 * Complete the exchange
1065 *
1066 * Input:
1067 * xch = the exchange that will be completed
1068 * frm = newly-allocated frame that has not been submitted
1069 * pkt_state = LV fpkt state
1070 * pkt_reason = LV fpkt reason
1071 *
1072 * Return:
1073 * N/A
1074 *
1075 * Comment:
1076 * N/A
1077 */
1078 void
fcoei_complete_xch(fcoei_exchange_t * xch,fcoe_frame_t * frm,uint8_t pkt_state,uint8_t pkt_reason)1079 fcoei_complete_xch(fcoei_exchange_t *xch, fcoe_frame_t *frm,
1080 uint8_t pkt_state, uint8_t pkt_reason)
1081 {
1082 mod_hash_val_t val;
1083
1084 if (pkt_state != FC_PKT_SUCCESS) {
1085 FCOEI_LOG(__FUNCTION__, "FHDR: %x/%x/%x, %x/%x/%x",
1086 xch->xch_fpkt->pkt_cmd_fhdr.r_ctl,
1087 xch->xch_fpkt->pkt_cmd_fhdr.f_ctl,
1088 xch->xch_fpkt->pkt_cmd_fhdr.type,
1089 xch->xch_fpkt->pkt_resp_fhdr.r_ctl,
1090 xch->xch_fpkt->pkt_resp_fhdr.f_ctl,
1091 xch->xch_fpkt->pkt_resp_fhdr.type);
1092 FCOEI_LOG(__FUNCTION__, "%p/%p/%x/%x",
1093 xch, frm, pkt_state, pkt_reason);
1094 }
1095
1096 if (frm != NULL) {
1097 /*
1098 * It's newly-allocated frame , which we haven't sent out
1099 */
1100 xch->xch_ss->ss_eport->eport_free_netb(frm->frm_netb);
1101 xch->xch_ss->ss_eport->eport_release_frame(frm);
1102 FCOEI_LOG(__FUNCTION__, "xch: %p, not submitted", xch);
1103 }
1104
1105 /*
1106 * If xch is in hash table, we need remove it
1107 */
1108 if (xch->xch_flags & XCH_FLAG_IN_SOL_HASH) {
1109 (void) mod_hash_remove(xch->xch_ss->ss_sol_oxid_hash,
1110 FMHK(xch->xch_oxid), &val);
1111 ASSERT((fcoei_exchange_t *)val == xch);
1112 xch->xch_flags &= ~XCH_FLAG_IN_SOL_HASH;
1113 } else if (xch->xch_flags & XCH_FLAG_IN_UNSOL_HASH) {
1114 (void) mod_hash_remove(xch->xch_ss->ss_unsol_rxid_hash,
1115 FMHK(xch->xch_rxid), &val);
1116 ASSERT((fcoei_exchange_t *)val == xch);
1117 xch->xch_flags &= ~XCH_FLAG_IN_UNSOL_HASH;
1118 } else {
1119 FCOEI_LOG(__FUNCTION__, "xch not in any hash: %p", xch);
1120 }
1121
1122 xch->xch_fpkt->pkt_state = pkt_state;
1123 xch->xch_fpkt->pkt_reason = pkt_reason;
1124 if (xch->xch_fpkt->pkt_tran_flags & FC_TRAN_NO_INTR) {
1125 FCOEI_LOG(__FUNCTION__, "polled xch is done: %p", xch);
1126 sema_v(&xch->xch_sema);
1127 } else {
1128 xch->xch_fpkt->pkt_comp(xch->xch_fpkt);
1129 }
1130 }
1131