1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 * Copyright 2019 Joyent, Inc.
26 */
27
28 /*
29 * This file defines interfaces between fcoe and its clients (FCoEI/FCoET)
30 */
31
32 #include <sys/ddi.h>
33 #include <sys/sunddi.h>
34 #include <sys/sunndi.h>
35 #include <sys/byteorder.h>
36 #include <sys/atomic.h>
37 #include <sys/sysmacros.h>
38 #include <sys/cmn_err.h>
39 #include <sys/crc32.h>
40 #include <sys/fcntl.h>
41 #include <sys/unistd.h>
42 #include <sys/mac_client.h>
43 #include <sys/strsubr.h>
44
45 /*
46 * FCoE header files
47 */
48 #include <sys/fcoe/fcoeio.h>
49 #include <sys/fcoe/fcoe_common.h>
50
51 /*
52 * Driver's own header files
53 */
54 #include <fcoe.h>
55 #include <fcoe_fc.h>
56 #include <fcoe_eth.h>
57
58 static void fcoe_fill_frame_headers(fcoe_frame_t *frm);
59 static void fcoe_fill_frame_tailers(fcoe_frame_t *frm);
60 static void fcoe_deregister_client(fcoe_port_t *eport);
61 static int fcoe_ctl(fcoe_port_t *eport, int cmd, void *arg);
62 static void fcoe_tx_frame(fcoe_frame_t *frm);
63 static void *fcoe_alloc_netb(fcoe_port_t *eport,
64 uint32_t fc_frame_size, uint8_t **ppfc);
65 static void fcoe_free_netb(void *netb);
66
67 /*
68 * Only this function will be called explicitly by clients
69 * Register the specified client port (fcoei/fcoet)
70 */
71 fcoe_port_t *
fcoe_register_client(fcoe_client_t * client)72 fcoe_register_client(fcoe_client_t *client)
73 {
74 fcoe_mac_t *mac;
75 fcoe_port_t *eport;
76
77 if (client->ect_fcoe_ver != fcoe_ver_now) {
78 cmn_err(CE_WARN, "FCoE modules version mismatch, "
79 "fail registering client.");
80 return (NULL);
81 }
82
83 /*
84 * We will not come here, when someone is changing ss_mac_list,
85 * so it's safe to go through ss_mac_list.
86 */
87 for (mac = list_head(&fcoe_global_ss->ss_mac_list); mac;
88 mac = list_next(&fcoe_global_ss->ss_mac_list, mac)) {
89 if (client->ect_channelid == mac->fm_linkid) {
90 break;
91 }
92 }
93
94 if (mac == NULL) {
95 FCOE_LOG(0, "can't find the MAC you want to bind");
96 return (NULL);
97 }
98
99 if (mac->fm_flags & FCOE_MAC_FLAG_BOUND) {
100 FCOE_LOG(0, "the MAC you want to bind is bound already");
101 return (NULL);
102 }
103
104 atomic_or_32(&mac->fm_flags, FCOE_MAC_FLAG_BOUND);
105 bcopy(client, &mac->fm_client, sizeof (fcoe_client_t));
106
107 /*
108 * fcoe_port_t initialization
109 */
110 eport = &mac->fm_eport;
111 eport->eport_flags = client->ect_eport_flags | EPORT_FLAG_MAC_IN_USE;
112 eport->eport_fcoe_private = mac;
113 eport->eport_client_private = client->ect_client_port_struct;
114 eport->eport_max_fc_frame_size = 2136;
115 eport->eport_tx_frame = fcoe_tx_frame;
116 eport->eport_alloc_frame = fcoe_allocate_frame;
117 eport->eport_release_frame = fcoe_release_frame;
118 eport->eport_alloc_netb = fcoe_alloc_netb;
119 eport->eport_free_netb = fcoe_free_netb;
120 eport->eport_deregister_client = fcoe_deregister_client;
121 eport->eport_ctl = fcoe_ctl;
122 eport->eport_set_mac_address = fcoe_mac_set_address;
123
124 return (eport);
125 }
126
127 /*
128 * The following routines will be called through vectors in fcoe_port_t
129 */
130
131 /*
132 * Deregister fcoet/fcoei modules, client should make sure the port is in
133 * offline status already
134 */
135 static void
fcoe_deregister_client(fcoe_port_t * eport)136 fcoe_deregister_client(fcoe_port_t *eport)
137 {
138 fcoe_mac_t *mac = EPORT2MAC(eport);
139
140 /*
141 * Wait for all the related frame to be freed, this should be fast
142 * because before deregister fcoei/fcoet will make sure its port
143 * is already in offline status so no frame will be received or sent
144 * any more
145 */
146 while (mac->fm_frm_cnt > 0) {
147 delay(10);
148 }
149
150 atomic_and_32(&EPORT2MAC(eport)->fm_flags, ~FCOE_MAC_FLAG_BOUND);
151 atomic_and_32(&mac->fm_eport.eport_flags, ~EPORT_FLAG_MAC_IN_USE);
152 if (!(EPORT2MAC(eport)->fm_flags & FCOE_MAC_FLAG_USER_DEL)) {
153 (void) fcoe_close_mac(mac);
154 fcoe_destroy_mac(mac);
155 }
156 }
157
158 /* ARGSUSED */
159 static int
fcoe_ctl(fcoe_port_t * eport,int cmd,void * arg)160 fcoe_ctl(fcoe_port_t *eport, int cmd, void *arg)
161 {
162 fcoe_mac_t *mac = EPORT2MAC(eport);
163
164 switch (cmd) {
165 case FCOE_CMD_PORT_ONLINE:
166 /*
167 * client ask us to online, so it's safe to post event
168 * and data up
169 */
170 if (fcoe_enable_callback(mac) == FCOE_FAILURE) {
171 return (FCOE_FAILURE);
172 }
173 mac->fm_state = FCOE_MAC_STATE_ONLINE;
174 if (mac->fm_link_state == FCOE_MAC_LINK_STATE_UP)
175 (void) ddi_taskq_dispatch(
176 fcoe_global_ss->ss_watchdog_taskq,
177 fcoe_mac_notify_link_up, mac, DDI_SLEEP);
178 break;
179 case FCOE_CMD_PORT_OFFLINE:
180 if (fcoe_disable_callback(mac) == FCOE_FAILURE) {
181 return (FCOE_FAILURE);
182 }
183 mac->fm_state = FCOE_MAC_STATE_OFFLINE;
184 // in case there are threads waiting
185 mutex_enter(&mac->fm_mutex);
186 cv_broadcast(&mac->fm_tx_cv);
187 mutex_exit(&mac->fm_mutex);
188 break;
189 default:
190 FCOE_LOG("fcoe", "fcoe_ctl, unsupported cmd %x", cmd);
191 break;
192 }
193
194 return (FCOE_SUCCESS);
195 }
196
197 /*
198 * Transmit the specified frame to the link
199 */
200 static void
fcoe_tx_frame(fcoe_frame_t * frm)201 fcoe_tx_frame(fcoe_frame_t *frm)
202 {
203 mblk_t *ret_mblk = NULL;
204 fcoe_mac_t *mac = FRM2MAC(frm);
205 mac_tx_cookie_t ret_cookie;
206
207 fcoe_fill_frame_headers(frm);
208 fcoe_fill_frame_tailers(frm);
209
210 tx_frame:
211 ret_cookie = mac_tx(mac->fm_cli_handle, FRM2MBLK(frm), 0,
212 MAC_TX_NO_ENQUEUE, &ret_mblk);
213 if (ret_cookie != (mac_tx_cookie_t)NULL) {
214 frm->frm_netb = ret_mblk;
215 mutex_enter(&mac->fm_mutex);
216 (void) cv_reltimedwait(&mac->fm_tx_cv, &mac->fm_mutex,
217 drv_usectohz(100000), TR_CLOCK_TICK);
218 mutex_exit(&mac->fm_mutex);
219
220 if (mac->fm_state == FCOE_MAC_STATE_OFFLINE) {
221 /*
222 * we are doing offline, so just tell the upper that
223 * this is finished, the cmd will be aborted soon.
224 */
225 fcoe_free_netb(ret_mblk);
226 } else {
227 goto tx_frame;
228 }
229 }
230
231 /*
232 * MAC driver will release the mblk of the frame
233 * We need only release the frame itself
234 */
235 mutex_enter(&FRM2MAC(frm)->fm_ss->ss_watch_mutex);
236 list_insert_tail(&FRM2MAC(frm)->fm_ss->ss_pfrm_list,
237 FRM2FMI(frm));
238 mac->fm_frm_cnt ++;
239 if (FRM2MAC(frm)->fm_ss->ss_flags & SS_FLAG_DOG_WAITING) {
240 cv_signal(&FRM2MAC(frm)->fm_ss->ss_watch_cv);
241 }
242 mutex_exit(&FRM2MAC(frm)->fm_ss->ss_watch_mutex);
243 }
244
245 /*
246 * Consider cache allocation in the future
247 */
248 void
fcoe_release_frame(fcoe_frame_t * frame)249 fcoe_release_frame(fcoe_frame_t *frame)
250 {
251 kmem_free(frame, frame->frm_alloc_size);
252 }
253
254 static void *
fcoe_alloc_netb(fcoe_port_t * eport,uint32_t fc_frame_size,uint8_t ** ppfc)255 fcoe_alloc_netb(fcoe_port_t *eport, uint32_t fc_frame_size, uint8_t **ppfc)
256 {
257 mblk_t *mp;
258
259 mp = fcoe_get_mblk(eport->eport_fcoe_private,
260 fc_frame_size + PADDING_SIZE);
261 if (mp != NULL) {
262 *ppfc = mp->b_rptr + PADDING_HEADER_SIZE;
263 }
264
265 return (mp);
266 }
267
268 static void
fcoe_free_netb(void * netb)269 fcoe_free_netb(void *netb)
270 {
271 freemsgchain((mblk_t *)netb);
272 }
273
274 fcoe_frame_t *
fcoe_allocate_frame(fcoe_port_t * eport,uint32_t fc_frame_size,void * xmp)275 fcoe_allocate_frame(fcoe_port_t *eport, uint32_t fc_frame_size, void *xmp)
276 {
277 fcoe_frame_t *frm;
278 fcoe_i_frame_t *fmi;
279 mblk_t *mp = xmp;
280 uint32_t alloc_size;
281 uint32_t raw_frame_size;
282
283 if (fc_frame_size > 2136) {
284 FCOE_LOG("fcoe", "fcoe_allocate_frame %d > 2136",
285 fc_frame_size);
286 return (NULL);
287 }
288
289 if (mp == NULL) {
290 /*
291 * We are allocating solicited frame now
292 */
293 raw_frame_size = PADDING_SIZE + fc_frame_size;
294 mp = fcoe_get_mblk(EPORT2MAC(eport), raw_frame_size);
295 if (mp == NULL) {
296 return (NULL);
297 }
298 }
299
300 alloc_size = sizeof (fcoe_frame_t) + sizeof (fcoe_i_frame_t) +
301 EPORT2MAC(eport)->fm_client.ect_private_frame_struct_size;
302
303 /*
304 * fcoe_frame_t initialization
305 */
306 frm = (fcoe_frame_t *)kmem_alloc(alloc_size, KM_SLEEP);
307 frm->frm_alloc_size = alloc_size;
308 frm->frm_fc_frame_size = fc_frame_size;
309 frm->frm_payload_size = fc_frame_size -
310 sizeof (fcoe_fc_frame_header_t);
311 frm->frm_fcoe_private = sizeof (fcoe_frame_t) + (uint8_t *)frm;
312 frm->frm_client_private = sizeof (fcoe_i_frame_t) +
313 (uint8_t *)frm->frm_fcoe_private;
314 frm->frm_flags = 0;
315 frm->frm_eport = eport;
316 frm->frm_netb = mp;
317
318 /*
319 * fcoe_i_frame_t initialization
320 */
321 fmi = FRM2FMI(frm);
322 fmi->fmi_frame = frm;
323 fmi->fmi_mac = EPORT2MAC(eport);
324 fmi->fmi_efh = (void *)mp->b_rptr;
325
326 fmi->fmi_ffh = (fcoe_frame_header_t *)
327 (sizeof (struct ether_header) + (uint8_t *)fmi->fmi_efh);
328
329 fmi->fmi_fc_frame = sizeof (fcoe_frame_header_t) +
330 (uint8_t *)fmi->fmi_ffh;
331 fmi->fmi_fft = (fcoe_frame_tailer_t *)
332 (fc_frame_size + (uint8_t *)fmi->fmi_fc_frame);
333
334 /*
335 * Continue to initialize fcoe_frame_t
336 */
337 frm->frm_hdr = (fcoe_fc_frame_header_t *)fmi->fmi_fc_frame;
338 frm->frm_ofh1 = NULL;
339 frm->frm_ofh2 = NULL;
340 frm->frm_fc_frame = (uint8_t *)frm->frm_hdr;
341 frm->frm_payload = sizeof (fcoe_fc_frame_header_t) +
342 (uint8_t *)frm->frm_fc_frame;
343 return (frm);
344 }
345
346 /*
347 * Sub routines called by interface functions
348 */
349
350 /*
351 * According to spec, fill EthernetII frame header, FCoE frame header
352 * VLAN (not included for now)
353 */
354 static void
fcoe_fill_frame_headers(fcoe_frame_t * frm)355 fcoe_fill_frame_headers(fcoe_frame_t *frm)
356 {
357 fcoe_i_frame_t *fmi = FRM2FMI(frm);
358
359 /*
360 * Initialize ethernet frame header
361 */
362 bcopy(FRM2MAC(frm)->fm_current_addr, &fmi->fmi_efh->ether_shost,
363 ETHERADDRL);
364 bcopy(frm->frm_eport->eport_efh_dst,
365 &fmi->fmi_efh->ether_dhost, ETHERADDRL);
366 fmi->fmi_efh->ether_type = htons(ETHERTYPE_FCOE);
367
368 /*
369 * Initialize FCoE frame header
370 */
371 bzero(fmi->fmi_ffh, sizeof (fcoe_frame_header_t));
372 FCOE_ENCAPS_VER(fmi->fmi_ffh, FCOE_VER);
373 /* set to SOFi3 for the first frame of a sequence */
374 if (FRM_SEQ_CNT(frm) == 0) {
375 FCOE_V2B_1(0x2E, fmi->fmi_ffh->ffh_sof);
376 } else {
377 FCOE_V2B_1(0x36, fmi->fmi_ffh->ffh_sof);
378 }
379 }
380
381 /*
382 * According to spec, fill FCOE frame tailer including CRC
383 * VLAN (not included for now)
384 */
385 static void
fcoe_fill_frame_tailers(fcoe_frame_t * frm)386 fcoe_fill_frame_tailers(fcoe_frame_t *frm)
387 {
388 uint32_t crc;
389
390 /*
391 * Initialize FCoE frame tailer
392 * CRC is not big endian, can't use macro V2B
393 */
394 CRC32(crc, frm->frm_fc_frame, frm->frm_fc_frame_size,
395 (uint32_t)~0, crc32_table);
396 FRM2FMI(frm)->fmi_fft->fft_crc[0] = 0xFF & (~crc);
397 FRM2FMI(frm)->fmi_fft->fft_crc[1] = 0xFF & (~crc >> 8);
398 FRM2FMI(frm)->fmi_fft->fft_crc[2] = 0xFF & (~crc >> 16);
399 FRM2FMI(frm)->fmi_fft->fft_crc[3] = 0xFF & (~crc >> 24);
400 if (FRM_F_CTL(frm) & 0x080000) {
401 FCOE_V2B_1(0x42, FRM2FMI(frm)->fmi_fft->fft_eof);
402 } else {
403 FCOE_V2B_1(0x41, FRM2FMI(frm)->fmi_fft->fft_eof);
404 }
405
406 FRM2FMI(frm)->fmi_fft->fft_resvd[0] = 0;
407 FRM2FMI(frm)->fmi_fft->fft_resvd[1] = 0;
408 FRM2FMI(frm)->fmi_fft->fft_resvd[2] = 0;
409 }
410
411 void
fcoe_mac_notify_link_up(void * arg)412 fcoe_mac_notify_link_up(void *arg)
413 {
414 fcoe_mac_t *mac = (fcoe_mac_t *)arg;
415
416 ASSERT(mac->fm_flags & FCOE_MAC_FLAG_BOUND);
417
418 mac->fm_client.ect_port_event(&mac->fm_eport,
419 FCOE_NOTIFY_EPORT_LINK_UP);
420 }
421 void
fcoe_mac_notify_link_down(void * arg)422 fcoe_mac_notify_link_down(void *arg)
423 {
424 fcoe_mac_t *mac = (fcoe_mac_t *)arg;
425
426 if (mac->fm_flags & FCOE_MAC_FLAG_BOUND) {
427 mac->fm_client.ect_port_event(&mac->fm_eport,
428 FCOE_NOTIFY_EPORT_LINK_DOWN);
429 }
430 }
431
432 int
fcoe_create_port(dev_info_t * parent,fcoe_mac_t * mac,int is_target)433 fcoe_create_port(dev_info_t *parent, fcoe_mac_t *mac, int is_target)
434 {
435 int rval = 0;
436 dev_info_t *child = NULL;
437 char *devname = is_target ? FCOET_DRIVER_NAME : FCOEI_DRIVER_NAME;
438
439 ndi_devi_alloc_sleep(parent, devname, DEVI_PSEUDO_NODEID, &child);
440 if (child == NULL) {
441 FCOE_LOG("fcoe", "fail to create new devinfo");
442 return (NDI_FAILURE);
443 }
444
445 if (ddi_prop_update_int(DDI_DEV_T_NONE, child,
446 "mac_id", mac->fm_linkid) != DDI_PROP_SUCCESS) {
447 FCOE_LOG("fcoe",
448 "fcoe%d: prop_update port mac id failed for mac %d",
449 ddi_get_instance(parent), mac->fm_linkid);
450 (void) ndi_devi_free(child);
451 return (NDI_FAILURE);
452 }
453
454 rval = ndi_devi_online(child, NDI_ONLINE_ATTACH);
455 if (rval != NDI_SUCCESS) {
456 FCOE_LOG("fcoe", "fcoe%d: online_driver failed for mac %d",
457 ddi_get_instance(parent), mac->fm_linkid);
458 return (NDI_FAILURE);
459 }
460 mac->fm_client_dev = child;
461
462 return (rval);
463 }
464
465 int
fcoe_delete_port(dev_info_t * parent,fcoeio_t * fcoeio,datalink_id_t linkid,uint64_t * is_target)466 fcoe_delete_port(dev_info_t *parent, fcoeio_t *fcoeio, datalink_id_t linkid,
467 uint64_t *is_target)
468 {
469 int rval = 0;
470 fcoe_mac_t *mac;
471
472 mac = fcoe_lookup_mac_by_id(linkid);
473 if (mac == NULL) {
474 fcoeio->fcoeio_status = FCOEIOE_MAC_NOT_FOUND;
475 return (EINVAL);
476 }
477
478 *is_target = EPORT_CLT_TYPE(&mac->fm_eport);
479 if ((mac->fm_flags & FCOE_MAC_FLAG_ENABLED) != FCOE_MAC_FLAG_ENABLED) {
480 fcoeio->fcoeio_status = FCOEIOE_ALREADY;
481 return (EALREADY);
482 }
483
484 if (!(mac->fm_flags & FCOE_MAC_FLAG_BOUND)) {
485 /*
486 * It means that deferred detach has finished
487 * of last delete operation
488 */
489 goto skip_devi_offline;
490 }
491
492 atomic_and_32(&mac->fm_eport.eport_flags, ~EPORT_FLAG_MAC_IN_USE);
493 mac->fm_flags |= FCOE_MAC_FLAG_USER_DEL;
494 rval = ndi_devi_offline(mac->fm_client_dev, NDI_DEVI_REMOVE);
495 if (rval != NDI_SUCCESS) {
496 FCOE_LOG("fcoe", "fcoe%d: offline_driver %s failed",
497 ddi_get_instance(parent),
498 ddi_get_name(mac->fm_client_dev));
499 atomic_or_32(&mac->fm_eport.eport_flags,
500 EPORT_FLAG_MAC_IN_USE);
501
502 fcoeio->fcoeio_status = FCOEIOE_OFFLINE_FAILURE;
503 return (EBUSY);
504 }
505
506 skip_devi_offline:
507 (void) fcoe_close_mac(mac);
508 fcoe_destroy_mac(mac);
509 return (0);
510 }
511