1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 /*
27 * sun4v LDC Link Layer
28 */
29 #include <sys/types.h>
30 #include <sys/file.h>
31 #include <sys/errno.h>
32 #include <sys/open.h>
33 #include <sys/cred.h>
34 #include <sys/kmem.h>
35 #include <sys/conf.h>
36 #include <sys/cmn_err.h>
37 #include <sys/ksynch.h>
38 #include <sys/modctl.h>
39 #include <sys/stat.h> /* needed for S_IFBLK and S_IFCHR */
40 #include <sys/debug.h>
41 #include <sys/cred.h>
42 #include <sys/promif.h>
43 #include <sys/ddi.h>
44 #include <sys/sunddi.h>
45 #include <sys/cyclic.h>
46 #include <sys/machsystm.h>
47 #include <sys/vm.h>
48 #include <sys/cpu.h>
49 #include <sys/intreg.h>
50 #include <sys/machcpuvar.h>
51 #include <sys/mmu.h>
52 #include <sys/pte.h>
53 #include <vm/hat.h>
54 #include <vm/as.h>
55 #include <vm/hat_sfmmu.h>
56 #include <sys/vm_machparam.h>
57 #include <vm/seg_kmem.h>
58 #include <vm/seg_kpm.h>
59 #include <sys/note.h>
60 #include <sys/ivintr.h>
61 #include <sys/hypervisor_api.h>
62 #include <sys/ldc.h>
63 #include <sys/ldc_impl.h>
64 #include <sys/cnex.h>
65 #include <sys/hsvc.h>
66 #include <sys/sdt.h>
67 #include <sys/kldc.h>
68
69 /* Core internal functions */
70 int i_ldc_h2v_error(int h_error);
71 void i_ldc_reset(ldc_chan_t *ldcp, boolean_t force_reset);
72
73 static int i_ldc_txq_reconf(ldc_chan_t *ldcp);
74 static int i_ldc_rxq_reconf(ldc_chan_t *ldcp, boolean_t force_reset);
75 static void i_ldc_rxq_drain(ldc_chan_t *ldcp);
76 static void i_ldc_reset_state(ldc_chan_t *ldcp);
77 static void i_ldc_debug_enter(void);
78
79 static int i_ldc_get_tx_tail(ldc_chan_t *ldcp, uint64_t *tail);
80 static void i_ldc_get_tx_head(ldc_chan_t *ldcp, uint64_t *head);
81 static int i_ldc_set_tx_tail(ldc_chan_t *ldcp, uint64_t tail);
82 static int i_ldc_set_rx_head(ldc_chan_t *ldcp, uint64_t head);
83 static int i_ldc_send_pkt(ldc_chan_t *ldcp, uint8_t pkttype, uint8_t subtype,
84 uint8_t ctrlmsg);
85
86 static int i_ldc_set_rxdq_head(ldc_chan_t *ldcp, uint64_t head);
87 static void i_ldc_rxdq_copy(ldc_chan_t *ldcp, uint64_t *head);
88 static uint64_t i_ldc_dq_rx_get_state(ldc_chan_t *ldcp, uint64_t *head,
89 uint64_t *tail, uint64_t *link_state);
90 static uint64_t i_ldc_hvq_rx_get_state(ldc_chan_t *ldcp, uint64_t *head,
91 uint64_t *tail, uint64_t *link_state);
92 static int i_ldc_rx_ackpeek(ldc_chan_t *ldcp, uint64_t rx_head,
93 uint64_t rx_tail);
94 static uint_t i_ldc_chkq(ldc_chan_t *ldcp);
95
96 /* Interrupt handling functions */
97 static uint_t i_ldc_tx_hdlr(caddr_t arg1, caddr_t arg2);
98 static uint_t i_ldc_rx_hdlr(caddr_t arg1, caddr_t arg2);
99 static uint_t i_ldc_rx_process_hvq(ldc_chan_t *ldcp, boolean_t *notify_client,
100 uint64_t *notify_event);
101 static void i_ldc_clear_intr(ldc_chan_t *ldcp, cnex_intrtype_t itype);
102
103 /* Read method functions */
104 static int i_ldc_read_raw(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep);
105 static int i_ldc_read_packet(ldc_chan_t *ldcp, caddr_t target_bufp,
106 size_t *sizep);
107 static int i_ldc_read_stream(ldc_chan_t *ldcp, caddr_t target_bufp,
108 size_t *sizep);
109
110 /* Write method functions */
111 static int i_ldc_write_raw(ldc_chan_t *ldcp, caddr_t target_bufp,
112 size_t *sizep);
113 static int i_ldc_write_packet(ldc_chan_t *ldcp, caddr_t target_bufp,
114 size_t *sizep);
115 static int i_ldc_write_stream(ldc_chan_t *ldcp, caddr_t target_bufp,
116 size_t *sizep);
117
118 /* Pkt processing internal functions */
119 static int i_ldc_check_seqid(ldc_chan_t *ldcp, ldc_msg_t *ldcmsg);
120 static int i_ldc_ctrlmsg(ldc_chan_t *ldcp, ldc_msg_t *ldcmsg);
121 static int i_ldc_process_VER(ldc_chan_t *ldcp, ldc_msg_t *msg);
122 static int i_ldc_process_RTS(ldc_chan_t *ldcp, ldc_msg_t *msg);
123 static int i_ldc_process_RTR(ldc_chan_t *ldcp, ldc_msg_t *msg);
124 static int i_ldc_process_RDX(ldc_chan_t *ldcp, ldc_msg_t *msg);
125 static int i_ldc_process_data_ACK(ldc_chan_t *ldcp, ldc_msg_t *msg);
126
127 /* Imported functions */
128 extern void i_ldc_mem_set_hsvc_vers(uint64_t major, uint64_t minor);
129 extern void i_ldc_init_mapin(ldc_soft_state_t *ldcssp, uint64_t major,
130 uint64_t minor);
131
132 /* LDC Version */
133 static ldc_ver_t ldc_versions[] = { {1, 0} };
134
135 /* number of supported versions */
136 #define LDC_NUM_VERS (sizeof (ldc_versions) / sizeof (ldc_versions[0]))
137
138 /* Invalid value for the ldc_chan_t rx_ack_head field */
139 #define ACKPEEK_HEAD_INVALID ((uint64_t)-1)
140
141
142 /* Module State Pointer */
143 ldc_soft_state_t *ldcssp;
144
145 static struct modldrv md = {
146 &mod_miscops, /* This is a misc module */
147 "sun4v LDC module", /* Name of the module */
148 };
149
150 static struct modlinkage ml = {
151 MODREV_1,
152 &md,
153 NULL
154 };
155
156 static uint64_t ldc_sup_minor; /* Supported minor number */
157 static hsvc_info_t ldc_hsvc = {
158 HSVC_REV_1, NULL, HSVC_GROUP_LDC, 1, 2, "ldc"
159 };
160
161 /*
162 * The no. of MTU size messages that can be stored in
163 * the LDC Tx queue. The number of Tx queue entries is
164 * then computed as (mtu * mtu_msgs)/sizeof(queue_entry)
165 */
166 uint64_t ldc_mtu_msgs = LDC_MTU_MSGS;
167
168 /*
169 * The minimum queue length. This is the size of the smallest
170 * LDC queue. If the computed value is less than this default,
171 * the queue length is rounded up to 'ldc_queue_entries'.
172 */
173 uint64_t ldc_queue_entries = LDC_QUEUE_ENTRIES;
174
175 /*
176 * The length of the reliable-mode data queue in terms of the LDC
177 * receive queue length. i.e., the number of times larger than the
178 * LDC receive queue that the data queue should be. The HV receive
179 * queue is required to be a power of 2 and this implementation
180 * assumes the data queue will also be a power of 2. By making the
181 * multiplier a power of 2, we ensure the data queue will be a
182 * power of 2. We use a multiplier because the receive queue is
183 * sized to be sane relative to the MTU and the same is needed for
184 * the data queue.
185 */
186 uint64_t ldc_rxdq_multiplier = LDC_RXDQ_MULTIPLIER;
187
188 /*
189 * LDC retry count and delay - when the HV returns EWOULDBLOCK
190 * the operation is retried 'ldc_max_retries' times with a
191 * wait of 'ldc_delay' usecs between each retry.
192 */
193 int ldc_max_retries = LDC_MAX_RETRIES;
194 clock_t ldc_delay = LDC_DELAY;
195
196 /*
197 * Channels which have a devclass satisfying the following
198 * will be reset when entering the prom or kmdb.
199 *
200 * LDC_DEVCLASS_PROM_RESET(devclass) != 0
201 *
202 * By default, only block device service channels are reset.
203 */
204 #define LDC_DEVCLASS_BIT(dc) (0x1 << (dc))
205 #define LDC_DEVCLASS_PROM_RESET(dc) \
206 (LDC_DEVCLASS_BIT(dc) & ldc_debug_reset_mask)
207 static uint64_t ldc_debug_reset_mask = LDC_DEVCLASS_BIT(LDC_DEV_BLK_SVC) |
208 LDC_DEVCLASS_BIT(LDC_DEV_GENERIC);
209
210 /*
211 * delay between each retry of channel unregistration in
212 * ldc_close(), to wait for pending interrupts to complete.
213 */
214 clock_t ldc_close_delay = LDC_CLOSE_DELAY;
215
216
217 /*
218 * Reserved mapin space for descriptor rings.
219 */
220 uint64_t ldc_dring_direct_map_rsvd = LDC_DIRECT_MAP_SIZE_DEFAULT;
221
222 /*
223 * Maximum direct map space allowed per channel.
224 */
225 uint64_t ldc_direct_map_size_max = (16 * 1024 * 1024); /* 16 MB */
226
227 #ifdef DEBUG
228
229 /*
230 * Print debug messages
231 *
232 * set ldcdbg to 0x7 for enabling all msgs
233 * 0x4 - Warnings
234 * 0x2 - All debug messages
235 * 0x1 - Minimal debug messages
236 *
237 * set ldcdbgchan to the channel number you want to debug
238 * setting it to -1 prints debug messages for all channels
239 * NOTE: ldcdbgchan has no effect on error messages
240 */
241
242 int ldcdbg = 0x0;
243 int64_t ldcdbgchan = DBG_ALL_LDCS;
244 uint64_t ldc_inject_err_flag = 0;
245
246 void
ldcdebug(int64_t id,const char * fmt,...)247 ldcdebug(int64_t id, const char *fmt, ...)
248 {
249 char buf[512];
250 va_list ap;
251
252 /*
253 * Do not return if,
254 * caller wants to print it anyway - (id == DBG_ALL_LDCS)
255 * debug channel is set to all LDCs - (ldcdbgchan == DBG_ALL_LDCS)
256 * debug channel = caller specified channel
257 */
258 if ((id != DBG_ALL_LDCS) &&
259 (ldcdbgchan != DBG_ALL_LDCS) &&
260 (ldcdbgchan != id)) {
261 return;
262 }
263
264 va_start(ap, fmt);
265 (void) vsprintf(buf, fmt, ap);
266 va_end(ap);
267
268 cmn_err(CE_CONT, "?%s", buf);
269 }
270
271 #define LDC_ERR_RESET 0x1
272 #define LDC_ERR_PKTLOSS 0x2
273 #define LDC_ERR_DQFULL 0x4
274 #define LDC_ERR_DRNGCLEAR 0x8
275
276 static boolean_t
ldc_inject_error(ldc_chan_t * ldcp,uint64_t error)277 ldc_inject_error(ldc_chan_t *ldcp, uint64_t error)
278 {
279 if ((ldcdbgchan != DBG_ALL_LDCS) && (ldcdbgchan != ldcp->id))
280 return (B_FALSE);
281
282 if ((ldc_inject_err_flag & error) == 0)
283 return (B_FALSE);
284
285 /* clear the injection state */
286 ldc_inject_err_flag &= ~error;
287
288 return (B_TRUE);
289 }
290
291 #define DUMP_PAYLOAD(id, addr) \
292 { \
293 char buf[65*3]; \
294 int i; \
295 uint8_t *src = (uint8_t *)addr; \
296 for (i = 0; i < 64; i++, src++) \
297 (void) sprintf(&buf[i * 3], "|%02x", *src); \
298 (void) sprintf(&buf[i * 3], "|\n"); \
299 D2((id), "payload: %s", buf); \
300 }
301
302 #define DUMP_LDC_PKT(c, s, addr) \
303 { \
304 ldc_msg_t *msg = (ldc_msg_t *)(addr); \
305 uint32_t mid = ((c)->mode != LDC_MODE_RAW) ? msg->seqid : 0; \
306 if (msg->type == LDC_DATA) { \
307 D2((c)->id, "%s: msg%d (/%x/%x/%x/,env[%c%c,sz=%d])", \
308 (s), mid, msg->type, msg->stype, msg->ctrl, \
309 (msg->env & LDC_FRAG_START) ? 'B' : ' ', \
310 (msg->env & LDC_FRAG_STOP) ? 'E' : ' ', \
311 (msg->env & LDC_LEN_MASK)); \
312 } else { \
313 D2((c)->id, "%s: msg%d (/%x/%x/%x/,env=%x)", (s), \
314 mid, msg->type, msg->stype, msg->ctrl, msg->env); \
315 } \
316 }
317
318 #define LDC_INJECT_RESET(_ldcp) ldc_inject_error(_ldcp, LDC_ERR_RESET)
319 #define LDC_INJECT_PKTLOSS(_ldcp) ldc_inject_error(_ldcp, LDC_ERR_PKTLOSS)
320 #define LDC_INJECT_DQFULL(_ldcp) ldc_inject_error(_ldcp, LDC_ERR_DQFULL)
321 #define LDC_INJECT_DRNGCLEAR(_ldcp) ldc_inject_error(_ldcp, LDC_ERR_DRNGCLEAR)
322 extern void i_ldc_mem_inject_dring_clear(ldc_chan_t *ldcp);
323
324 #else
325
326 #define DBG_ALL_LDCS -1
327
328 #define DUMP_PAYLOAD(id, addr)
329 #define DUMP_LDC_PKT(c, s, addr)
330
331 #define LDC_INJECT_RESET(_ldcp) (B_FALSE)
332 #define LDC_INJECT_PKTLOSS(_ldcp) (B_FALSE)
333 #define LDC_INJECT_DQFULL(_ldcp) (B_FALSE)
334 #define LDC_INJECT_DRNGCLEAR(_ldcp) (B_FALSE)
335
336 #endif
337
338 /*
339 * dtrace SDT probes to ease tracing of the rx data queue and HV queue
340 * lengths. Just pass the head, tail, and entries values so that the
341 * length can be calculated in a dtrace script when the probe is enabled.
342 */
343 #define TRACE_RXDQ_LENGTH(ldcp) \
344 DTRACE_PROBE4(rxdq__size, \
345 uint64_t, ldcp->id, \
346 uint64_t, ldcp->rx_dq_head, \
347 uint64_t, ldcp->rx_dq_tail, \
348 uint64_t, ldcp->rx_dq_entries)
349
350 #define TRACE_RXHVQ_LENGTH(ldcp, head, tail) \
351 DTRACE_PROBE4(rxhvq__size, \
352 uint64_t, ldcp->id, \
353 uint64_t, head, \
354 uint64_t, tail, \
355 uint64_t, ldcp->rx_q_entries)
356
357 /* A dtrace SDT probe to ease tracing of data queue copy operations */
358 #define TRACE_RXDQ_COPY(ldcp, bytes) \
359 DTRACE_PROBE2(rxdq__copy, uint64_t, ldcp->id, uint64_t, bytes) \
360
361 /* The amount of contiguous space at the tail of the queue */
362 #define Q_CONTIG_SPACE(head, tail, size) \
363 ((head) <= (tail) ? ((size) - (tail)) : \
364 ((head) - (tail) - LDC_PACKET_SIZE))
365
366 #define ZERO_PKT(p) \
367 bzero((p), sizeof (ldc_msg_t));
368
369 #define IDX2COOKIE(idx, pg_szc, pg_shift) \
370 (((pg_szc) << LDC_COOKIE_PGSZC_SHIFT) | ((idx) << (pg_shift)))
371
372 int
_init(void)373 _init(void)
374 {
375 int status;
376
377 status = hsvc_register(&ldc_hsvc, &ldc_sup_minor);
378 if (status != 0) {
379 cmn_err(CE_NOTE, "!%s: cannot negotiate hypervisor LDC services"
380 " group: 0x%lx major: %ld minor: %ld errno: %d",
381 ldc_hsvc.hsvc_modname, ldc_hsvc.hsvc_group,
382 ldc_hsvc.hsvc_major, ldc_hsvc.hsvc_minor, status);
383 return (-1);
384 }
385
386 /* Initialize shared memory HV API version checking */
387 i_ldc_mem_set_hsvc_vers(ldc_hsvc.hsvc_major, ldc_sup_minor);
388
389 /* allocate soft state structure */
390 ldcssp = kmem_zalloc(sizeof (ldc_soft_state_t), KM_SLEEP);
391
392 i_ldc_init_mapin(ldcssp, ldc_hsvc.hsvc_major, ldc_sup_minor);
393
394 /* Link the module into the system */
395 status = mod_install(&ml);
396 if (status != 0) {
397 kmem_free(ldcssp, sizeof (ldc_soft_state_t));
398 return (status);
399 }
400
401 /* Initialize the LDC state structure */
402 mutex_init(&ldcssp->lock, NULL, MUTEX_DRIVER, NULL);
403
404 mutex_enter(&ldcssp->lock);
405
406 /* Create a cache for memory handles */
407 ldcssp->memhdl_cache = kmem_cache_create("ldc_memhdl_cache",
408 sizeof (ldc_mhdl_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
409 if (ldcssp->memhdl_cache == NULL) {
410 DWARN(DBG_ALL_LDCS, "_init: ldc_memhdl cache create failed\n");
411 mutex_exit(&ldcssp->lock);
412 return (-1);
413 }
414
415 /* Create cache for memory segment structures */
416 ldcssp->memseg_cache = kmem_cache_create("ldc_memseg_cache",
417 sizeof (ldc_memseg_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
418 if (ldcssp->memseg_cache == NULL) {
419 DWARN(DBG_ALL_LDCS, "_init: ldc_memseg cache create failed\n");
420 mutex_exit(&ldcssp->lock);
421 return (-1);
422 }
423
424
425 ldcssp->channel_count = 0;
426 ldcssp->channels_open = 0;
427 ldcssp->chan_list = NULL;
428 ldcssp->dring_list = NULL;
429
430 /* Register debug_enter callback */
431 kldc_set_debug_cb(&i_ldc_debug_enter);
432
433 mutex_exit(&ldcssp->lock);
434
435 return (0);
436 }
437
438 int
_info(struct modinfo * modinfop)439 _info(struct modinfo *modinfop)
440 {
441 /* Report status of the dynamically loadable driver module */
442 return (mod_info(&ml, modinfop));
443 }
444
445 int
_fini(void)446 _fini(void)
447 {
448 int rv, status;
449 ldc_chan_t *tmp_ldcp, *ldcp;
450 ldc_dring_t *tmp_dringp, *dringp;
451 ldc_mem_info_t minfo;
452
453 /* Unlink the driver module from the system */
454 status = mod_remove(&ml);
455 if (status) {
456 DWARN(DBG_ALL_LDCS, "_fini: mod_remove failed\n");
457 return (EIO);
458 }
459
460 /* Unregister debug_enter callback */
461 kldc_set_debug_cb(NULL);
462
463 /* Free descriptor rings */
464 dringp = ldcssp->dring_list;
465 while (dringp != NULL) {
466 tmp_dringp = dringp->next;
467
468 rv = ldc_mem_dring_info((ldc_dring_handle_t)dringp, &minfo);
469 if (rv == 0 && minfo.status != LDC_UNBOUND) {
470 if (minfo.status == LDC_BOUND) {
471 (void) ldc_mem_dring_unbind(
472 (ldc_dring_handle_t)dringp);
473 }
474 if (minfo.status == LDC_MAPPED) {
475 (void) ldc_mem_dring_unmap(
476 (ldc_dring_handle_t)dringp);
477 }
478 }
479
480 (void) ldc_mem_dring_destroy((ldc_dring_handle_t)dringp);
481 dringp = tmp_dringp;
482 }
483 ldcssp->dring_list = NULL;
484
485 /* close and finalize channels */
486 ldcp = ldcssp->chan_list;
487 while (ldcp != NULL) {
488 tmp_ldcp = ldcp->next;
489
490 (void) ldc_close((ldc_handle_t)ldcp);
491 (void) ldc_fini((ldc_handle_t)ldcp);
492
493 ldcp = tmp_ldcp;
494 }
495 ldcssp->chan_list = NULL;
496
497 /* Destroy kmem caches */
498 kmem_cache_destroy(ldcssp->memhdl_cache);
499 kmem_cache_destroy(ldcssp->memseg_cache);
500
501 /*
502 * We have successfully "removed" the driver.
503 * Destroying soft states
504 */
505 mutex_destroy(&ldcssp->lock);
506 kmem_free(ldcssp, sizeof (ldc_soft_state_t));
507
508 (void) hsvc_unregister(&ldc_hsvc);
509
510 return (status);
511 }
512
513 /* -------------------------------------------------------------------------- */
514
515 /*
516 * LDC Link Layer Internal Functions
517 */
518
519 /*
520 * Translate HV Errors to sun4v error codes
521 */
522 int
i_ldc_h2v_error(int h_error)523 i_ldc_h2v_error(int h_error)
524 {
525 switch (h_error) {
526
527 case H_EOK:
528 return (0);
529
530 case H_ENORADDR:
531 return (EFAULT);
532
533 case H_EBADPGSZ:
534 case H_EINVAL:
535 return (EINVAL);
536
537 case H_EWOULDBLOCK:
538 return (EWOULDBLOCK);
539
540 case H_ENOACCESS:
541 case H_ENOMAP:
542 return (EACCES);
543
544 case H_EIO:
545 case H_ECPUERROR:
546 return (EIO);
547
548 case H_ENOTSUPPORTED:
549 return (ENOTSUP);
550
551 case H_ETOOMANY:
552 return (ENOSPC);
553
554 case H_ECHANNEL:
555 return (ECHRNG);
556 default:
557 break;
558 }
559
560 return (EIO);
561 }
562
563 /*
564 * Reconfigure the transmit queue
565 */
566 static int
i_ldc_txq_reconf(ldc_chan_t * ldcp)567 i_ldc_txq_reconf(ldc_chan_t *ldcp)
568 {
569 int rv;
570
571 ASSERT(MUTEX_HELD(&ldcp->lock));
572 ASSERT(MUTEX_HELD(&ldcp->tx_lock));
573
574 rv = hv_ldc_tx_qconf(ldcp->id, ldcp->tx_q_ra, ldcp->tx_q_entries);
575 if (rv) {
576 cmn_err(CE_WARN,
577 "i_ldc_txq_reconf: (0x%lx) cannot set qconf", ldcp->id);
578 return (EIO);
579 }
580 rv = hv_ldc_tx_get_state(ldcp->id, &(ldcp->tx_head),
581 &(ldcp->tx_tail), &(ldcp->link_state));
582 if (rv) {
583 cmn_err(CE_WARN,
584 "i_ldc_txq_reconf: (0x%lx) cannot get qptrs", ldcp->id);
585 return (EIO);
586 }
587 D1(ldcp->id, "i_ldc_txq_reconf: (0x%llx) h=0x%llx,t=0x%llx,"
588 "s=0x%llx\n", ldcp->id, ldcp->tx_head, ldcp->tx_tail,
589 ldcp->link_state);
590
591 return (0);
592 }
593
594 /*
595 * Reconfigure the receive queue
596 */
597 static int
i_ldc_rxq_reconf(ldc_chan_t * ldcp,boolean_t force_reset)598 i_ldc_rxq_reconf(ldc_chan_t *ldcp, boolean_t force_reset)
599 {
600 int rv;
601 uint64_t rx_head, rx_tail;
602
603 ASSERT(MUTEX_HELD(&ldcp->lock));
604 rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail,
605 &(ldcp->link_state));
606 if (rv) {
607 cmn_err(CE_WARN,
608 "i_ldc_rxq_reconf: (0x%lx) cannot get state",
609 ldcp->id);
610 return (EIO);
611 }
612
613 if (force_reset || (ldcp->tstate & ~TS_IN_RESET) == TS_UP) {
614 rv = hv_ldc_rx_qconf(ldcp->id, ldcp->rx_q_ra,
615 ldcp->rx_q_entries);
616 if (rv) {
617 cmn_err(CE_WARN,
618 "i_ldc_rxq_reconf: (0x%lx) cannot set qconf",
619 ldcp->id);
620 return (EIO);
621 }
622 D1(ldcp->id, "i_ldc_rxq_reconf: (0x%llx) completed q reconf",
623 ldcp->id);
624 }
625
626 return (0);
627 }
628
629
630 /*
631 * Drain the contents of the receive queue
632 */
633 static void
i_ldc_rxq_drain(ldc_chan_t * ldcp)634 i_ldc_rxq_drain(ldc_chan_t *ldcp)
635 {
636 int rv;
637 uint64_t rx_head, rx_tail;
638 int retries = 0;
639
640 ASSERT(MUTEX_HELD(&ldcp->lock));
641 rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail,
642 &(ldcp->link_state));
643 if (rv) {
644 cmn_err(CE_WARN, "i_ldc_rxq_drain: (0x%lx) cannot get state, "
645 "rv = 0x%x", ldcp->id, rv);
646 return;
647 }
648
649 /* If the queue is already empty just return success. */
650 if (rx_head == rx_tail)
651 return;
652
653 /*
654 * We are draining the queue in order to close the channel.
655 * Call hv_ldc_rx_set_qhead directly instead of i_ldc_set_rx_head
656 * because we do not need to reset the channel if the set
657 * qhead fails.
658 */
659 if ((rv = hv_ldc_rx_set_qhead(ldcp->id, rx_tail)) == 0)
660 return;
661
662 while ((rv == H_EWOULDBLOCK) && (retries++ < ldc_max_retries)) {
663 drv_usecwait(ldc_delay);
664 if ((rv = hv_ldc_rx_set_qhead(ldcp->id, rx_tail)) == 0)
665 return;
666 }
667
668 cmn_err(CE_WARN, "i_ldc_rxq_drain: (0x%lx) cannot set qhead 0x%lx, "
669 "rv = 0x%x", ldcp->id, rx_tail, rv);
670 }
671
672
673 /*
674 * Reset LDC state structure and its contents
675 */
676 static void
i_ldc_reset_state(ldc_chan_t * ldcp)677 i_ldc_reset_state(ldc_chan_t *ldcp)
678 {
679 ASSERT(MUTEX_HELD(&ldcp->lock));
680 ldcp->last_msg_snt = LDC_INIT_SEQID;
681 ldcp->last_ack_rcd = 0;
682 ldcp->last_msg_rcd = 0;
683 ldcp->tx_ackd_head = ldcp->tx_head;
684 ldcp->stream_remains = 0;
685 ldcp->next_vidx = 0;
686 ldcp->hstate = 0;
687 ldcp->tstate = TS_OPEN;
688 ldcp->status = LDC_OPEN;
689 ldcp->rx_ack_head = ACKPEEK_HEAD_INVALID;
690 ldcp->rx_dq_head = 0;
691 ldcp->rx_dq_tail = 0;
692
693 if (ldcp->link_state == LDC_CHANNEL_UP ||
694 ldcp->link_state == LDC_CHANNEL_RESET) {
695
696 if (ldcp->mode == LDC_MODE_RAW) {
697 ldcp->status = LDC_UP;
698 ldcp->tstate = TS_UP;
699 } else {
700 ldcp->status = LDC_READY;
701 ldcp->tstate |= TS_LINK_READY;
702 }
703 }
704 }
705
706 /*
707 * Reset a LDC channel
708 */
709 void
i_ldc_reset(ldc_chan_t * ldcp,boolean_t force_reset)710 i_ldc_reset(ldc_chan_t *ldcp, boolean_t force_reset)
711 {
712 DWARN(ldcp->id, "i_ldc_reset: (0x%llx) channel reset\n", ldcp->id);
713
714 ASSERT(MUTEX_HELD(&ldcp->lock));
715 ASSERT(MUTEX_HELD(&ldcp->tx_lock));
716
717 /* reconfig Tx and Rx queues */
718 (void) i_ldc_txq_reconf(ldcp);
719 (void) i_ldc_rxq_reconf(ldcp, force_reset);
720
721 /* Clear Tx and Rx interrupts */
722 (void) i_ldc_clear_intr(ldcp, CNEX_TX_INTR);
723 (void) i_ldc_clear_intr(ldcp, CNEX_RX_INTR);
724
725 /* Reset channel state */
726 i_ldc_reset_state(ldcp);
727
728 /* Mark channel in reset */
729 ldcp->tstate |= TS_IN_RESET;
730 }
731
732 /*
733 * Walk the channel list and reset channels if they are of the right
734 * devclass and their Rx queues have been configured. No locks are
735 * taken because the function is only invoked by the kernel just before
736 * entering the prom or debugger when the system is single-threaded.
737 */
738 static void
i_ldc_debug_enter(void)739 i_ldc_debug_enter(void)
740 {
741 ldc_chan_t *ldcp;
742
743 ldcp = ldcssp->chan_list;
744 while (ldcp != NULL) {
745 if (((ldcp->tstate & TS_QCONF_RDY) == TS_QCONF_RDY) &&
746 (LDC_DEVCLASS_PROM_RESET(ldcp->devclass) != 0)) {
747 (void) hv_ldc_rx_qconf(ldcp->id, ldcp->rx_q_ra,
748 ldcp->rx_q_entries);
749 }
750 ldcp = ldcp->next;
751 }
752 }
753
754 /*
755 * Clear pending interrupts
756 */
757 static void
i_ldc_clear_intr(ldc_chan_t * ldcp,cnex_intrtype_t itype)758 i_ldc_clear_intr(ldc_chan_t *ldcp, cnex_intrtype_t itype)
759 {
760 ldc_cnex_t *cinfo = &ldcssp->cinfo;
761
762 ASSERT(MUTEX_HELD(&ldcp->lock));
763 ASSERT(cinfo->dip != NULL);
764
765 switch (itype) {
766 case CNEX_TX_INTR:
767 /* check Tx interrupt */
768 if (ldcp->tx_intr_state)
769 ldcp->tx_intr_state = LDC_INTR_NONE;
770 else
771 return;
772 break;
773
774 case CNEX_RX_INTR:
775 /* check Rx interrupt */
776 if (ldcp->rx_intr_state)
777 ldcp->rx_intr_state = LDC_INTR_NONE;
778 else
779 return;
780 break;
781 }
782
783 (void) cinfo->clr_intr(cinfo->dip, ldcp->id, itype);
784 D2(ldcp->id,
785 "i_ldc_clear_intr: (0x%llx) cleared 0x%x intr\n",
786 ldcp->id, itype);
787 }
788
789 /*
790 * Set the receive queue head
791 * Resets connection and returns an error if it fails.
792 */
793 static int
i_ldc_set_rx_head(ldc_chan_t * ldcp,uint64_t head)794 i_ldc_set_rx_head(ldc_chan_t *ldcp, uint64_t head)
795 {
796 int rv;
797 int retries;
798
799 ASSERT(MUTEX_HELD(&ldcp->lock));
800 for (retries = 0; retries < ldc_max_retries; retries++) {
801
802 if ((rv = hv_ldc_rx_set_qhead(ldcp->id, head)) == 0)
803 return (0);
804
805 if (rv != H_EWOULDBLOCK)
806 break;
807
808 /* wait for ldc_delay usecs */
809 drv_usecwait(ldc_delay);
810 }
811
812 cmn_err(CE_WARN, "ldc_set_rx_qhead: (0x%lx) cannot set qhead 0x%lx, "
813 "rv = 0x%x", ldcp->id, head, rv);
814 mutex_enter(&ldcp->tx_lock);
815 i_ldc_reset(ldcp, B_TRUE);
816 mutex_exit(&ldcp->tx_lock);
817
818 return (ECONNRESET);
819 }
820
821 /*
822 * Returns the tx_head to be used for transfer
823 */
824 static void
i_ldc_get_tx_head(ldc_chan_t * ldcp,uint64_t * head)825 i_ldc_get_tx_head(ldc_chan_t *ldcp, uint64_t *head)
826 {
827 ldc_msg_t *pkt;
828
829 ASSERT(MUTEX_HELD(&ldcp->tx_lock));
830
831 /* get current Tx head */
832 *head = ldcp->tx_head;
833
834 /*
835 * Reliable mode will use the ACKd head instead of the regular tx_head.
836 * Also in Reliable mode, advance ackd_head for all non DATA/INFO pkts,
837 * up to the current location of tx_head. This needs to be done
838 * as the peer will only ACK DATA/INFO pkts.
839 */
840 if (ldcp->mode == LDC_MODE_RELIABLE) {
841 while (ldcp->tx_ackd_head != ldcp->tx_head) {
842 pkt = (ldc_msg_t *)(ldcp->tx_q_va + ldcp->tx_ackd_head);
843 if ((pkt->type & LDC_DATA) && (pkt->stype & LDC_INFO)) {
844 break;
845 }
846 /* advance ACKd head */
847 ldcp->tx_ackd_head =
848 (ldcp->tx_ackd_head + LDC_PACKET_SIZE) %
849 (ldcp->tx_q_entries << LDC_PACKET_SHIFT);
850 }
851 *head = ldcp->tx_ackd_head;
852 }
853 }
854
855 /*
856 * Returns the tx_tail to be used for transfer
857 * Re-reads the TX queue ptrs if and only if the
858 * the cached head and tail are equal (queue is full)
859 */
860 static int
i_ldc_get_tx_tail(ldc_chan_t * ldcp,uint64_t * tail)861 i_ldc_get_tx_tail(ldc_chan_t *ldcp, uint64_t *tail)
862 {
863 int rv;
864 uint64_t current_head, new_tail;
865
866 ASSERT(MUTEX_HELD(&ldcp->tx_lock));
867 /* Read the head and tail ptrs from HV */
868 rv = hv_ldc_tx_get_state(ldcp->id,
869 &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state);
870 if (rv) {
871 cmn_err(CE_WARN,
872 "i_ldc_get_tx_tail: (0x%lx) cannot read qptrs\n",
873 ldcp->id);
874 return (EIO);
875 }
876 if (ldcp->link_state == LDC_CHANNEL_DOWN) {
877 D1(ldcp->id, "i_ldc_get_tx_tail: (0x%llx) channel not ready\n",
878 ldcp->id);
879 return (ECONNRESET);
880 }
881
882 i_ldc_get_tx_head(ldcp, ¤t_head);
883
884 /* increment the tail */
885 new_tail = (ldcp->tx_tail + LDC_PACKET_SIZE) %
886 (ldcp->tx_q_entries << LDC_PACKET_SHIFT);
887
888 if (new_tail == current_head) {
889 DWARN(ldcp->id,
890 "i_ldc_get_tx_tail: (0x%llx) TX queue is full\n",
891 ldcp->id);
892 return (EWOULDBLOCK);
893 }
894
895 D2(ldcp->id, "i_ldc_get_tx_tail: (0x%llx) head=0x%llx, tail=0x%llx\n",
896 ldcp->id, ldcp->tx_head, ldcp->tx_tail);
897
898 *tail = ldcp->tx_tail;
899 return (0);
900 }
901
902 /*
903 * Set the tail pointer. If HV returns EWOULDBLOCK, it will back off
904 * and retry ldc_max_retries times before returning an error.
905 * Returns 0, EWOULDBLOCK or EIO
906 */
907 static int
i_ldc_set_tx_tail(ldc_chan_t * ldcp,uint64_t tail)908 i_ldc_set_tx_tail(ldc_chan_t *ldcp, uint64_t tail)
909 {
910 int rv, retval = EWOULDBLOCK;
911 int retries;
912
913 ASSERT(MUTEX_HELD(&ldcp->tx_lock));
914 for (retries = 0; retries < ldc_max_retries; retries++) {
915
916 if ((rv = hv_ldc_tx_set_qtail(ldcp->id, tail)) == 0) {
917 retval = 0;
918 break;
919 }
920 if (rv != H_EWOULDBLOCK) {
921 DWARN(ldcp->id, "i_ldc_set_tx_tail: (0x%llx) set "
922 "qtail=0x%llx failed, rv=%d\n", ldcp->id, tail, rv);
923 retval = EIO;
924 break;
925 }
926
927 /* wait for ldc_delay usecs */
928 drv_usecwait(ldc_delay);
929 }
930 return (retval);
931 }
932
933 /*
934 * Copy a data packet from the HV receive queue to the data queue.
935 * Caller must ensure that the data queue is not already full.
936 *
937 * The *head argument represents the current head pointer for the HV
938 * receive queue. After copying a packet from the HV receive queue,
939 * the *head pointer will be updated. This allows the caller to update
940 * the head pointer in HV using the returned *head value.
941 */
942 void
i_ldc_rxdq_copy(ldc_chan_t * ldcp,uint64_t * head)943 i_ldc_rxdq_copy(ldc_chan_t *ldcp, uint64_t *head)
944 {
945 uint64_t q_size, dq_size;
946
947 ASSERT(MUTEX_HELD(&ldcp->lock));
948
949 q_size = ldcp->rx_q_entries << LDC_PACKET_SHIFT;
950 dq_size = ldcp->rx_dq_entries << LDC_PACKET_SHIFT;
951
952 ASSERT(Q_CONTIG_SPACE(ldcp->rx_dq_head, ldcp->rx_dq_tail,
953 dq_size) >= LDC_PACKET_SIZE);
954
955 bcopy((void *)(ldcp->rx_q_va + *head),
956 (void *)(ldcp->rx_dq_va + ldcp->rx_dq_tail), LDC_PACKET_SIZE);
957 TRACE_RXDQ_COPY(ldcp, LDC_PACKET_SIZE);
958
959 /* Update rx head */
960 *head = (*head + LDC_PACKET_SIZE) % q_size;
961
962 /* Update dq tail */
963 ldcp->rx_dq_tail = (ldcp->rx_dq_tail + LDC_PACKET_SIZE) % dq_size;
964 }
965
966 /*
967 * Update the Rx data queue head pointer
968 */
969 static int
i_ldc_set_rxdq_head(ldc_chan_t * ldcp,uint64_t head)970 i_ldc_set_rxdq_head(ldc_chan_t *ldcp, uint64_t head)
971 {
972 ldcp->rx_dq_head = head;
973 return (0);
974 }
975
976 /*
977 * Get the Rx data queue head and tail pointers
978 */
979 static uint64_t
i_ldc_dq_rx_get_state(ldc_chan_t * ldcp,uint64_t * head,uint64_t * tail,uint64_t * link_state)980 i_ldc_dq_rx_get_state(ldc_chan_t *ldcp, uint64_t *head, uint64_t *tail,
981 uint64_t *link_state)
982 {
983 _NOTE(ARGUNUSED(link_state))
984 *head = ldcp->rx_dq_head;
985 *tail = ldcp->rx_dq_tail;
986 return (0);
987 }
988
989 /*
990 * Wrapper for the Rx HV queue set head function. Giving the
991 * data queue and HV queue set head functions the same type.
992 */
993 static uint64_t
i_ldc_hvq_rx_get_state(ldc_chan_t * ldcp,uint64_t * head,uint64_t * tail,uint64_t * link_state)994 i_ldc_hvq_rx_get_state(ldc_chan_t *ldcp, uint64_t *head, uint64_t *tail,
995 uint64_t *link_state)
996 {
997 return (i_ldc_h2v_error(hv_ldc_rx_get_state(ldcp->id, head, tail,
998 link_state)));
999 }
1000
1001 /*
1002 * LDC receive interrupt handler
1003 * triggered for channel with data pending to read
1004 * i.e. Rx queue content changes
1005 */
1006 static uint_t
i_ldc_rx_hdlr(caddr_t arg1,caddr_t arg2)1007 i_ldc_rx_hdlr(caddr_t arg1, caddr_t arg2)
1008 {
1009 _NOTE(ARGUNUSED(arg2))
1010
1011 ldc_chan_t *ldcp;
1012 boolean_t notify;
1013 uint64_t event;
1014 int rv, status;
1015
1016 /* Get the channel for which interrupt was received */
1017 if (arg1 == NULL) {
1018 cmn_err(CE_WARN, "i_ldc_rx_hdlr: invalid arg\n");
1019 return (DDI_INTR_UNCLAIMED);
1020 }
1021
1022 ldcp = (ldc_chan_t *)arg1;
1023
1024 D1(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) Received intr, ldcp=0x%p\n",
1025 ldcp->id, ldcp);
1026 D1(ldcp->id, "i_ldc_rx_hdlr: (%llx) USR%lx/TS%lx/HS%lx, LSTATE=%lx\n",
1027 ldcp->id, ldcp->status, ldcp->tstate, ldcp->hstate,
1028 ldcp->link_state);
1029
1030 /* Lock channel */
1031 mutex_enter(&ldcp->lock);
1032
1033 /* Mark the interrupt as being actively handled */
1034 ldcp->rx_intr_state = LDC_INTR_ACTIVE;
1035
1036 status = i_ldc_rx_process_hvq(ldcp, ¬ify, &event);
1037
1038 if (ldcp->mode != LDC_MODE_RELIABLE) {
1039 /*
1040 * If there are no data packets on the queue, clear
1041 * the interrupt. Otherwise, the ldc_read will clear
1042 * interrupts after draining the queue. To indicate the
1043 * interrupt has not yet been cleared, it is marked
1044 * as pending.
1045 */
1046 if ((event & LDC_EVT_READ) == 0) {
1047 i_ldc_clear_intr(ldcp, CNEX_RX_INTR);
1048 } else {
1049 ldcp->rx_intr_state = LDC_INTR_PEND;
1050 }
1051 }
1052
1053 /* if callbacks are disabled, do not notify */
1054 if (notify && ldcp->cb_enabled) {
1055 ldcp->cb_inprogress = B_TRUE;
1056 mutex_exit(&ldcp->lock);
1057 rv = ldcp->cb(event, ldcp->cb_arg);
1058 if (rv) {
1059 DWARN(ldcp->id,
1060 "i_ldc_rx_hdlr: (0x%llx) callback failure",
1061 ldcp->id);
1062 }
1063 mutex_enter(&ldcp->lock);
1064 ldcp->cb_inprogress = B_FALSE;
1065 }
1066
1067 if (ldcp->mode == LDC_MODE_RELIABLE) {
1068 if (status == ENOSPC) {
1069 /*
1070 * Here, ENOSPC indicates the secondary data
1071 * queue is full and the Rx queue is non-empty.
1072 * Much like how reliable and raw modes are
1073 * handled above, since the Rx queue is non-
1074 * empty, we mark the interrupt as pending to
1075 * indicate it has not yet been cleared.
1076 */
1077 ldcp->rx_intr_state = LDC_INTR_PEND;
1078 } else {
1079 /*
1080 * We have processed all CTRL packets and
1081 * copied all DATA packets to the secondary
1082 * queue. Clear the interrupt.
1083 */
1084 i_ldc_clear_intr(ldcp, CNEX_RX_INTR);
1085 }
1086 }
1087
1088 mutex_exit(&ldcp->lock);
1089
1090 D1(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) exiting handler", ldcp->id);
1091
1092 return (DDI_INTR_CLAIMED);
1093 }
1094
1095 /*
1096 * Wrapper for the Rx HV queue processing function to be used when
1097 * checking the Rx HV queue for data packets. Unlike the interrupt
1098 * handler code flow, the Rx interrupt is not cleared here and
1099 * callbacks are not made.
1100 */
1101 static uint_t
i_ldc_chkq(ldc_chan_t * ldcp)1102 i_ldc_chkq(ldc_chan_t *ldcp)
1103 {
1104 boolean_t notify;
1105 uint64_t event;
1106
1107 return (i_ldc_rx_process_hvq(ldcp, ¬ify, &event));
1108 }
1109
1110 /*
1111 * Send a LDC message
1112 */
1113 static int
i_ldc_send_pkt(ldc_chan_t * ldcp,uint8_t pkttype,uint8_t subtype,uint8_t ctrlmsg)1114 i_ldc_send_pkt(ldc_chan_t *ldcp, uint8_t pkttype, uint8_t subtype,
1115 uint8_t ctrlmsg)
1116 {
1117 int rv;
1118 ldc_msg_t *pkt;
1119 uint64_t tx_tail;
1120 uint32_t curr_seqid;
1121
1122 /* Obtain Tx lock */
1123 mutex_enter(&ldcp->tx_lock);
1124
1125 curr_seqid = ldcp->last_msg_snt;
1126
1127 /* get the current tail for the message */
1128 rv = i_ldc_get_tx_tail(ldcp, &tx_tail);
1129 if (rv) {
1130 DWARN(ldcp->id,
1131 "i_ldc_send_pkt: (0x%llx) error sending pkt, "
1132 "type=0x%x,subtype=0x%x,ctrl=0x%x\n",
1133 ldcp->id, pkttype, subtype, ctrlmsg);
1134 mutex_exit(&ldcp->tx_lock);
1135 return (rv);
1136 }
1137
1138 pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail);
1139 ZERO_PKT(pkt);
1140
1141 /* Initialize the packet */
1142 pkt->type = pkttype;
1143 pkt->stype = subtype;
1144 pkt->ctrl = ctrlmsg;
1145
1146 /* Store ackid/seqid iff it is RELIABLE mode & not a RTS/RTR message */
1147 if (((ctrlmsg & LDC_CTRL_MASK) != LDC_RTS) &&
1148 ((ctrlmsg & LDC_CTRL_MASK) != LDC_RTR)) {
1149 curr_seqid++;
1150 if (ldcp->mode != LDC_MODE_RAW) {
1151 pkt->seqid = curr_seqid;
1152 pkt->ackid = ldcp->last_msg_rcd;
1153 }
1154 }
1155 DUMP_LDC_PKT(ldcp, "i_ldc_send_pkt", (uint64_t)pkt);
1156
1157 /* initiate the send by calling into HV and set the new tail */
1158 tx_tail = (tx_tail + LDC_PACKET_SIZE) %
1159 (ldcp->tx_q_entries << LDC_PACKET_SHIFT);
1160
1161 rv = i_ldc_set_tx_tail(ldcp, tx_tail);
1162 if (rv) {
1163 DWARN(ldcp->id,
1164 "i_ldc_send_pkt:(0x%llx) error sending pkt, "
1165 "type=0x%x,stype=0x%x,ctrl=0x%x\n",
1166 ldcp->id, pkttype, subtype, ctrlmsg);
1167 mutex_exit(&ldcp->tx_lock);
1168 return (EIO);
1169 }
1170
1171 ldcp->last_msg_snt = curr_seqid;
1172 ldcp->tx_tail = tx_tail;
1173
1174 mutex_exit(&ldcp->tx_lock);
1175 return (0);
1176 }
1177
1178 /*
1179 * Checks if packet was received in right order
1180 * in the case of a reliable link.
1181 * Returns 0 if in order, else EIO
1182 */
1183 static int
i_ldc_check_seqid(ldc_chan_t * ldcp,ldc_msg_t * msg)1184 i_ldc_check_seqid(ldc_chan_t *ldcp, ldc_msg_t *msg)
1185 {
1186 /* No seqid checking for RAW mode */
1187 if (ldcp->mode == LDC_MODE_RAW)
1188 return (0);
1189
1190 /* No seqid checking for version, RTS, RTR message */
1191 if (msg->ctrl == LDC_VER ||
1192 msg->ctrl == LDC_RTS ||
1193 msg->ctrl == LDC_RTR)
1194 return (0);
1195
1196 /* Initial seqid to use is sent in RTS/RTR and saved in last_msg_rcd */
1197 if (msg->seqid != (ldcp->last_msg_rcd + 1)) {
1198 DWARN(ldcp->id,
1199 "i_ldc_check_seqid: (0x%llx) out-of-order pkt, got 0x%x, "
1200 "expecting 0x%x\n", ldcp->id, msg->seqid,
1201 (ldcp->last_msg_rcd + 1));
1202 return (EIO);
1203 }
1204
1205 #ifdef DEBUG
1206 if (LDC_INJECT_PKTLOSS(ldcp)) {
1207 DWARN(ldcp->id,
1208 "i_ldc_check_seqid: (0x%llx) inject pkt loss\n", ldcp->id);
1209 return (EIO);
1210 }
1211 #endif
1212
1213 return (0);
1214 }
1215
1216
1217 /*
1218 * Process an incoming version ctrl message
1219 */
1220 static int
i_ldc_process_VER(ldc_chan_t * ldcp,ldc_msg_t * msg)1221 i_ldc_process_VER(ldc_chan_t *ldcp, ldc_msg_t *msg)
1222 {
1223 int rv = 0, idx = ldcp->next_vidx;
1224 ldc_msg_t *pkt;
1225 uint64_t tx_tail;
1226 ldc_ver_t *rcvd_ver;
1227
1228 /* get the received version */
1229 rcvd_ver = (ldc_ver_t *)((uint64_t)msg + LDC_PAYLOAD_VER_OFF);
1230
1231 D2(ldcp->id, "i_ldc_process_VER: (0x%llx) received VER v%u.%u\n",
1232 ldcp->id, rcvd_ver->major, rcvd_ver->minor);
1233
1234 /* Obtain Tx lock */
1235 mutex_enter(&ldcp->tx_lock);
1236
1237 switch (msg->stype) {
1238 case LDC_INFO:
1239
1240 if ((ldcp->tstate & ~TS_IN_RESET) == TS_VREADY) {
1241 (void) i_ldc_txq_reconf(ldcp);
1242 i_ldc_reset_state(ldcp);
1243 mutex_exit(&ldcp->tx_lock);
1244 return (EAGAIN);
1245 }
1246
1247 /* get the current tail and pkt for the response */
1248 rv = i_ldc_get_tx_tail(ldcp, &tx_tail);
1249 if (rv != 0) {
1250 DWARN(ldcp->id,
1251 "i_ldc_process_VER: (0x%llx) err sending "
1252 "version ACK/NACK\n", ldcp->id);
1253 i_ldc_reset(ldcp, B_TRUE);
1254 mutex_exit(&ldcp->tx_lock);
1255 return (ECONNRESET);
1256 }
1257
1258 pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail);
1259 ZERO_PKT(pkt);
1260
1261 /* initialize the packet */
1262 pkt->type = LDC_CTRL;
1263 pkt->ctrl = LDC_VER;
1264
1265 for (;;) {
1266
1267 D1(ldcp->id, "i_ldc_process_VER: got %u.%u chk %u.%u\n",
1268 rcvd_ver->major, rcvd_ver->minor,
1269 ldc_versions[idx].major, ldc_versions[idx].minor);
1270
1271 if (rcvd_ver->major == ldc_versions[idx].major) {
1272 /* major version match - ACK version */
1273 pkt->stype = LDC_ACK;
1274
1275 /*
1276 * lower minor version to the one this endpt
1277 * supports, if necessary
1278 */
1279 if (rcvd_ver->minor > ldc_versions[idx].minor)
1280 rcvd_ver->minor =
1281 ldc_versions[idx].minor;
1282 bcopy(rcvd_ver, pkt->udata, sizeof (*rcvd_ver));
1283
1284 break;
1285 }
1286
1287 if (rcvd_ver->major > ldc_versions[idx].major) {
1288
1289 D1(ldcp->id, "i_ldc_process_VER: using next"
1290 " lower idx=%d, v%u.%u\n", idx,
1291 ldc_versions[idx].major,
1292 ldc_versions[idx].minor);
1293
1294 /* nack with next lower version */
1295 pkt->stype = LDC_NACK;
1296 bcopy(&ldc_versions[idx], pkt->udata,
1297 sizeof (ldc_versions[idx]));
1298 ldcp->next_vidx = idx;
1299 break;
1300 }
1301
1302 /* next major version */
1303 idx++;
1304
1305 D1(ldcp->id, "i_ldc_process_VER: inc idx %x\n", idx);
1306
1307 if (idx == LDC_NUM_VERS) {
1308 /* no version match - send NACK */
1309 pkt->stype = LDC_NACK;
1310 bzero(pkt->udata, sizeof (ldc_ver_t));
1311 ldcp->next_vidx = 0;
1312 break;
1313 }
1314 }
1315
1316 /* initiate the send by calling into HV and set the new tail */
1317 tx_tail = (tx_tail + LDC_PACKET_SIZE) %
1318 (ldcp->tx_q_entries << LDC_PACKET_SHIFT);
1319
1320 rv = i_ldc_set_tx_tail(ldcp, tx_tail);
1321 if (rv == 0) {
1322 ldcp->tx_tail = tx_tail;
1323 if (pkt->stype == LDC_ACK) {
1324 D2(ldcp->id, "i_ldc_process_VER: (0x%llx) sent"
1325 " version ACK\n", ldcp->id);
1326 /* Save the ACK'd version */
1327 ldcp->version.major = rcvd_ver->major;
1328 ldcp->version.minor = rcvd_ver->minor;
1329 ldcp->hstate |= TS_RCVD_VER;
1330 ldcp->tstate |= TS_VER_DONE;
1331 D1(DBG_ALL_LDCS,
1332 "(0x%llx) Sent ACK, "
1333 "Agreed on version v%u.%u\n",
1334 ldcp->id, rcvd_ver->major, rcvd_ver->minor);
1335 }
1336 } else {
1337 DWARN(ldcp->id,
1338 "i_ldc_process_VER: (0x%llx) error sending "
1339 "ACK/NACK\n", ldcp->id);
1340 i_ldc_reset(ldcp, B_TRUE);
1341 mutex_exit(&ldcp->tx_lock);
1342 return (ECONNRESET);
1343 }
1344
1345 break;
1346
1347 case LDC_ACK:
1348 if ((ldcp->tstate & ~TS_IN_RESET) == TS_VREADY) {
1349 if (ldcp->version.major != rcvd_ver->major ||
1350 ldcp->version.minor != rcvd_ver->minor) {
1351
1352 /* mismatched version - reset connection */
1353 DWARN(ldcp->id,
1354 "i_ldc_process_VER: (0x%llx) recvd"
1355 " ACK ver != sent ACK ver\n", ldcp->id);
1356 i_ldc_reset(ldcp, B_TRUE);
1357 mutex_exit(&ldcp->tx_lock);
1358 return (ECONNRESET);
1359 }
1360 } else {
1361 /* SUCCESS - we have agreed on a version */
1362 ldcp->version.major = rcvd_ver->major;
1363 ldcp->version.minor = rcvd_ver->minor;
1364 ldcp->tstate |= TS_VER_DONE;
1365 }
1366
1367 D1(ldcp->id, "(0x%llx) Got ACK, Agreed on version v%u.%u\n",
1368 ldcp->id, rcvd_ver->major, rcvd_ver->minor);
1369
1370 /* initiate RTS-RTR-RDX handshake */
1371 rv = i_ldc_get_tx_tail(ldcp, &tx_tail);
1372 if (rv) {
1373 DWARN(ldcp->id,
1374 "i_ldc_process_VER: (0x%llx) cannot send RTS\n",
1375 ldcp->id);
1376 i_ldc_reset(ldcp, B_TRUE);
1377 mutex_exit(&ldcp->tx_lock);
1378 return (ECONNRESET);
1379 }
1380
1381 pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail);
1382 ZERO_PKT(pkt);
1383
1384 pkt->type = LDC_CTRL;
1385 pkt->stype = LDC_INFO;
1386 pkt->ctrl = LDC_RTS;
1387 pkt->env = ldcp->mode;
1388 if (ldcp->mode != LDC_MODE_RAW)
1389 pkt->seqid = LDC_INIT_SEQID;
1390
1391 ldcp->last_msg_rcd = LDC_INIT_SEQID;
1392
1393 DUMP_LDC_PKT(ldcp, "i_ldc_process_VER snd rts", (uint64_t)pkt);
1394
1395 /* initiate the send by calling into HV and set the new tail */
1396 tx_tail = (tx_tail + LDC_PACKET_SIZE) %
1397 (ldcp->tx_q_entries << LDC_PACKET_SHIFT);
1398
1399 rv = i_ldc_set_tx_tail(ldcp, tx_tail);
1400 if (rv) {
1401 D2(ldcp->id,
1402 "i_ldc_process_VER: (0x%llx) no listener\n",
1403 ldcp->id);
1404 i_ldc_reset(ldcp, B_TRUE);
1405 mutex_exit(&ldcp->tx_lock);
1406 return (ECONNRESET);
1407 }
1408
1409 ldcp->tx_tail = tx_tail;
1410 ldcp->hstate |= TS_SENT_RTS;
1411
1412 break;
1413
1414 case LDC_NACK:
1415 /* check if version in NACK is zero */
1416 if (rcvd_ver->major == 0 && rcvd_ver->minor == 0) {
1417 /* version handshake failure */
1418 DWARN(DBG_ALL_LDCS,
1419 "i_ldc_process_VER: (0x%llx) no version match\n",
1420 ldcp->id);
1421 i_ldc_reset(ldcp, B_TRUE);
1422 mutex_exit(&ldcp->tx_lock);
1423 return (ECONNRESET);
1424 }
1425
1426 /* get the current tail and pkt for the response */
1427 rv = i_ldc_get_tx_tail(ldcp, &tx_tail);
1428 if (rv != 0) {
1429 cmn_err(CE_NOTE,
1430 "i_ldc_process_VER: (0x%lx) err sending "
1431 "version ACK/NACK\n", ldcp->id);
1432 i_ldc_reset(ldcp, B_TRUE);
1433 mutex_exit(&ldcp->tx_lock);
1434 return (ECONNRESET);
1435 }
1436
1437 pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail);
1438 ZERO_PKT(pkt);
1439
1440 /* initialize the packet */
1441 pkt->type = LDC_CTRL;
1442 pkt->ctrl = LDC_VER;
1443 pkt->stype = LDC_INFO;
1444
1445 /* check ver in NACK msg has a match */
1446 for (;;) {
1447 if (rcvd_ver->major == ldc_versions[idx].major) {
1448 /*
1449 * major version match - resubmit request
1450 * if lower minor version to the one this endpt
1451 * supports, if necessary
1452 */
1453 if (rcvd_ver->minor > ldc_versions[idx].minor)
1454 rcvd_ver->minor =
1455 ldc_versions[idx].minor;
1456 bcopy(rcvd_ver, pkt->udata, sizeof (*rcvd_ver));
1457 break;
1458 }
1459
1460 if (rcvd_ver->major > ldc_versions[idx].major) {
1461
1462 D1(ldcp->id, "i_ldc_process_VER: using next"
1463 " lower idx=%d, v%u.%u\n", idx,
1464 ldc_versions[idx].major,
1465 ldc_versions[idx].minor);
1466
1467 /* send next lower version */
1468 bcopy(&ldc_versions[idx], pkt->udata,
1469 sizeof (ldc_versions[idx]));
1470 ldcp->next_vidx = idx;
1471 break;
1472 }
1473
1474 /* next version */
1475 idx++;
1476
1477 D1(ldcp->id, "i_ldc_process_VER: inc idx %x\n", idx);
1478
1479 if (idx == LDC_NUM_VERS) {
1480 /* no version match - terminate */
1481 ldcp->next_vidx = 0;
1482 mutex_exit(&ldcp->tx_lock);
1483 return (ECONNRESET);
1484 }
1485 }
1486
1487 /* initiate the send by calling into HV and set the new tail */
1488 tx_tail = (tx_tail + LDC_PACKET_SIZE) %
1489 (ldcp->tx_q_entries << LDC_PACKET_SHIFT);
1490
1491 rv = i_ldc_set_tx_tail(ldcp, tx_tail);
1492 if (rv == 0) {
1493 D2(ldcp->id, "i_ldc_process_VER: (0x%llx) sent version"
1494 "INFO v%u.%u\n", ldcp->id, ldc_versions[idx].major,
1495 ldc_versions[idx].minor);
1496 ldcp->tx_tail = tx_tail;
1497 } else {
1498 cmn_err(CE_NOTE,
1499 "i_ldc_process_VER: (0x%lx) error sending version"
1500 "INFO\n", ldcp->id);
1501 i_ldc_reset(ldcp, B_TRUE);
1502 mutex_exit(&ldcp->tx_lock);
1503 return (ECONNRESET);
1504 }
1505
1506 break;
1507 }
1508
1509 mutex_exit(&ldcp->tx_lock);
1510 return (rv);
1511 }
1512
1513
1514 /*
1515 * Process an incoming RTS ctrl message
1516 */
1517 static int
i_ldc_process_RTS(ldc_chan_t * ldcp,ldc_msg_t * msg)1518 i_ldc_process_RTS(ldc_chan_t *ldcp, ldc_msg_t *msg)
1519 {
1520 int rv = 0;
1521 ldc_msg_t *pkt;
1522 uint64_t tx_tail;
1523 boolean_t sent_NACK = B_FALSE;
1524
1525 D2(ldcp->id, "i_ldc_process_RTS: (0x%llx) received RTS\n", ldcp->id);
1526
1527 switch (msg->stype) {
1528 case LDC_NACK:
1529 DWARN(ldcp->id,
1530 "i_ldc_process_RTS: (0x%llx) RTS NACK received\n",
1531 ldcp->id);
1532
1533 /* Reset the channel -- as we cannot continue */
1534 mutex_enter(&ldcp->tx_lock);
1535 i_ldc_reset(ldcp, B_TRUE);
1536 mutex_exit(&ldcp->tx_lock);
1537 rv = ECONNRESET;
1538 break;
1539
1540 case LDC_INFO:
1541
1542 /* check mode */
1543 if (ldcp->mode != (ldc_mode_t)msg->env) {
1544 cmn_err(CE_NOTE,
1545 "i_ldc_process_RTS: (0x%lx) mode mismatch\n",
1546 ldcp->id);
1547 /*
1548 * send NACK in response to MODE message
1549 * get the current tail for the response
1550 */
1551 rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_NACK, LDC_RTS);
1552 if (rv) {
1553 /* if cannot send NACK - reset channel */
1554 mutex_enter(&ldcp->tx_lock);
1555 i_ldc_reset(ldcp, B_TRUE);
1556 mutex_exit(&ldcp->tx_lock);
1557 rv = ECONNRESET;
1558 break;
1559 }
1560 sent_NACK = B_TRUE;
1561 }
1562 break;
1563 default:
1564 DWARN(ldcp->id, "i_ldc_process_RTS: (0x%llx) unexp ACK\n",
1565 ldcp->id);
1566 mutex_enter(&ldcp->tx_lock);
1567 i_ldc_reset(ldcp, B_TRUE);
1568 mutex_exit(&ldcp->tx_lock);
1569 rv = ECONNRESET;
1570 break;
1571 }
1572
1573 /*
1574 * If either the connection was reset (when rv != 0) or
1575 * a NACK was sent, we return. In the case of a NACK
1576 * we dont want to consume the packet that came in but
1577 * not record that we received the RTS
1578 */
1579 if (rv || sent_NACK)
1580 return (rv);
1581
1582 /* record RTS received */
1583 ldcp->hstate |= TS_RCVD_RTS;
1584
1585 /* store initial SEQID info */
1586 ldcp->last_msg_snt = msg->seqid;
1587
1588 /* Obtain Tx lock */
1589 mutex_enter(&ldcp->tx_lock);
1590
1591 /* get the current tail for the response */
1592 rv = i_ldc_get_tx_tail(ldcp, &tx_tail);
1593 if (rv != 0) {
1594 cmn_err(CE_NOTE,
1595 "i_ldc_process_RTS: (0x%lx) err sending RTR\n",
1596 ldcp->id);
1597 i_ldc_reset(ldcp, B_TRUE);
1598 mutex_exit(&ldcp->tx_lock);
1599 return (ECONNRESET);
1600 }
1601
1602 pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail);
1603 ZERO_PKT(pkt);
1604
1605 /* initialize the packet */
1606 pkt->type = LDC_CTRL;
1607 pkt->stype = LDC_INFO;
1608 pkt->ctrl = LDC_RTR;
1609 pkt->env = ldcp->mode;
1610 if (ldcp->mode != LDC_MODE_RAW)
1611 pkt->seqid = LDC_INIT_SEQID;
1612
1613 ldcp->last_msg_rcd = msg->seqid;
1614
1615 /* initiate the send by calling into HV and set the new tail */
1616 tx_tail = (tx_tail + LDC_PACKET_SIZE) %
1617 (ldcp->tx_q_entries << LDC_PACKET_SHIFT);
1618
1619 rv = i_ldc_set_tx_tail(ldcp, tx_tail);
1620 if (rv == 0) {
1621 D2(ldcp->id,
1622 "i_ldc_process_RTS: (0x%llx) sent RTR\n", ldcp->id);
1623 DUMP_LDC_PKT(ldcp, "i_ldc_process_RTS sent rtr", (uint64_t)pkt);
1624
1625 ldcp->tx_tail = tx_tail;
1626 ldcp->hstate |= TS_SENT_RTR;
1627
1628 } else {
1629 cmn_err(CE_NOTE,
1630 "i_ldc_process_RTS: (0x%lx) error sending RTR\n",
1631 ldcp->id);
1632 i_ldc_reset(ldcp, B_TRUE);
1633 mutex_exit(&ldcp->tx_lock);
1634 return (ECONNRESET);
1635 }
1636
1637 mutex_exit(&ldcp->tx_lock);
1638 return (0);
1639 }
1640
1641 /*
1642 * Process an incoming RTR ctrl message
1643 */
1644 static int
i_ldc_process_RTR(ldc_chan_t * ldcp,ldc_msg_t * msg)1645 i_ldc_process_RTR(ldc_chan_t *ldcp, ldc_msg_t *msg)
1646 {
1647 int rv = 0;
1648 boolean_t sent_NACK = B_FALSE;
1649
1650 D2(ldcp->id, "i_ldc_process_RTR: (0x%llx) received RTR\n", ldcp->id);
1651
1652 switch (msg->stype) {
1653 case LDC_NACK:
1654 /* RTR NACK received */
1655 DWARN(ldcp->id,
1656 "i_ldc_process_RTR: (0x%llx) RTR NACK received\n",
1657 ldcp->id);
1658
1659 /* Reset the channel -- as we cannot continue */
1660 mutex_enter(&ldcp->tx_lock);
1661 i_ldc_reset(ldcp, B_TRUE);
1662 mutex_exit(&ldcp->tx_lock);
1663 rv = ECONNRESET;
1664
1665 break;
1666
1667 case LDC_INFO:
1668
1669 /* check mode */
1670 if (ldcp->mode != (ldc_mode_t)msg->env) {
1671 DWARN(ldcp->id,
1672 "i_ldc_process_RTR: (0x%llx) mode mismatch, "
1673 "expecting 0x%x, got 0x%x\n",
1674 ldcp->id, ldcp->mode, (ldc_mode_t)msg->env);
1675 /*
1676 * send NACK in response to MODE message
1677 * get the current tail for the response
1678 */
1679 rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_NACK, LDC_RTR);
1680 if (rv) {
1681 /* if cannot send NACK - reset channel */
1682 mutex_enter(&ldcp->tx_lock);
1683 i_ldc_reset(ldcp, B_TRUE);
1684 mutex_exit(&ldcp->tx_lock);
1685 rv = ECONNRESET;
1686 break;
1687 }
1688 sent_NACK = B_TRUE;
1689 }
1690 break;
1691
1692 default:
1693 DWARN(ldcp->id, "i_ldc_process_RTR: (0x%llx) unexp ACK\n",
1694 ldcp->id);
1695
1696 /* Reset the channel -- as we cannot continue */
1697 mutex_enter(&ldcp->tx_lock);
1698 i_ldc_reset(ldcp, B_TRUE);
1699 mutex_exit(&ldcp->tx_lock);
1700 rv = ECONNRESET;
1701 break;
1702 }
1703
1704 /*
1705 * If either the connection was reset (when rv != 0) or
1706 * a NACK was sent, we return. In the case of a NACK
1707 * we dont want to consume the packet that came in but
1708 * not record that we received the RTR
1709 */
1710 if (rv || sent_NACK)
1711 return (rv);
1712
1713 ldcp->last_msg_snt = msg->seqid;
1714 ldcp->hstate |= TS_RCVD_RTR;
1715
1716 rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_INFO, LDC_RDX);
1717 if (rv) {
1718 cmn_err(CE_NOTE,
1719 "i_ldc_process_RTR: (0x%lx) cannot send RDX\n",
1720 ldcp->id);
1721 mutex_enter(&ldcp->tx_lock);
1722 i_ldc_reset(ldcp, B_TRUE);
1723 mutex_exit(&ldcp->tx_lock);
1724 return (ECONNRESET);
1725 }
1726 D2(ldcp->id,
1727 "i_ldc_process_RTR: (0x%llx) sent RDX\n", ldcp->id);
1728
1729 ldcp->hstate |= TS_SENT_RDX;
1730 ldcp->tstate |= TS_HSHAKE_DONE;
1731 if ((ldcp->tstate & TS_IN_RESET) == 0)
1732 ldcp->status = LDC_UP;
1733
1734 D1(ldcp->id, "(0x%llx) Handshake Complete\n", ldcp->id);
1735
1736 return (0);
1737 }
1738
1739
1740 /*
1741 * Process an incoming RDX ctrl message
1742 */
1743 static int
i_ldc_process_RDX(ldc_chan_t * ldcp,ldc_msg_t * msg)1744 i_ldc_process_RDX(ldc_chan_t *ldcp, ldc_msg_t *msg)
1745 {
1746 int rv = 0;
1747
1748 D2(ldcp->id, "i_ldc_process_RDX: (0x%llx) received RDX\n", ldcp->id);
1749
1750 switch (msg->stype) {
1751 case LDC_NACK:
1752 /* RDX NACK received */
1753 DWARN(ldcp->id,
1754 "i_ldc_process_RDX: (0x%llx) RDX NACK received\n",
1755 ldcp->id);
1756
1757 /* Reset the channel -- as we cannot continue */
1758 mutex_enter(&ldcp->tx_lock);
1759 i_ldc_reset(ldcp, B_TRUE);
1760 mutex_exit(&ldcp->tx_lock);
1761 rv = ECONNRESET;
1762
1763 break;
1764
1765 case LDC_INFO:
1766
1767 /*
1768 * if channel is UP and a RDX received after data transmission
1769 * has commenced it is an error
1770 */
1771 if ((ldcp->tstate == TS_UP) && (ldcp->hstate & TS_RCVD_RDX)) {
1772 DWARN(DBG_ALL_LDCS,
1773 "i_ldc_process_RDX: (0x%llx) unexpected RDX"
1774 " - LDC reset\n", ldcp->id);
1775 mutex_enter(&ldcp->tx_lock);
1776 i_ldc_reset(ldcp, B_TRUE);
1777 mutex_exit(&ldcp->tx_lock);
1778 return (ECONNRESET);
1779 }
1780
1781 ldcp->hstate |= TS_RCVD_RDX;
1782 ldcp->tstate |= TS_HSHAKE_DONE;
1783 if ((ldcp->tstate & TS_IN_RESET) == 0)
1784 ldcp->status = LDC_UP;
1785
1786 D1(DBG_ALL_LDCS, "(0x%llx) Handshake Complete\n", ldcp->id);
1787 break;
1788
1789 default:
1790 DWARN(ldcp->id, "i_ldc_process_RDX: (0x%llx) unexp ACK\n",
1791 ldcp->id);
1792
1793 /* Reset the channel -- as we cannot continue */
1794 mutex_enter(&ldcp->tx_lock);
1795 i_ldc_reset(ldcp, B_TRUE);
1796 mutex_exit(&ldcp->tx_lock);
1797 rv = ECONNRESET;
1798 break;
1799 }
1800
1801 return (rv);
1802 }
1803
1804 /*
1805 * Process an incoming ACK for a data packet
1806 */
1807 static int
i_ldc_process_data_ACK(ldc_chan_t * ldcp,ldc_msg_t * msg)1808 i_ldc_process_data_ACK(ldc_chan_t *ldcp, ldc_msg_t *msg)
1809 {
1810 int rv;
1811 uint64_t tx_head;
1812 ldc_msg_t *pkt;
1813
1814 /* Obtain Tx lock */
1815 mutex_enter(&ldcp->tx_lock);
1816
1817 /*
1818 * Read the current Tx head and tail
1819 */
1820 rv = hv_ldc_tx_get_state(ldcp->id,
1821 &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state);
1822 if (rv != 0) {
1823 cmn_err(CE_WARN,
1824 "i_ldc_process_data_ACK: (0x%lx) cannot read qptrs\n",
1825 ldcp->id);
1826
1827 /* Reset the channel -- as we cannot continue */
1828 i_ldc_reset(ldcp, B_TRUE);
1829 mutex_exit(&ldcp->tx_lock);
1830 return (ECONNRESET);
1831 }
1832
1833 /*
1834 * loop from where the previous ACK location was to the
1835 * current head location. This is how far the HV has
1836 * actually send pkts. Pkts between head and tail are
1837 * yet to be sent by HV.
1838 */
1839 tx_head = ldcp->tx_ackd_head;
1840 for (;;) {
1841 pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_head);
1842 tx_head = (tx_head + LDC_PACKET_SIZE) %
1843 (ldcp->tx_q_entries << LDC_PACKET_SHIFT);
1844
1845 if (pkt->seqid == msg->ackid) {
1846 D2(ldcp->id,
1847 "i_ldc_process_data_ACK: (0x%llx) found packet\n",
1848 ldcp->id);
1849 ldcp->last_ack_rcd = msg->ackid;
1850 ldcp->tx_ackd_head = tx_head;
1851 break;
1852 }
1853 if (tx_head == ldcp->tx_head) {
1854 /* could not find packet */
1855 DWARN(ldcp->id,
1856 "i_ldc_process_data_ACK: (0x%llx) invalid ACKid\n",
1857 ldcp->id);
1858
1859 /* Reset the channel -- as we cannot continue */
1860 i_ldc_reset(ldcp, B_TRUE);
1861 mutex_exit(&ldcp->tx_lock);
1862 return (ECONNRESET);
1863 }
1864 }
1865
1866 mutex_exit(&ldcp->tx_lock);
1867 return (0);
1868 }
1869
1870 /*
1871 * Process incoming control message
1872 * Return 0 - session can continue
1873 * EAGAIN - reprocess packet - state was changed
1874 * ECONNRESET - channel was reset
1875 */
1876 static int
i_ldc_ctrlmsg(ldc_chan_t * ldcp,ldc_msg_t * msg)1877 i_ldc_ctrlmsg(ldc_chan_t *ldcp, ldc_msg_t *msg)
1878 {
1879 int rv = 0;
1880
1881 D1(ldcp->id, "i_ldc_ctrlmsg: (%llx) tstate = %lx, hstate = %lx\n",
1882 ldcp->id, ldcp->tstate, ldcp->hstate);
1883
1884 switch (ldcp->tstate & ~TS_IN_RESET) {
1885
1886 case TS_OPEN:
1887 case TS_READY:
1888
1889 switch (msg->ctrl & LDC_CTRL_MASK) {
1890 case LDC_VER:
1891 /* process version message */
1892 rv = i_ldc_process_VER(ldcp, msg);
1893 break;
1894 default:
1895 DWARN(ldcp->id,
1896 "i_ldc_ctrlmsg: (0x%llx) unexp ctrl 0x%x "
1897 "tstate=0x%x\n", ldcp->id,
1898 (msg->ctrl & LDC_CTRL_MASK), ldcp->tstate);
1899 break;
1900 }
1901
1902 break;
1903
1904 case TS_VREADY:
1905
1906 switch (msg->ctrl & LDC_CTRL_MASK) {
1907 case LDC_VER:
1908 /* process version message */
1909 rv = i_ldc_process_VER(ldcp, msg);
1910 break;
1911 case LDC_RTS:
1912 /* process RTS message */
1913 rv = i_ldc_process_RTS(ldcp, msg);
1914 break;
1915 case LDC_RTR:
1916 /* process RTR message */
1917 rv = i_ldc_process_RTR(ldcp, msg);
1918 break;
1919 case LDC_RDX:
1920 /* process RDX message */
1921 rv = i_ldc_process_RDX(ldcp, msg);
1922 break;
1923 default:
1924 DWARN(ldcp->id,
1925 "i_ldc_ctrlmsg: (0x%llx) unexp ctrl 0x%x "
1926 "tstate=0x%x\n", ldcp->id,
1927 (msg->ctrl & LDC_CTRL_MASK), ldcp->tstate);
1928 break;
1929 }
1930
1931 break;
1932
1933 case TS_UP:
1934
1935 switch (msg->ctrl & LDC_CTRL_MASK) {
1936 case LDC_VER:
1937 DWARN(ldcp->id,
1938 "i_ldc_ctrlmsg: (0x%llx) unexpected VER "
1939 "- LDC reset\n", ldcp->id);
1940 /* peer is redoing version negotiation */
1941 mutex_enter(&ldcp->tx_lock);
1942 (void) i_ldc_txq_reconf(ldcp);
1943 i_ldc_reset_state(ldcp);
1944 mutex_exit(&ldcp->tx_lock);
1945 rv = EAGAIN;
1946 break;
1947
1948 case LDC_RDX:
1949 /* process RDX message */
1950 rv = i_ldc_process_RDX(ldcp, msg);
1951 break;
1952
1953 default:
1954 DWARN(ldcp->id,
1955 "i_ldc_ctrlmsg: (0x%llx) unexp ctrl 0x%x "
1956 "tstate=0x%x\n", ldcp->id,
1957 (msg->ctrl & LDC_CTRL_MASK), ldcp->tstate);
1958 break;
1959 }
1960 }
1961
1962 return (rv);
1963 }
1964
1965 /*
1966 * Register channel with the channel nexus
1967 */
1968 static int
i_ldc_register_channel(ldc_chan_t * ldcp)1969 i_ldc_register_channel(ldc_chan_t *ldcp)
1970 {
1971 int rv = 0;
1972 ldc_cnex_t *cinfo = &ldcssp->cinfo;
1973
1974 if (cinfo->dip == NULL) {
1975 DWARN(ldcp->id,
1976 "i_ldc_register_channel: cnex has not registered\n");
1977 return (EAGAIN);
1978 }
1979
1980 rv = cinfo->reg_chan(cinfo->dip, ldcp->id, ldcp->devclass);
1981 if (rv) {
1982 DWARN(ldcp->id,
1983 "i_ldc_register_channel: cannot register channel\n");
1984 return (rv);
1985 }
1986
1987 rv = cinfo->add_intr(cinfo->dip, ldcp->id, CNEX_TX_INTR,
1988 i_ldc_tx_hdlr, ldcp, NULL);
1989 if (rv) {
1990 DWARN(ldcp->id,
1991 "i_ldc_register_channel: cannot add Tx interrupt\n");
1992 (void) cinfo->unreg_chan(cinfo->dip, ldcp->id);
1993 return (rv);
1994 }
1995
1996 rv = cinfo->add_intr(cinfo->dip, ldcp->id, CNEX_RX_INTR,
1997 i_ldc_rx_hdlr, ldcp, NULL);
1998 if (rv) {
1999 DWARN(ldcp->id,
2000 "i_ldc_register_channel: cannot add Rx interrupt\n");
2001 (void) cinfo->rem_intr(cinfo->dip, ldcp->id, CNEX_TX_INTR);
2002 (void) cinfo->unreg_chan(cinfo->dip, ldcp->id);
2003 return (rv);
2004 }
2005
2006 ldcp->tstate |= TS_CNEX_RDY;
2007
2008 return (0);
2009 }
2010
2011 /*
2012 * Unregister a channel with the channel nexus
2013 */
2014 static int
i_ldc_unregister_channel(ldc_chan_t * ldcp)2015 i_ldc_unregister_channel(ldc_chan_t *ldcp)
2016 {
2017 int rv = 0;
2018 ldc_cnex_t *cinfo = &ldcssp->cinfo;
2019
2020 if (cinfo->dip == NULL) {
2021 DWARN(ldcp->id,
2022 "i_ldc_unregister_channel: cnex has not registered\n");
2023 return (EAGAIN);
2024 }
2025
2026 if (ldcp->tstate & TS_CNEX_RDY) {
2027
2028 /* Remove the Rx interrupt */
2029 rv = cinfo->rem_intr(cinfo->dip, ldcp->id, CNEX_RX_INTR);
2030 if (rv) {
2031 if (rv != EAGAIN) {
2032 DWARN(ldcp->id,
2033 "i_ldc_unregister_channel: err removing "
2034 "Rx intr\n");
2035 return (rv);
2036 }
2037
2038 /*
2039 * If interrupts are pending and handler has
2040 * finished running, clear interrupt and try
2041 * again
2042 */
2043 if (ldcp->rx_intr_state != LDC_INTR_PEND)
2044 return (rv);
2045
2046 (void) i_ldc_clear_intr(ldcp, CNEX_RX_INTR);
2047 rv = cinfo->rem_intr(cinfo->dip, ldcp->id,
2048 CNEX_RX_INTR);
2049 if (rv) {
2050 DWARN(ldcp->id, "i_ldc_unregister_channel: "
2051 "err removing Rx interrupt\n");
2052 return (rv);
2053 }
2054 }
2055
2056 /* Remove the Tx interrupt */
2057 rv = cinfo->rem_intr(cinfo->dip, ldcp->id, CNEX_TX_INTR);
2058 if (rv) {
2059 DWARN(ldcp->id,
2060 "i_ldc_unregister_channel: err removing Tx intr\n");
2061 return (rv);
2062 }
2063
2064 /* Unregister the channel */
2065 rv = cinfo->unreg_chan(ldcssp->cinfo.dip, ldcp->id);
2066 if (rv) {
2067 DWARN(ldcp->id,
2068 "i_ldc_unregister_channel: cannot unreg channel\n");
2069 return (rv);
2070 }
2071
2072 ldcp->tstate &= ~TS_CNEX_RDY;
2073 }
2074
2075 return (0);
2076 }
2077
2078
2079 /*
2080 * LDC transmit interrupt handler
2081 * triggered for chanel up/down/reset events
2082 * and Tx queue content changes
2083 */
2084 static uint_t
i_ldc_tx_hdlr(caddr_t arg1,caddr_t arg2)2085 i_ldc_tx_hdlr(caddr_t arg1, caddr_t arg2)
2086 {
2087 _NOTE(ARGUNUSED(arg2))
2088
2089 int rv;
2090 ldc_chan_t *ldcp;
2091 boolean_t notify_client = B_FALSE;
2092 uint64_t notify_event = 0, link_state;
2093
2094 /* Get the channel for which interrupt was received */
2095 ASSERT(arg1 != NULL);
2096 ldcp = (ldc_chan_t *)arg1;
2097
2098 D1(ldcp->id, "i_ldc_tx_hdlr: (0x%llx) Received intr, ldcp=0x%p\n",
2099 ldcp->id, ldcp);
2100
2101 /* Lock channel */
2102 mutex_enter(&ldcp->lock);
2103
2104 /* Obtain Tx lock */
2105 mutex_enter(&ldcp->tx_lock);
2106
2107 /* mark interrupt as pending */
2108 ldcp->tx_intr_state = LDC_INTR_ACTIVE;
2109
2110 /* save current link state */
2111 link_state = ldcp->link_state;
2112
2113 rv = hv_ldc_tx_get_state(ldcp->id, &ldcp->tx_head, &ldcp->tx_tail,
2114 &ldcp->link_state);
2115 if (rv) {
2116 cmn_err(CE_WARN,
2117 "i_ldc_tx_hdlr: (0x%lx) cannot read queue ptrs rv=0x%d\n",
2118 ldcp->id, rv);
2119 i_ldc_clear_intr(ldcp, CNEX_TX_INTR);
2120 mutex_exit(&ldcp->tx_lock);
2121 mutex_exit(&ldcp->lock);
2122 return (DDI_INTR_CLAIMED);
2123 }
2124
2125 /*
2126 * reset the channel state if the channel went down
2127 * (other side unconfigured queue) or channel was reset
2128 * (other side reconfigured its queue)
2129 */
2130 if (link_state != ldcp->link_state &&
2131 ldcp->link_state == LDC_CHANNEL_DOWN) {
2132 D1(ldcp->id, "i_ldc_tx_hdlr: channel link down\n", ldcp->id);
2133 i_ldc_reset(ldcp, B_FALSE);
2134 notify_client = B_TRUE;
2135 notify_event = LDC_EVT_DOWN;
2136 }
2137
2138 if (link_state != ldcp->link_state &&
2139 ldcp->link_state == LDC_CHANNEL_RESET) {
2140 D1(ldcp->id, "i_ldc_tx_hdlr: channel link reset\n", ldcp->id);
2141 i_ldc_reset(ldcp, B_FALSE);
2142 notify_client = B_TRUE;
2143 notify_event = LDC_EVT_RESET;
2144 }
2145
2146 if (link_state != ldcp->link_state &&
2147 (ldcp->tstate & ~TS_IN_RESET) == TS_OPEN &&
2148 ldcp->link_state == LDC_CHANNEL_UP) {
2149 D1(ldcp->id, "i_ldc_tx_hdlr: channel link up\n", ldcp->id);
2150 notify_client = B_TRUE;
2151 notify_event = LDC_EVT_RESET;
2152 ldcp->tstate |= TS_LINK_READY;
2153 ldcp->status = LDC_READY;
2154 }
2155
2156 /* if callbacks are disabled, do not notify */
2157 if (!ldcp->cb_enabled)
2158 notify_client = B_FALSE;
2159
2160 i_ldc_clear_intr(ldcp, CNEX_TX_INTR);
2161 mutex_exit(&ldcp->tx_lock);
2162
2163 if (notify_client) {
2164 ldcp->cb_inprogress = B_TRUE;
2165 mutex_exit(&ldcp->lock);
2166 rv = ldcp->cb(notify_event, ldcp->cb_arg);
2167 if (rv) {
2168 DWARN(ldcp->id, "i_ldc_tx_hdlr: (0x%llx) callback "
2169 "failure", ldcp->id);
2170 }
2171 mutex_enter(&ldcp->lock);
2172 ldcp->cb_inprogress = B_FALSE;
2173 }
2174
2175 mutex_exit(&ldcp->lock);
2176
2177 D1(ldcp->id, "i_ldc_tx_hdlr: (0x%llx) exiting handler", ldcp->id);
2178
2179 return (DDI_INTR_CLAIMED);
2180 }
2181
2182 /*
2183 * Process the Rx HV queue.
2184 *
2185 * Returns 0 if data packets were found and no errors were encountered,
2186 * otherwise returns an error. In either case, the *notify argument is
2187 * set to indicate whether or not the client callback function should
2188 * be invoked. The *event argument is set to contain the callback event.
2189 *
2190 * Depending on the channel mode, packets are handled differently:
2191 *
2192 * RAW MODE
2193 * For raw mode channels, when a data packet is encountered,
2194 * processing stops and all packets are left on the queue to be removed
2195 * and processed by the ldc_read code path.
2196 *
2197 * UNRELIABLE MODE
2198 * For unreliable mode, when a data packet is encountered, processing
2199 * stops, and all packets are left on the queue to be removed and
2200 * processed by the ldc_read code path. Control packets are processed
2201 * inline if they are encountered before any data packets.
2202 *
2203 * RELIABLE MODE
2204 * For reliable mode channels, all packets on the receive queue
2205 * are processed: data packets are copied to the data queue and
2206 * control packets are processed inline. Packets are only left on
2207 * the receive queue when the data queue is full.
2208 */
2209 static uint_t
i_ldc_rx_process_hvq(ldc_chan_t * ldcp,boolean_t * notify_client,uint64_t * notify_event)2210 i_ldc_rx_process_hvq(ldc_chan_t *ldcp, boolean_t *notify_client,
2211 uint64_t *notify_event)
2212 {
2213 int rv;
2214 uint64_t rx_head, rx_tail;
2215 ldc_msg_t *msg;
2216 uint64_t link_state, first_fragment = 0;
2217 boolean_t trace_length = B_TRUE;
2218
2219 ASSERT(MUTEX_HELD(&ldcp->lock));
2220 *notify_client = B_FALSE;
2221 *notify_event = 0;
2222
2223 /*
2224 * Read packet(s) from the queue
2225 */
2226 for (;;) {
2227
2228 link_state = ldcp->link_state;
2229 rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail,
2230 &ldcp->link_state);
2231 if (rv) {
2232 cmn_err(CE_WARN,
2233 "i_ldc_rx_process_hvq: (0x%lx) cannot read "
2234 "queue ptrs, rv=0x%d\n", ldcp->id, rv);
2235 i_ldc_clear_intr(ldcp, CNEX_RX_INTR);
2236 return (EIO);
2237 }
2238
2239 /*
2240 * reset the channel state if the channel went down
2241 * (other side unconfigured queue) or channel was reset
2242 * (other side reconfigured its queue)
2243 */
2244
2245 if (link_state != ldcp->link_state) {
2246
2247 switch (ldcp->link_state) {
2248 case LDC_CHANNEL_DOWN:
2249 D1(ldcp->id, "i_ldc_rx_process_hvq: channel "
2250 "link down\n", ldcp->id);
2251 mutex_enter(&ldcp->tx_lock);
2252 i_ldc_reset(ldcp, B_FALSE);
2253 mutex_exit(&ldcp->tx_lock);
2254 *notify_client = B_TRUE;
2255 *notify_event = LDC_EVT_DOWN;
2256 goto loop_exit;
2257
2258 case LDC_CHANNEL_UP:
2259 D1(ldcp->id, "i_ldc_rx_process_hvq: "
2260 "channel link up\n", ldcp->id);
2261
2262 if ((ldcp->tstate & ~TS_IN_RESET) == TS_OPEN) {
2263 *notify_client = B_TRUE;
2264 *notify_event = LDC_EVT_RESET;
2265 ldcp->tstate |= TS_LINK_READY;
2266 ldcp->status = LDC_READY;
2267 }
2268 break;
2269
2270 case LDC_CHANNEL_RESET:
2271 default:
2272 #ifdef DEBUG
2273 force_reset:
2274 #endif
2275 D1(ldcp->id, "i_ldc_rx_process_hvq: channel "
2276 "link reset\n", ldcp->id);
2277 mutex_enter(&ldcp->tx_lock);
2278 i_ldc_reset(ldcp, B_FALSE);
2279 mutex_exit(&ldcp->tx_lock);
2280 *notify_client = B_TRUE;
2281 *notify_event = LDC_EVT_RESET;
2282 break;
2283 }
2284 }
2285
2286 #ifdef DEBUG
2287 if (LDC_INJECT_RESET(ldcp))
2288 goto force_reset;
2289 if (LDC_INJECT_DRNGCLEAR(ldcp))
2290 i_ldc_mem_inject_dring_clear(ldcp);
2291 #endif
2292 if (trace_length) {
2293 TRACE_RXHVQ_LENGTH(ldcp, rx_head, rx_tail);
2294 trace_length = B_FALSE;
2295 }
2296
2297 if (rx_head == rx_tail) {
2298 D2(ldcp->id, "i_ldc_rx_process_hvq: (0x%llx) "
2299 "No packets\n", ldcp->id);
2300 break;
2301 }
2302
2303 D2(ldcp->id, "i_ldc_rx_process_hvq: head=0x%llx, "
2304 "tail=0x%llx\n", rx_head, rx_tail);
2305 DUMP_LDC_PKT(ldcp, "i_ldc_rx_process_hvq rcd",
2306 ldcp->rx_q_va + rx_head);
2307
2308 /* get the message */
2309 msg = (ldc_msg_t *)(ldcp->rx_q_va + rx_head);
2310
2311 /* if channel is in RAW mode or data pkt, notify and return */
2312 if (ldcp->mode == LDC_MODE_RAW) {
2313 *notify_client = B_TRUE;
2314 *notify_event |= LDC_EVT_READ;
2315 break;
2316 }
2317
2318 if ((msg->type & LDC_DATA) && (msg->stype & LDC_INFO)) {
2319
2320 /* discard packet if channel is not up */
2321 if ((ldcp->tstate & ~TS_IN_RESET) != TS_UP) {
2322
2323 /* move the head one position */
2324 rx_head = (rx_head + LDC_PACKET_SIZE) %
2325 (ldcp->rx_q_entries << LDC_PACKET_SHIFT);
2326
2327 if (rv = i_ldc_set_rx_head(ldcp, rx_head))
2328 break;
2329
2330 continue;
2331 } else {
2332 uint64_t dq_head, dq_tail;
2333
2334 /* process only RELIABLE mode data packets */
2335 if (ldcp->mode != LDC_MODE_RELIABLE) {
2336 if ((ldcp->tstate & TS_IN_RESET) == 0)
2337 *notify_client = B_TRUE;
2338 *notify_event |= LDC_EVT_READ;
2339 break;
2340 }
2341
2342 /* don't process packet if queue full */
2343 (void) i_ldc_dq_rx_get_state(ldcp, &dq_head,
2344 &dq_tail, NULL);
2345 dq_tail = (dq_tail + LDC_PACKET_SIZE) %
2346 (ldcp->rx_dq_entries << LDC_PACKET_SHIFT);
2347 if (dq_tail == dq_head ||
2348 LDC_INJECT_DQFULL(ldcp)) {
2349 rv = ENOSPC;
2350 break;
2351 }
2352 }
2353 }
2354
2355 /* Check the sequence ID for the message received */
2356 rv = i_ldc_check_seqid(ldcp, msg);
2357 if (rv != 0) {
2358
2359 DWARN(ldcp->id, "i_ldc_rx_process_hvq: (0x%llx) "
2360 "seqid error, q_ptrs=0x%lx,0x%lx", ldcp->id,
2361 rx_head, rx_tail);
2362
2363 /* Reset last_msg_rcd to start of message */
2364 if (first_fragment != 0) {
2365 ldcp->last_msg_rcd = first_fragment - 1;
2366 first_fragment = 0;
2367 }
2368
2369 /*
2370 * Send a NACK due to seqid mismatch
2371 */
2372 rv = i_ldc_send_pkt(ldcp, msg->type, LDC_NACK,
2373 (msg->ctrl & LDC_CTRL_MASK));
2374
2375 if (rv) {
2376 cmn_err(CE_NOTE, "i_ldc_rx_process_hvq: "
2377 "(0x%lx) err sending CTRL/DATA NACK msg\n",
2378 ldcp->id);
2379
2380 /* if cannot send NACK - reset channel */
2381 mutex_enter(&ldcp->tx_lock);
2382 i_ldc_reset(ldcp, B_TRUE);
2383 mutex_exit(&ldcp->tx_lock);
2384
2385 *notify_client = B_TRUE;
2386 *notify_event = LDC_EVT_RESET;
2387 break;
2388 }
2389
2390 /* purge receive queue */
2391 (void) i_ldc_set_rx_head(ldcp, rx_tail);
2392 break;
2393 }
2394
2395 /* record the message ID */
2396 ldcp->last_msg_rcd = msg->seqid;
2397
2398 /* process control messages */
2399 if (msg->type & LDC_CTRL) {
2400 /* save current internal state */
2401 uint64_t tstate = ldcp->tstate;
2402
2403 rv = i_ldc_ctrlmsg(ldcp, msg);
2404 if (rv == EAGAIN) {
2405 /* re-process pkt - state was adjusted */
2406 continue;
2407 }
2408 if (rv == ECONNRESET) {
2409 *notify_client = B_TRUE;
2410 *notify_event = LDC_EVT_RESET;
2411 break;
2412 }
2413
2414 /*
2415 * control message processing was successful
2416 * channel transitioned to ready for communication
2417 */
2418 if (rv == 0 && ldcp->tstate == TS_UP &&
2419 (tstate & ~TS_IN_RESET) !=
2420 (ldcp->tstate & ~TS_IN_RESET)) {
2421 *notify_client = B_TRUE;
2422 *notify_event = LDC_EVT_UP;
2423 }
2424 }
2425
2426 /* process data NACKs */
2427 if ((msg->type & LDC_DATA) && (msg->stype & LDC_NACK)) {
2428 DWARN(ldcp->id,
2429 "i_ldc_rx_process_hvq: (0x%llx) received DATA/NACK",
2430 ldcp->id);
2431 mutex_enter(&ldcp->tx_lock);
2432 i_ldc_reset(ldcp, B_TRUE);
2433 mutex_exit(&ldcp->tx_lock);
2434 *notify_client = B_TRUE;
2435 *notify_event = LDC_EVT_RESET;
2436 break;
2437 }
2438
2439 /* process data ACKs */
2440 if ((msg->type & LDC_DATA) && (msg->stype & LDC_ACK)) {
2441 if (rv = i_ldc_process_data_ACK(ldcp, msg)) {
2442 *notify_client = B_TRUE;
2443 *notify_event = LDC_EVT_RESET;
2444 break;
2445 }
2446 }
2447
2448 if ((msg->type & LDC_DATA) && (msg->stype & LDC_INFO)) {
2449 ASSERT(ldcp->mode == LDC_MODE_RELIABLE);
2450
2451 /*
2452 * Copy the data packet to the data queue. Note
2453 * that the copy routine updates the rx_head pointer.
2454 */
2455 i_ldc_rxdq_copy(ldcp, &rx_head);
2456
2457 if ((ldcp->tstate & TS_IN_RESET) == 0)
2458 *notify_client = B_TRUE;
2459 *notify_event |= LDC_EVT_READ;
2460 } else {
2461 rx_head = (rx_head + LDC_PACKET_SIZE) %
2462 (ldcp->rx_q_entries << LDC_PACKET_SHIFT);
2463 }
2464
2465 /* move the head one position */
2466 if (rv = i_ldc_set_rx_head(ldcp, rx_head)) {
2467 *notify_client = B_TRUE;
2468 *notify_event = LDC_EVT_RESET;
2469 break;
2470 }
2471
2472 } /* for */
2473
2474 loop_exit:
2475
2476 if (ldcp->mode == LDC_MODE_RELIABLE) {
2477 /* ACK data packets */
2478 if ((*notify_event &
2479 (LDC_EVT_READ | LDC_EVT_RESET)) == LDC_EVT_READ) {
2480 int ack_rv;
2481 ack_rv = i_ldc_send_pkt(ldcp, LDC_DATA, LDC_ACK, 0);
2482 if (ack_rv && ack_rv != EWOULDBLOCK) {
2483 cmn_err(CE_NOTE,
2484 "i_ldc_rx_process_hvq: (0x%lx) cannot "
2485 "send ACK\n", ldcp->id);
2486
2487 mutex_enter(&ldcp->tx_lock);
2488 i_ldc_reset(ldcp, B_FALSE);
2489 mutex_exit(&ldcp->tx_lock);
2490
2491 *notify_client = B_TRUE;
2492 *notify_event = LDC_EVT_RESET;
2493 goto skip_ackpeek;
2494 }
2495 }
2496
2497 /*
2498 * If we have no more space on the data queue, make sure
2499 * there are no ACKs on the rx queue waiting to be processed.
2500 */
2501 if (rv == ENOSPC) {
2502 if (i_ldc_rx_ackpeek(ldcp, rx_head, rx_tail) != 0) {
2503 ldcp->rx_ack_head = ACKPEEK_HEAD_INVALID;
2504 *notify_client = B_TRUE;
2505 *notify_event = LDC_EVT_RESET;
2506 }
2507 return (rv);
2508 } else {
2509 ldcp->rx_ack_head = ACKPEEK_HEAD_INVALID;
2510 }
2511 }
2512
2513 skip_ackpeek:
2514
2515 /* Return, indicating whether or not data packets were found */
2516 if ((*notify_event & (LDC_EVT_READ | LDC_EVT_RESET)) == LDC_EVT_READ)
2517 return (0);
2518
2519 return (ENOMSG);
2520 }
2521
2522 /*
2523 * Process any ACK packets on the HV receive queue.
2524 *
2525 * This function is only used by RELIABLE mode channels when the
2526 * secondary data queue fills up and there are packets remaining on
2527 * the HV receive queue.
2528 */
2529 int
i_ldc_rx_ackpeek(ldc_chan_t * ldcp,uint64_t rx_head,uint64_t rx_tail)2530 i_ldc_rx_ackpeek(ldc_chan_t *ldcp, uint64_t rx_head, uint64_t rx_tail)
2531 {
2532 int rv = 0;
2533 ldc_msg_t *msg;
2534
2535 if (ldcp->rx_ack_head == ACKPEEK_HEAD_INVALID)
2536 ldcp->rx_ack_head = rx_head;
2537
2538 while (ldcp->rx_ack_head != rx_tail) {
2539 msg = (ldc_msg_t *)(ldcp->rx_q_va + ldcp->rx_ack_head);
2540
2541 if ((msg->type & LDC_DATA) && (msg->stype & LDC_ACK)) {
2542 if (rv = i_ldc_process_data_ACK(ldcp, msg))
2543 break;
2544 msg->stype &= ~LDC_ACK;
2545 }
2546
2547 ldcp->rx_ack_head =
2548 (ldcp->rx_ack_head + LDC_PACKET_SIZE) %
2549 (ldcp->rx_q_entries << LDC_PACKET_SHIFT);
2550 }
2551 return (rv);
2552 }
2553
2554 /* -------------------------------------------------------------------------- */
2555
2556 /*
2557 * LDC API functions
2558 */
2559
2560 /*
2561 * Initialize the channel. Allocate internal structure and memory for
2562 * TX/RX queues, and initialize locks.
2563 */
2564 int
ldc_init(uint64_t id,ldc_attr_t * attr,ldc_handle_t * handle)2565 ldc_init(uint64_t id, ldc_attr_t *attr, ldc_handle_t *handle)
2566 {
2567 ldc_chan_t *ldcp;
2568 int rv, exit_val;
2569 uint64_t ra_base, nentries;
2570 uint64_t qlen;
2571
2572 exit_val = EINVAL; /* guarantee an error if exit on failure */
2573
2574 if (attr == NULL) {
2575 DWARN(id, "ldc_init: (0x%llx) invalid attr\n", id);
2576 return (EINVAL);
2577 }
2578 if (handle == NULL) {
2579 DWARN(id, "ldc_init: (0x%llx) invalid handle\n", id);
2580 return (EINVAL);
2581 }
2582
2583 /* check if channel is valid */
2584 rv = hv_ldc_tx_qinfo(id, &ra_base, &nentries);
2585 if (rv == H_ECHANNEL) {
2586 DWARN(id, "ldc_init: (0x%llx) invalid channel id\n", id);
2587 return (EINVAL);
2588 }
2589
2590 /* check if the channel has already been initialized */
2591 mutex_enter(&ldcssp->lock);
2592 ldcp = ldcssp->chan_list;
2593 while (ldcp != NULL) {
2594 if (ldcp->id == id) {
2595 DWARN(id, "ldc_init: (0x%llx) already initialized\n",
2596 id);
2597 mutex_exit(&ldcssp->lock);
2598 return (EADDRINUSE);
2599 }
2600 ldcp = ldcp->next;
2601 }
2602 mutex_exit(&ldcssp->lock);
2603
2604 ASSERT(ldcp == NULL);
2605
2606 *handle = 0;
2607
2608 /* Allocate an ldcp structure */
2609 ldcp = kmem_zalloc(sizeof (ldc_chan_t), KM_SLEEP);
2610
2611 /*
2612 * Initialize the channel and Tx lock
2613 *
2614 * The channel 'lock' protects the entire channel and
2615 * should be acquired before initializing, resetting,
2616 * destroying or reading from a channel.
2617 *
2618 * The 'tx_lock' should be acquired prior to transmitting
2619 * data over the channel. The lock should also be acquired
2620 * prior to channel reconfiguration (in order to prevent
2621 * concurrent writes).
2622 *
2623 * ORDERING: When both locks are being acquired, to prevent
2624 * deadlocks, the channel lock should be always acquired prior
2625 * to the tx_lock.
2626 */
2627 mutex_init(&ldcp->lock, NULL, MUTEX_DRIVER, NULL);
2628 mutex_init(&ldcp->tx_lock, NULL, MUTEX_DRIVER, NULL);
2629
2630 /* Initialize the channel */
2631 ldcp->id = id;
2632 ldcp->cb = NULL;
2633 ldcp->cb_arg = NULL;
2634 ldcp->cb_inprogress = B_FALSE;
2635 ldcp->cb_enabled = B_FALSE;
2636 ldcp->next = NULL;
2637
2638 /* Read attributes */
2639 ldcp->mode = attr->mode;
2640 ldcp->devclass = attr->devclass;
2641 ldcp->devinst = attr->instance;
2642 ldcp->mtu = (attr->mtu > 0) ? attr->mtu : LDC_DEFAULT_MTU;
2643
2644 D1(ldcp->id,
2645 "ldc_init: (0x%llx) channel attributes, class=0x%x, "
2646 "instance=0x%llx, mode=%d, mtu=%d\n",
2647 ldcp->id, ldcp->devclass, ldcp->devinst, ldcp->mode, ldcp->mtu);
2648
2649 ldcp->next_vidx = 0;
2650 ldcp->tstate = TS_IN_RESET;
2651 ldcp->hstate = 0;
2652 ldcp->last_msg_snt = LDC_INIT_SEQID;
2653 ldcp->last_ack_rcd = 0;
2654 ldcp->last_msg_rcd = 0;
2655 ldcp->rx_ack_head = ACKPEEK_HEAD_INVALID;
2656
2657 ldcp->stream_bufferp = NULL;
2658 ldcp->exp_dring_list = NULL;
2659 ldcp->imp_dring_list = NULL;
2660 ldcp->mhdl_list = NULL;
2661
2662 ldcp->tx_intr_state = LDC_INTR_NONE;
2663 ldcp->rx_intr_state = LDC_INTR_NONE;
2664
2665 /* Initialize payload size depending on whether channel is reliable */
2666 switch (ldcp->mode) {
2667 case LDC_MODE_RAW:
2668 ldcp->pkt_payload = LDC_PAYLOAD_SIZE_RAW;
2669 ldcp->read_p = i_ldc_read_raw;
2670 ldcp->write_p = i_ldc_write_raw;
2671 break;
2672 case LDC_MODE_UNRELIABLE:
2673 ldcp->pkt_payload = LDC_PAYLOAD_SIZE_UNRELIABLE;
2674 ldcp->read_p = i_ldc_read_packet;
2675 ldcp->write_p = i_ldc_write_packet;
2676 break;
2677 case LDC_MODE_RELIABLE:
2678 ldcp->pkt_payload = LDC_PAYLOAD_SIZE_RELIABLE;
2679
2680 ldcp->stream_remains = 0;
2681 ldcp->stream_offset = 0;
2682 ldcp->stream_bufferp = kmem_alloc(ldcp->mtu, KM_SLEEP);
2683 ldcp->read_p = i_ldc_read_stream;
2684 ldcp->write_p = i_ldc_write_stream;
2685 break;
2686 default:
2687 exit_val = EINVAL;
2688 goto cleanup_on_exit;
2689 }
2690
2691 /*
2692 * qlen is (mtu * ldc_mtu_msgs) / pkt_payload. If this
2693 * value is smaller than default length of ldc_queue_entries,
2694 * qlen is set to ldc_queue_entries. Ensure that computed
2695 * length is a power-of-two value.
2696 */
2697 qlen = (ldcp->mtu * ldc_mtu_msgs) / ldcp->pkt_payload;
2698 if (!ISP2(qlen)) {
2699 uint64_t tmp = 1;
2700 while (qlen) {
2701 qlen >>= 1; tmp <<= 1;
2702 }
2703 qlen = tmp;
2704 }
2705
2706 ldcp->rx_q_entries =
2707 (qlen < ldc_queue_entries) ? ldc_queue_entries : qlen;
2708 ldcp->tx_q_entries = ldcp->rx_q_entries;
2709
2710 D1(ldcp->id, "ldc_init: queue length = 0x%llx\n", ldcp->rx_q_entries);
2711
2712 /* Create a transmit queue */
2713 ldcp->tx_q_va = (uint64_t)
2714 contig_mem_alloc(ldcp->tx_q_entries << LDC_PACKET_SHIFT);
2715 if (ldcp->tx_q_va == 0) {
2716 cmn_err(CE_WARN,
2717 "ldc_init: (0x%lx) TX queue allocation failed\n",
2718 ldcp->id);
2719 exit_val = ENOMEM;
2720 goto cleanup_on_exit;
2721 }
2722 ldcp->tx_q_ra = va_to_pa((caddr_t)ldcp->tx_q_va);
2723
2724 D2(ldcp->id, "ldc_init: txq_va=0x%llx, txq_ra=0x%llx, entries=0x%llx\n",
2725 ldcp->tx_q_va, ldcp->tx_q_ra, ldcp->tx_q_entries);
2726
2727 ldcp->tstate |= TS_TXQ_RDY;
2728
2729 /* Create a receive queue */
2730 ldcp->rx_q_va = (uint64_t)
2731 contig_mem_alloc(ldcp->rx_q_entries << LDC_PACKET_SHIFT);
2732 if (ldcp->rx_q_va == 0) {
2733 cmn_err(CE_WARN,
2734 "ldc_init: (0x%lx) RX queue allocation failed\n",
2735 ldcp->id);
2736 exit_val = ENOMEM;
2737 goto cleanup_on_exit;
2738 }
2739 ldcp->rx_q_ra = va_to_pa((caddr_t)ldcp->rx_q_va);
2740
2741 D2(ldcp->id, "ldc_init: rxq_va=0x%llx, rxq_ra=0x%llx, entries=0x%llx\n",
2742 ldcp->rx_q_va, ldcp->rx_q_ra, ldcp->rx_q_entries);
2743
2744 ldcp->tstate |= TS_RXQ_RDY;
2745
2746 /* Setup a separate read data queue */
2747 if (ldcp->mode == LDC_MODE_RELIABLE) {
2748 ldcp->readq_get_state = i_ldc_dq_rx_get_state;
2749 ldcp->readq_set_head = i_ldc_set_rxdq_head;
2750
2751 /* Make sure the data queue multiplier is a power of 2 */
2752 if (!ISP2(ldc_rxdq_multiplier)) {
2753 D1(ldcp->id, "ldc_init: (0x%llx) ldc_rxdq_multiplier "
2754 "not a power of 2, resetting", ldcp->id);
2755 ldc_rxdq_multiplier = LDC_RXDQ_MULTIPLIER;
2756 }
2757
2758 ldcp->rx_dq_entries = ldc_rxdq_multiplier * ldcp->rx_q_entries;
2759 ldcp->rx_dq_va = (uint64_t)
2760 kmem_alloc(ldcp->rx_dq_entries << LDC_PACKET_SHIFT,
2761 KM_SLEEP);
2762 if (ldcp->rx_dq_va == 0) {
2763 cmn_err(CE_WARN,
2764 "ldc_init: (0x%lx) RX data queue "
2765 "allocation failed\n", ldcp->id);
2766 exit_val = ENOMEM;
2767 goto cleanup_on_exit;
2768 }
2769
2770 ldcp->rx_dq_head = ldcp->rx_dq_tail = 0;
2771
2772 D2(ldcp->id, "ldc_init: rx_dq_va=0x%llx, "
2773 "rx_dq_entries=0x%llx\n", ldcp->rx_dq_va,
2774 ldcp->rx_dq_entries);
2775 } else {
2776 ldcp->readq_get_state = i_ldc_hvq_rx_get_state;
2777 ldcp->readq_set_head = i_ldc_set_rx_head;
2778 }
2779
2780 /* Init descriptor ring and memory handle list lock */
2781 mutex_init(&ldcp->exp_dlist_lock, NULL, MUTEX_DRIVER, NULL);
2782 mutex_init(&ldcp->imp_dlist_lock, NULL, MUTEX_DRIVER, NULL);
2783 mutex_init(&ldcp->mlist_lock, NULL, MUTEX_DRIVER, NULL);
2784
2785 /* mark status as INITialized */
2786 ldcp->status = LDC_INIT;
2787
2788 /* Add to channel list */
2789 mutex_enter(&ldcssp->lock);
2790 ldcp->next = ldcssp->chan_list;
2791 ldcssp->chan_list = ldcp;
2792 ldcssp->channel_count++;
2793 mutex_exit(&ldcssp->lock);
2794
2795 /* set the handle */
2796 *handle = (ldc_handle_t)ldcp;
2797
2798 D1(ldcp->id, "ldc_init: (0x%llx) channel initialized\n", ldcp->id);
2799
2800 return (0);
2801
2802 cleanup_on_exit:
2803
2804 if (ldcp->mode == LDC_MODE_RELIABLE && ldcp->stream_bufferp)
2805 kmem_free(ldcp->stream_bufferp, ldcp->mtu);
2806
2807 if (ldcp->tstate & TS_TXQ_RDY)
2808 contig_mem_free((caddr_t)ldcp->tx_q_va,
2809 (ldcp->tx_q_entries << LDC_PACKET_SHIFT));
2810
2811 if (ldcp->tstate & TS_RXQ_RDY)
2812 contig_mem_free((caddr_t)ldcp->rx_q_va,
2813 (ldcp->rx_q_entries << LDC_PACKET_SHIFT));
2814
2815 mutex_destroy(&ldcp->tx_lock);
2816 mutex_destroy(&ldcp->lock);
2817
2818 kmem_free(ldcp, sizeof (ldc_chan_t));
2819
2820 return (exit_val);
2821 }
2822
2823 /*
2824 * Finalizes the LDC connection. It will return EBUSY if the
2825 * channel is open. A ldc_close() has to be done prior to
2826 * a ldc_fini operation. It frees TX/RX queues, associated
2827 * with the channel
2828 */
2829 int
ldc_fini(ldc_handle_t handle)2830 ldc_fini(ldc_handle_t handle)
2831 {
2832 ldc_chan_t *ldcp;
2833 ldc_chan_t *tmp_ldcp;
2834 uint64_t id;
2835
2836 if (handle == 0) {
2837 DWARN(DBG_ALL_LDCS, "ldc_fini: invalid channel handle\n");
2838 return (EINVAL);
2839 }
2840 ldcp = (ldc_chan_t *)handle;
2841 id = ldcp->id;
2842
2843 mutex_enter(&ldcp->lock);
2844
2845 if ((ldcp->tstate & ~TS_IN_RESET) > TS_INIT) {
2846 DWARN(ldcp->id, "ldc_fini: (0x%llx) channel is open\n",
2847 ldcp->id);
2848 mutex_exit(&ldcp->lock);
2849 return (EBUSY);
2850 }
2851
2852 /* Remove from the channel list */
2853 mutex_enter(&ldcssp->lock);
2854 tmp_ldcp = ldcssp->chan_list;
2855 if (tmp_ldcp == ldcp) {
2856 ldcssp->chan_list = ldcp->next;
2857 ldcp->next = NULL;
2858 } else {
2859 while (tmp_ldcp != NULL) {
2860 if (tmp_ldcp->next == ldcp) {
2861 tmp_ldcp->next = ldcp->next;
2862 ldcp->next = NULL;
2863 break;
2864 }
2865 tmp_ldcp = tmp_ldcp->next;
2866 }
2867 if (tmp_ldcp == NULL) {
2868 DWARN(DBG_ALL_LDCS, "ldc_fini: invalid channel hdl\n");
2869 mutex_exit(&ldcssp->lock);
2870 mutex_exit(&ldcp->lock);
2871 return (EINVAL);
2872 }
2873 }
2874
2875 ldcssp->channel_count--;
2876
2877 mutex_exit(&ldcssp->lock);
2878
2879 /* Free the map table for this channel */
2880 if (ldcp->mtbl) {
2881 (void) hv_ldc_set_map_table(ldcp->id, 0, 0);
2882 if (ldcp->mtbl->contigmem)
2883 contig_mem_free(ldcp->mtbl->table, ldcp->mtbl->size);
2884 else
2885 kmem_free(ldcp->mtbl->table, ldcp->mtbl->size);
2886 mutex_destroy(&ldcp->mtbl->lock);
2887 kmem_free(ldcp->mtbl, sizeof (ldc_mtbl_t));
2888 }
2889
2890 /* Destroy descriptor ring and memory handle list lock */
2891 mutex_destroy(&ldcp->exp_dlist_lock);
2892 mutex_destroy(&ldcp->imp_dlist_lock);
2893 mutex_destroy(&ldcp->mlist_lock);
2894
2895 /* Free the stream buffer for RELIABLE_MODE */
2896 if (ldcp->mode == LDC_MODE_RELIABLE && ldcp->stream_bufferp)
2897 kmem_free(ldcp->stream_bufferp, ldcp->mtu);
2898
2899 /* Free the RX queue */
2900 contig_mem_free((caddr_t)ldcp->rx_q_va,
2901 (ldcp->rx_q_entries << LDC_PACKET_SHIFT));
2902 ldcp->tstate &= ~TS_RXQ_RDY;
2903
2904 /* Free the RX data queue */
2905 if (ldcp->mode == LDC_MODE_RELIABLE) {
2906 kmem_free((caddr_t)ldcp->rx_dq_va,
2907 (ldcp->rx_dq_entries << LDC_PACKET_SHIFT));
2908 }
2909
2910 /* Free the TX queue */
2911 contig_mem_free((caddr_t)ldcp->tx_q_va,
2912 (ldcp->tx_q_entries << LDC_PACKET_SHIFT));
2913 ldcp->tstate &= ~TS_TXQ_RDY;
2914
2915 mutex_exit(&ldcp->lock);
2916
2917 /* Destroy mutex */
2918 mutex_destroy(&ldcp->tx_lock);
2919 mutex_destroy(&ldcp->lock);
2920
2921 /* free channel structure */
2922 kmem_free(ldcp, sizeof (ldc_chan_t));
2923
2924 D1(id, "ldc_fini: (0x%llx) channel finalized\n", id);
2925
2926 return (0);
2927 }
2928
2929 /*
2930 * Open the LDC channel for use. It registers the TX/RX queues
2931 * with the Hypervisor. It also specifies the interrupt number
2932 * and target CPU for this channel
2933 */
2934 int
ldc_open(ldc_handle_t handle)2935 ldc_open(ldc_handle_t handle)
2936 {
2937 ldc_chan_t *ldcp;
2938 int rv;
2939
2940 if (handle == 0) {
2941 DWARN(DBG_ALL_LDCS, "ldc_open: invalid channel handle\n");
2942 return (EINVAL);
2943 }
2944
2945 ldcp = (ldc_chan_t *)handle;
2946
2947 mutex_enter(&ldcp->lock);
2948
2949 if (ldcp->tstate < TS_INIT) {
2950 DWARN(ldcp->id,
2951 "ldc_open: (0x%llx) channel not initialized\n", ldcp->id);
2952 mutex_exit(&ldcp->lock);
2953 return (EFAULT);
2954 }
2955 if ((ldcp->tstate & ~TS_IN_RESET) >= TS_OPEN) {
2956 DWARN(ldcp->id,
2957 "ldc_open: (0x%llx) channel is already open\n", ldcp->id);
2958 mutex_exit(&ldcp->lock);
2959 return (EFAULT);
2960 }
2961
2962 /*
2963 * Unregister/Register the tx queue with the hypervisor
2964 */
2965 rv = hv_ldc_tx_qconf(ldcp->id, 0, 0);
2966 if (rv) {
2967 cmn_err(CE_WARN,
2968 "ldc_open: (0x%lx) channel tx queue unconf failed\n",
2969 ldcp->id);
2970 mutex_exit(&ldcp->lock);
2971 return (EIO);
2972 }
2973
2974 rv = hv_ldc_tx_qconf(ldcp->id, ldcp->tx_q_ra, ldcp->tx_q_entries);
2975 if (rv) {
2976 cmn_err(CE_WARN,
2977 "ldc_open: (0x%lx) channel tx queue conf failed\n",
2978 ldcp->id);
2979 mutex_exit(&ldcp->lock);
2980 return (EIO);
2981 }
2982
2983 D2(ldcp->id, "ldc_open: (0x%llx) registered tx queue with LDC\n",
2984 ldcp->id);
2985
2986 /*
2987 * Unregister/Register the rx queue with the hypervisor
2988 */
2989 rv = hv_ldc_rx_qconf(ldcp->id, 0, 0);
2990 if (rv) {
2991 cmn_err(CE_WARN,
2992 "ldc_open: (0x%lx) channel rx queue unconf failed\n",
2993 ldcp->id);
2994 mutex_exit(&ldcp->lock);
2995 return (EIO);
2996 }
2997
2998 rv = hv_ldc_rx_qconf(ldcp->id, ldcp->rx_q_ra, ldcp->rx_q_entries);
2999 if (rv) {
3000 cmn_err(CE_WARN,
3001 "ldc_open: (0x%lx) channel rx queue conf failed\n",
3002 ldcp->id);
3003 mutex_exit(&ldcp->lock);
3004 return (EIO);
3005 }
3006
3007 D2(ldcp->id, "ldc_open: (0x%llx) registered rx queue with LDC\n",
3008 ldcp->id);
3009
3010 ldcp->tstate |= TS_QCONF_RDY;
3011
3012 /* Register the channel with the channel nexus */
3013 rv = i_ldc_register_channel(ldcp);
3014 if (rv && rv != EAGAIN) {
3015 cmn_err(CE_WARN,
3016 "ldc_open: (0x%lx) channel register failed\n", ldcp->id);
3017 ldcp->tstate &= ~TS_QCONF_RDY;
3018 (void) hv_ldc_tx_qconf(ldcp->id, 0, 0);
3019 (void) hv_ldc_rx_qconf(ldcp->id, 0, 0);
3020 mutex_exit(&ldcp->lock);
3021 return (EIO);
3022 }
3023
3024 /* mark channel in OPEN state */
3025 ldcp->status = LDC_OPEN;
3026
3027 /* Read channel state */
3028 rv = hv_ldc_tx_get_state(ldcp->id,
3029 &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state);
3030 if (rv) {
3031 cmn_err(CE_WARN,
3032 "ldc_open: (0x%lx) cannot read channel state\n",
3033 ldcp->id);
3034 (void) i_ldc_unregister_channel(ldcp);
3035 ldcp->tstate &= ~TS_QCONF_RDY;
3036 (void) hv_ldc_tx_qconf(ldcp->id, 0, 0);
3037 (void) hv_ldc_rx_qconf(ldcp->id, 0, 0);
3038 mutex_exit(&ldcp->lock);
3039 return (EIO);
3040 }
3041
3042 /*
3043 * set the ACKd head to current head location for reliable
3044 */
3045 ldcp->tx_ackd_head = ldcp->tx_head;
3046
3047 /* mark channel ready if HV report link is UP (peer alloc'd Rx queue) */
3048 if (ldcp->link_state == LDC_CHANNEL_UP ||
3049 ldcp->link_state == LDC_CHANNEL_RESET) {
3050 ldcp->tstate |= TS_LINK_READY;
3051 ldcp->status = LDC_READY;
3052 }
3053
3054 /*
3055 * if channel is being opened in RAW mode - no handshake is needed
3056 * switch the channel READY and UP state
3057 */
3058 if (ldcp->mode == LDC_MODE_RAW) {
3059 ldcp->tstate = TS_UP; /* set bits associated with LDC UP */
3060 ldcp->status = LDC_UP;
3061 }
3062
3063 mutex_exit(&ldcp->lock);
3064
3065 /*
3066 * Increment number of open channels
3067 */
3068 mutex_enter(&ldcssp->lock);
3069 ldcssp->channels_open++;
3070 mutex_exit(&ldcssp->lock);
3071
3072 D1(ldcp->id,
3073 "ldc_open: (0x%llx) channel (0x%p) open for use "
3074 "(tstate=0x%x, status=0x%x)\n",
3075 ldcp->id, ldcp, ldcp->tstate, ldcp->status);
3076
3077 return (0);
3078 }
3079
3080 /*
3081 * Close the LDC connection. It will return EBUSY if there
3082 * are memory segments or descriptor rings either bound to or
3083 * mapped over the channel
3084 */
3085 int
ldc_close(ldc_handle_t handle)3086 ldc_close(ldc_handle_t handle)
3087 {
3088 ldc_chan_t *ldcp;
3089 int rv = 0, retries = 0;
3090 boolean_t chk_done = B_FALSE;
3091
3092 if (handle == 0) {
3093 DWARN(DBG_ALL_LDCS, "ldc_close: invalid channel handle\n");
3094 return (EINVAL);
3095 }
3096 ldcp = (ldc_chan_t *)handle;
3097
3098 mutex_enter(&ldcp->lock);
3099
3100 /* return error if channel is not open */
3101 if ((ldcp->tstate & ~TS_IN_RESET) < TS_OPEN) {
3102 DWARN(ldcp->id,
3103 "ldc_close: (0x%llx) channel is not open\n", ldcp->id);
3104 mutex_exit(&ldcp->lock);
3105 return (EFAULT);
3106 }
3107
3108 /* if any memory handles, drings, are bound or mapped cannot close */
3109 if (ldcp->mhdl_list != NULL) {
3110 DWARN(ldcp->id,
3111 "ldc_close: (0x%llx) channel has bound memory handles\n",
3112 ldcp->id);
3113 mutex_exit(&ldcp->lock);
3114 return (EBUSY);
3115 }
3116 if (ldcp->exp_dring_list != NULL) {
3117 DWARN(ldcp->id,
3118 "ldc_close: (0x%llx) channel has bound descriptor rings\n",
3119 ldcp->id);
3120 mutex_exit(&ldcp->lock);
3121 return (EBUSY);
3122 }
3123 if (ldcp->imp_dring_list != NULL) {
3124 DWARN(ldcp->id,
3125 "ldc_close: (0x%llx) channel has mapped descriptor rings\n",
3126 ldcp->id);
3127 mutex_exit(&ldcp->lock);
3128 return (EBUSY);
3129 }
3130
3131 if (ldcp->cb_inprogress) {
3132 DWARN(ldcp->id, "ldc_close: (0x%llx) callback active\n",
3133 ldcp->id);
3134 mutex_exit(&ldcp->lock);
3135 return (EWOULDBLOCK);
3136 }
3137
3138 /* Obtain Tx lock */
3139 mutex_enter(&ldcp->tx_lock);
3140
3141 /*
3142 * Wait for pending transmits to complete i.e Tx queue to drain
3143 * if there are pending pkts - wait 1 ms and retry again
3144 */
3145 for (;;) {
3146
3147 rv = hv_ldc_tx_get_state(ldcp->id,
3148 &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state);
3149 if (rv) {
3150 cmn_err(CE_WARN,
3151 "ldc_close: (0x%lx) cannot read qptrs\n", ldcp->id);
3152 mutex_exit(&ldcp->tx_lock);
3153 mutex_exit(&ldcp->lock);
3154 return (EIO);
3155 }
3156
3157 if (ldcp->tx_head == ldcp->tx_tail ||
3158 ldcp->link_state != LDC_CHANNEL_UP) {
3159 break;
3160 }
3161
3162 if (chk_done) {
3163 DWARN(ldcp->id,
3164 "ldc_close: (0x%llx) Tx queue drain timeout\n",
3165 ldcp->id);
3166 break;
3167 }
3168
3169 /* wait for one ms and try again */
3170 delay(drv_usectohz(1000));
3171 chk_done = B_TRUE;
3172 }
3173
3174 /*
3175 * Drain the Tx and Rx queues as we are closing the
3176 * channel. We dont care about any pending packets.
3177 * We have to also drain the queue prior to clearing
3178 * pending interrupts, otherwise the HV will trigger
3179 * an interrupt the moment the interrupt state is
3180 * cleared.
3181 */
3182 (void) i_ldc_txq_reconf(ldcp);
3183 i_ldc_rxq_drain(ldcp);
3184
3185 /*
3186 * Unregister the channel with the nexus
3187 */
3188 while ((rv = i_ldc_unregister_channel(ldcp)) != 0) {
3189
3190 mutex_exit(&ldcp->tx_lock);
3191 mutex_exit(&ldcp->lock);
3192
3193 /* if any error other than EAGAIN return back */
3194 if (rv != EAGAIN || retries >= ldc_max_retries) {
3195 cmn_err(CE_WARN,
3196 "ldc_close: (0x%lx) unregister failed, %d\n",
3197 ldcp->id, rv);
3198 return (rv);
3199 }
3200
3201 /*
3202 * As there could be pending interrupts we need
3203 * to wait and try again
3204 */
3205 drv_usecwait(ldc_close_delay);
3206 mutex_enter(&ldcp->lock);
3207 mutex_enter(&ldcp->tx_lock);
3208 retries++;
3209 }
3210
3211 ldcp->tstate &= ~TS_QCONF_RDY;
3212
3213 /*
3214 * Unregister queues
3215 */
3216 rv = hv_ldc_tx_qconf(ldcp->id, 0, 0);
3217 if (rv) {
3218 cmn_err(CE_WARN,
3219 "ldc_close: (0x%lx) channel TX queue unconf failed\n",
3220 ldcp->id);
3221 mutex_exit(&ldcp->tx_lock);
3222 mutex_exit(&ldcp->lock);
3223 return (EIO);
3224 }
3225 rv = hv_ldc_rx_qconf(ldcp->id, 0, 0);
3226 if (rv) {
3227 cmn_err(CE_WARN,
3228 "ldc_close: (0x%lx) channel RX queue unconf failed\n",
3229 ldcp->id);
3230 mutex_exit(&ldcp->tx_lock);
3231 mutex_exit(&ldcp->lock);
3232 return (EIO);
3233 }
3234
3235 /* Reset channel state information */
3236 i_ldc_reset_state(ldcp);
3237
3238 /* Mark channel as down and in initialized state */
3239 ldcp->tx_ackd_head = 0;
3240 ldcp->tx_head = 0;
3241 ldcp->tstate = TS_IN_RESET|TS_INIT;
3242 ldcp->status = LDC_INIT;
3243
3244 mutex_exit(&ldcp->tx_lock);
3245 mutex_exit(&ldcp->lock);
3246
3247 /* Decrement number of open channels */
3248 mutex_enter(&ldcssp->lock);
3249 ldcssp->channels_open--;
3250 mutex_exit(&ldcssp->lock);
3251
3252 D1(ldcp->id, "ldc_close: (0x%llx) channel closed\n", ldcp->id);
3253
3254 return (0);
3255 }
3256
3257 /*
3258 * Register channel callback
3259 */
3260 int
ldc_reg_callback(ldc_handle_t handle,uint_t (* cb)(uint64_t event,caddr_t arg),caddr_t arg)3261 ldc_reg_callback(ldc_handle_t handle,
3262 uint_t(*cb)(uint64_t event, caddr_t arg), caddr_t arg)
3263 {
3264 ldc_chan_t *ldcp;
3265
3266 if (handle == 0) {
3267 DWARN(DBG_ALL_LDCS,
3268 "ldc_reg_callback: invalid channel handle\n");
3269 return (EINVAL);
3270 }
3271 if (((uint64_t)cb) < KERNELBASE) {
3272 DWARN(DBG_ALL_LDCS, "ldc_reg_callback: invalid callback\n");
3273 return (EINVAL);
3274 }
3275 ldcp = (ldc_chan_t *)handle;
3276
3277 mutex_enter(&ldcp->lock);
3278
3279 if (ldcp->cb) {
3280 DWARN(ldcp->id, "ldc_reg_callback: (0x%llx) callback exists\n",
3281 ldcp->id);
3282 mutex_exit(&ldcp->lock);
3283 return (EIO);
3284 }
3285 if (ldcp->cb_inprogress) {
3286 DWARN(ldcp->id, "ldc_reg_callback: (0x%llx) callback active\n",
3287 ldcp->id);
3288 mutex_exit(&ldcp->lock);
3289 return (EWOULDBLOCK);
3290 }
3291
3292 ldcp->cb = cb;
3293 ldcp->cb_arg = arg;
3294 ldcp->cb_enabled = B_TRUE;
3295
3296 D1(ldcp->id,
3297 "ldc_reg_callback: (0x%llx) registered callback for channel\n",
3298 ldcp->id);
3299
3300 mutex_exit(&ldcp->lock);
3301
3302 return (0);
3303 }
3304
3305 /*
3306 * Unregister channel callback
3307 */
3308 int
ldc_unreg_callback(ldc_handle_t handle)3309 ldc_unreg_callback(ldc_handle_t handle)
3310 {
3311 ldc_chan_t *ldcp;
3312
3313 if (handle == 0) {
3314 DWARN(DBG_ALL_LDCS,
3315 "ldc_unreg_callback: invalid channel handle\n");
3316 return (EINVAL);
3317 }
3318 ldcp = (ldc_chan_t *)handle;
3319
3320 mutex_enter(&ldcp->lock);
3321
3322 if (ldcp->cb == NULL) {
3323 DWARN(ldcp->id,
3324 "ldc_unreg_callback: (0x%llx) no callback exists\n",
3325 ldcp->id);
3326 mutex_exit(&ldcp->lock);
3327 return (EIO);
3328 }
3329 if (ldcp->cb_inprogress) {
3330 DWARN(ldcp->id,
3331 "ldc_unreg_callback: (0x%llx) callback active\n",
3332 ldcp->id);
3333 mutex_exit(&ldcp->lock);
3334 return (EWOULDBLOCK);
3335 }
3336
3337 ldcp->cb = NULL;
3338 ldcp->cb_arg = NULL;
3339 ldcp->cb_enabled = B_FALSE;
3340
3341 D1(ldcp->id,
3342 "ldc_unreg_callback: (0x%llx) unregistered callback for channel\n",
3343 ldcp->id);
3344
3345 mutex_exit(&ldcp->lock);
3346
3347 return (0);
3348 }
3349
3350
3351 /*
3352 * Bring a channel up by initiating a handshake with the peer
3353 * This call is asynchronous. It will complete at a later point
3354 * in time when the peer responds back with an RTR.
3355 */
3356 int
ldc_up(ldc_handle_t handle)3357 ldc_up(ldc_handle_t handle)
3358 {
3359 int rv;
3360 ldc_chan_t *ldcp;
3361 ldc_msg_t *ldcmsg;
3362 uint64_t tx_tail, tstate, link_state;
3363
3364 if (handle == 0) {
3365 DWARN(DBG_ALL_LDCS, "ldc_up: invalid channel handle\n");
3366 return (EINVAL);
3367 }
3368 ldcp = (ldc_chan_t *)handle;
3369
3370 mutex_enter(&ldcp->lock);
3371
3372 D1(ldcp->id, "ldc_up: (0x%llx) doing channel UP\n", ldcp->id);
3373
3374 /* clear the reset state */
3375 tstate = ldcp->tstate;
3376 ldcp->tstate &= ~TS_IN_RESET;
3377
3378 if (ldcp->tstate == TS_UP) {
3379 DWARN(ldcp->id,
3380 "ldc_up: (0x%llx) channel is already in UP state\n",
3381 ldcp->id);
3382
3383 /* mark channel as up */
3384 ldcp->status = LDC_UP;
3385
3386 /*
3387 * if channel was in reset state and there was
3388 * pending data clear interrupt state. this will
3389 * trigger an interrupt, causing the RX handler to
3390 * to invoke the client's callback
3391 */
3392 if ((tstate & TS_IN_RESET) &&
3393 ldcp->rx_intr_state == LDC_INTR_PEND) {
3394 D1(ldcp->id,
3395 "ldc_up: (0x%llx) channel has pending data, "
3396 "clearing interrupt\n", ldcp->id);
3397 i_ldc_clear_intr(ldcp, CNEX_RX_INTR);
3398 }
3399
3400 mutex_exit(&ldcp->lock);
3401 return (0);
3402 }
3403
3404 /* if the channel is in RAW mode - mark it as UP, if READY */
3405 if (ldcp->mode == LDC_MODE_RAW && ldcp->tstate >= TS_READY) {
3406 ldcp->tstate = TS_UP;
3407 mutex_exit(&ldcp->lock);
3408 return (0);
3409 }
3410
3411 /* Don't start another handshake if there is one in progress */
3412 if (ldcp->hstate) {
3413 D1(ldcp->id,
3414 "ldc_up: (0x%llx) channel handshake in progress\n",
3415 ldcp->id);
3416 mutex_exit(&ldcp->lock);
3417 return (0);
3418 }
3419
3420 mutex_enter(&ldcp->tx_lock);
3421
3422 /* save current link state */
3423 link_state = ldcp->link_state;
3424
3425 /* get the current tail for the LDC msg */
3426 rv = i_ldc_get_tx_tail(ldcp, &tx_tail);
3427 if (rv) {
3428 D1(ldcp->id, "ldc_up: (0x%llx) cannot initiate handshake\n",
3429 ldcp->id);
3430 mutex_exit(&ldcp->tx_lock);
3431 mutex_exit(&ldcp->lock);
3432 return (ECONNREFUSED);
3433 }
3434
3435 /*
3436 * If i_ldc_get_tx_tail() changed link_state to either RESET or UP,
3437 * from a previous state of DOWN, then mark the channel as
3438 * being ready for handshake.
3439 */
3440 if ((link_state == LDC_CHANNEL_DOWN) &&
3441 (link_state != ldcp->link_state)) {
3442
3443 ASSERT((ldcp->link_state == LDC_CHANNEL_RESET) ||
3444 (ldcp->link_state == LDC_CHANNEL_UP));
3445
3446 if (ldcp->mode == LDC_MODE_RAW) {
3447 ldcp->status = LDC_UP;
3448 ldcp->tstate = TS_UP;
3449 mutex_exit(&ldcp->tx_lock);
3450 mutex_exit(&ldcp->lock);
3451 return (0);
3452 } else {
3453 ldcp->status = LDC_READY;
3454 ldcp->tstate |= TS_LINK_READY;
3455 }
3456
3457 }
3458
3459 ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail);
3460 ZERO_PKT(ldcmsg);
3461
3462 ldcmsg->type = LDC_CTRL;
3463 ldcmsg->stype = LDC_INFO;
3464 ldcmsg->ctrl = LDC_VER;
3465 ldcp->next_vidx = 0;
3466 bcopy(&ldc_versions[0], ldcmsg->udata, sizeof (ldc_versions[0]));
3467
3468 DUMP_LDC_PKT(ldcp, "ldc_up snd ver", (uint64_t)ldcmsg);
3469
3470 /* initiate the send by calling into HV and set the new tail */
3471 tx_tail = (tx_tail + LDC_PACKET_SIZE) %
3472 (ldcp->tx_q_entries << LDC_PACKET_SHIFT);
3473
3474 rv = i_ldc_set_tx_tail(ldcp, tx_tail);
3475 if (rv) {
3476 DWARN(ldcp->id,
3477 "ldc_up: (0x%llx) cannot initiate handshake rv=%d\n",
3478 ldcp->id, rv);
3479 mutex_exit(&ldcp->tx_lock);
3480 mutex_exit(&ldcp->lock);
3481 return (rv);
3482 }
3483
3484 ldcp->hstate |= TS_SENT_VER;
3485 ldcp->tx_tail = tx_tail;
3486 D1(ldcp->id, "ldc_up: (0x%llx) channel up initiated\n", ldcp->id);
3487
3488 mutex_exit(&ldcp->tx_lock);
3489 mutex_exit(&ldcp->lock);
3490
3491 return (rv);
3492 }
3493
3494
3495 /*
3496 * Bring a channel down by resetting its state and queues
3497 */
3498 int
ldc_down(ldc_handle_t handle)3499 ldc_down(ldc_handle_t handle)
3500 {
3501 ldc_chan_t *ldcp;
3502
3503 if (handle == 0) {
3504 DWARN(DBG_ALL_LDCS, "ldc_down: invalid channel handle\n");
3505 return (EINVAL);
3506 }
3507 ldcp = (ldc_chan_t *)handle;
3508 mutex_enter(&ldcp->lock);
3509 mutex_enter(&ldcp->tx_lock);
3510 i_ldc_reset(ldcp, B_TRUE);
3511 mutex_exit(&ldcp->tx_lock);
3512 mutex_exit(&ldcp->lock);
3513
3514 return (0);
3515 }
3516
3517 /*
3518 * Get the current channel status
3519 */
3520 int
ldc_status(ldc_handle_t handle,ldc_status_t * status)3521 ldc_status(ldc_handle_t handle, ldc_status_t *status)
3522 {
3523 ldc_chan_t *ldcp;
3524
3525 if (handle == 0 || status == NULL) {
3526 DWARN(DBG_ALL_LDCS, "ldc_status: invalid argument\n");
3527 return (EINVAL);
3528 }
3529 ldcp = (ldc_chan_t *)handle;
3530
3531 *status = ((ldc_chan_t *)handle)->status;
3532
3533 D1(ldcp->id,
3534 "ldc_status: (0x%llx) returned status %d\n", ldcp->id, *status);
3535 return (0);
3536 }
3537
3538
3539 /*
3540 * Set the channel's callback mode - enable/disable callbacks
3541 */
3542 int
ldc_set_cb_mode(ldc_handle_t handle,ldc_cb_mode_t cmode)3543 ldc_set_cb_mode(ldc_handle_t handle, ldc_cb_mode_t cmode)
3544 {
3545 ldc_chan_t *ldcp;
3546
3547 if (handle == 0) {
3548 DWARN(DBG_ALL_LDCS,
3549 "ldc_set_intr_mode: invalid channel handle\n");
3550 return (EINVAL);
3551 }
3552 ldcp = (ldc_chan_t *)handle;
3553
3554 /*
3555 * Record no callbacks should be invoked
3556 */
3557 mutex_enter(&ldcp->lock);
3558
3559 switch (cmode) {
3560 case LDC_CB_DISABLE:
3561 if (!ldcp->cb_enabled) {
3562 DWARN(ldcp->id,
3563 "ldc_set_cb_mode: (0x%llx) callbacks disabled\n",
3564 ldcp->id);
3565 break;
3566 }
3567 ldcp->cb_enabled = B_FALSE;
3568
3569 D1(ldcp->id, "ldc_set_cb_mode: (0x%llx) disabled callbacks\n",
3570 ldcp->id);
3571 break;
3572
3573 case LDC_CB_ENABLE:
3574 if (ldcp->cb_enabled) {
3575 DWARN(ldcp->id,
3576 "ldc_set_cb_mode: (0x%llx) callbacks enabled\n",
3577 ldcp->id);
3578 break;
3579 }
3580 ldcp->cb_enabled = B_TRUE;
3581
3582 D1(ldcp->id, "ldc_set_cb_mode: (0x%llx) enabled callbacks\n",
3583 ldcp->id);
3584 break;
3585 }
3586
3587 mutex_exit(&ldcp->lock);
3588
3589 return (0);
3590 }
3591
3592 /*
3593 * Check to see if there are packets on the incoming queue
3594 * Will return hasdata = B_FALSE if there are no packets
3595 */
3596 int
ldc_chkq(ldc_handle_t handle,boolean_t * hasdata)3597 ldc_chkq(ldc_handle_t handle, boolean_t *hasdata)
3598 {
3599 int rv;
3600 uint64_t rx_head, rx_tail;
3601 ldc_chan_t *ldcp;
3602
3603 if (handle == 0) {
3604 DWARN(DBG_ALL_LDCS, "ldc_chkq: invalid channel handle\n");
3605 return (EINVAL);
3606 }
3607 ldcp = (ldc_chan_t *)handle;
3608
3609 *hasdata = B_FALSE;
3610
3611 mutex_enter(&ldcp->lock);
3612
3613 if (ldcp->tstate != TS_UP) {
3614 D1(ldcp->id,
3615 "ldc_chkq: (0x%llx) channel is not up\n", ldcp->id);
3616 mutex_exit(&ldcp->lock);
3617 return (ECONNRESET);
3618 }
3619
3620 /* Read packet(s) from the queue */
3621 rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail,
3622 &ldcp->link_state);
3623 if (rv != 0) {
3624 cmn_err(CE_WARN,
3625 "ldc_chkq: (0x%lx) unable to read queue ptrs", ldcp->id);
3626 mutex_exit(&ldcp->lock);
3627 return (EIO);
3628 }
3629
3630 /* reset the channel state if the channel went down */
3631 if (ldcp->link_state == LDC_CHANNEL_DOWN ||
3632 ldcp->link_state == LDC_CHANNEL_RESET) {
3633 mutex_enter(&ldcp->tx_lock);
3634 i_ldc_reset(ldcp, B_FALSE);
3635 mutex_exit(&ldcp->tx_lock);
3636 mutex_exit(&ldcp->lock);
3637 return (ECONNRESET);
3638 }
3639
3640 switch (ldcp->mode) {
3641 case LDC_MODE_RAW:
3642 /*
3643 * In raw mode, there are no ctrl packets, so checking
3644 * if the queue is non-empty is sufficient.
3645 */
3646 *hasdata = (rx_head != rx_tail);
3647 break;
3648
3649 case LDC_MODE_UNRELIABLE:
3650 /*
3651 * In unreliable mode, if the queue is non-empty, we need
3652 * to check if it actually contains unread data packets.
3653 * The queue may just contain ctrl packets.
3654 */
3655 if (rx_head != rx_tail) {
3656 *hasdata = (i_ldc_chkq(ldcp) == 0);
3657 /*
3658 * If no data packets were found on the queue,
3659 * all packets must have been control packets
3660 * which will now have been processed, leaving
3661 * the queue empty. If the interrupt state
3662 * is pending, we need to clear the interrupt
3663 * here.
3664 */
3665 if (*hasdata == B_FALSE &&
3666 ldcp->rx_intr_state == LDC_INTR_PEND) {
3667 i_ldc_clear_intr(ldcp, CNEX_RX_INTR);
3668 }
3669 }
3670 break;
3671
3672 case LDC_MODE_RELIABLE:
3673 /*
3674 * In reliable mode, first check for 'stream_remains' > 0.
3675 * Otherwise, if the data queue head and tail pointers
3676 * differ, there must be data to read.
3677 */
3678 if (ldcp->stream_remains > 0)
3679 *hasdata = B_TRUE;
3680 else
3681 *hasdata = (ldcp->rx_dq_head != ldcp->rx_dq_tail);
3682 break;
3683
3684 default:
3685 cmn_err(CE_WARN, "ldc_chkq: (0x%lx) unexpected channel mode "
3686 "(0x%x)", ldcp->id, ldcp->mode);
3687 mutex_exit(&ldcp->lock);
3688 return (EIO);
3689 }
3690
3691 mutex_exit(&ldcp->lock);
3692
3693 return (0);
3694 }
3695
3696
3697 /*
3698 * Read 'size' amount of bytes or less. If incoming buffer
3699 * is more than 'size', ENOBUFS is returned.
3700 *
3701 * On return, size contains the number of bytes read.
3702 */
3703 int
ldc_read(ldc_handle_t handle,caddr_t bufp,size_t * sizep)3704 ldc_read(ldc_handle_t handle, caddr_t bufp, size_t *sizep)
3705 {
3706 ldc_chan_t *ldcp;
3707 uint64_t rx_head = 0, rx_tail = 0;
3708 int rv = 0, exit_val;
3709
3710 if (handle == 0) {
3711 DWARN(DBG_ALL_LDCS, "ldc_read: invalid channel handle\n");
3712 return (EINVAL);
3713 }
3714
3715 ldcp = (ldc_chan_t *)handle;
3716
3717 /* channel lock */
3718 mutex_enter(&ldcp->lock);
3719
3720 if (ldcp->tstate != TS_UP) {
3721 DWARN(ldcp->id,
3722 "ldc_read: (0x%llx) channel is not in UP state\n",
3723 ldcp->id);
3724 exit_val = ECONNRESET;
3725 } else if (ldcp->mode == LDC_MODE_RELIABLE) {
3726 TRACE_RXDQ_LENGTH(ldcp);
3727 exit_val = ldcp->read_p(ldcp, bufp, sizep);
3728
3729 /*
3730 * For reliable mode channels, the interrupt
3731 * state is only set to pending during
3732 * interrupt handling when the secondary data
3733 * queue became full, leaving unprocessed
3734 * packets on the Rx queue. If the interrupt
3735 * state is pending and space is now available
3736 * on the data queue, clear the interrupt.
3737 */
3738 if (ldcp->rx_intr_state == LDC_INTR_PEND &&
3739 Q_CONTIG_SPACE(ldcp->rx_dq_head, ldcp->rx_dq_tail,
3740 ldcp->rx_dq_entries << LDC_PACKET_SHIFT) >=
3741 LDC_PACKET_SIZE) {
3742 /* data queue is not full */
3743 i_ldc_clear_intr(ldcp, CNEX_RX_INTR);
3744 }
3745
3746 mutex_exit(&ldcp->lock);
3747 return (exit_val);
3748 } else {
3749 exit_val = ldcp->read_p(ldcp, bufp, sizep);
3750 }
3751
3752 /*
3753 * if queue has been drained - clear interrupt
3754 */
3755 rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail,
3756 &ldcp->link_state);
3757 if (rv != 0) {
3758 cmn_err(CE_WARN, "ldc_read: (0x%lx) unable to read queue ptrs",
3759 ldcp->id);
3760 mutex_enter(&ldcp->tx_lock);
3761 i_ldc_reset(ldcp, B_TRUE);
3762 mutex_exit(&ldcp->tx_lock);
3763 mutex_exit(&ldcp->lock);
3764 return (ECONNRESET);
3765 }
3766
3767 if (exit_val == 0) {
3768 if (ldcp->link_state == LDC_CHANNEL_DOWN ||
3769 ldcp->link_state == LDC_CHANNEL_RESET) {
3770 mutex_enter(&ldcp->tx_lock);
3771 i_ldc_reset(ldcp, B_FALSE);
3772 exit_val = ECONNRESET;
3773 mutex_exit(&ldcp->tx_lock);
3774 }
3775 if ((rv == 0) &&
3776 (ldcp->rx_intr_state == LDC_INTR_PEND) &&
3777 (rx_head == rx_tail)) {
3778 i_ldc_clear_intr(ldcp, CNEX_RX_INTR);
3779 }
3780 }
3781
3782 mutex_exit(&ldcp->lock);
3783 return (exit_val);
3784 }
3785
3786 /*
3787 * Basic raw mondo read -
3788 * no interpretation of mondo contents at all.
3789 *
3790 * Enter and exit with ldcp->lock held by caller
3791 */
3792 static int
i_ldc_read_raw(ldc_chan_t * ldcp,caddr_t target_bufp,size_t * sizep)3793 i_ldc_read_raw(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep)
3794 {
3795 uint64_t q_size_mask;
3796 ldc_msg_t *msgp;
3797 uint8_t *msgbufp;
3798 int rv = 0, space;
3799 uint64_t rx_head, rx_tail;
3800
3801 space = *sizep;
3802
3803 if (space < LDC_PAYLOAD_SIZE_RAW)
3804 return (ENOBUFS);
3805
3806 ASSERT(mutex_owned(&ldcp->lock));
3807
3808 /* compute mask for increment */
3809 q_size_mask = (ldcp->rx_q_entries-1)<<LDC_PACKET_SHIFT;
3810
3811 /*
3812 * Read packet(s) from the queue
3813 */
3814 rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail,
3815 &ldcp->link_state);
3816 if (rv != 0) {
3817 cmn_err(CE_WARN,
3818 "ldc_read_raw: (0x%lx) unable to read queue ptrs",
3819 ldcp->id);
3820 return (EIO);
3821 }
3822 D1(ldcp->id, "ldc_read_raw: (0x%llx) rxh=0x%llx,"
3823 " rxt=0x%llx, st=0x%llx\n",
3824 ldcp->id, rx_head, rx_tail, ldcp->link_state);
3825
3826 /* reset the channel state if the channel went down */
3827 if (ldcp->link_state == LDC_CHANNEL_DOWN ||
3828 ldcp->link_state == LDC_CHANNEL_RESET) {
3829 mutex_enter(&ldcp->tx_lock);
3830 i_ldc_reset(ldcp, B_FALSE);
3831 mutex_exit(&ldcp->tx_lock);
3832 return (ECONNRESET);
3833 }
3834
3835 /*
3836 * Check for empty queue
3837 */
3838 if (rx_head == rx_tail) {
3839 *sizep = 0;
3840 return (0);
3841 }
3842
3843 /* get the message */
3844 msgp = (ldc_msg_t *)(ldcp->rx_q_va + rx_head);
3845
3846 /* if channel is in RAW mode, copy data and return */
3847 msgbufp = (uint8_t *)&(msgp->raw[0]);
3848
3849 bcopy(msgbufp, target_bufp, LDC_PAYLOAD_SIZE_RAW);
3850
3851 DUMP_PAYLOAD(ldcp->id, msgbufp);
3852
3853 *sizep = LDC_PAYLOAD_SIZE_RAW;
3854
3855 rx_head = (rx_head + LDC_PACKET_SIZE) & q_size_mask;
3856 rv = i_ldc_set_rx_head(ldcp, rx_head);
3857
3858 return (rv);
3859 }
3860
3861 /*
3862 * Process LDC mondos to build larger packets
3863 * with either un-reliable or reliable delivery.
3864 *
3865 * Enter and exit with ldcp->lock held by caller
3866 */
3867 static int
i_ldc_read_packet(ldc_chan_t * ldcp,caddr_t target_bufp,size_t * sizep)3868 i_ldc_read_packet(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep)
3869 {
3870 int rv = 0;
3871 uint64_t rx_head = 0, rx_tail = 0;
3872 uint64_t curr_head = 0;
3873 ldc_msg_t *msg;
3874 caddr_t target;
3875 size_t len = 0, bytes_read = 0;
3876 int retries = 0;
3877 uint64_t q_va, q_size_mask;
3878 uint64_t first_fragment = 0;
3879
3880 target = target_bufp;
3881
3882 ASSERT(mutex_owned(&ldcp->lock));
3883
3884 /* check if the buffer and size are valid */
3885 if (target_bufp == NULL || *sizep == 0) {
3886 DWARN(ldcp->id, "ldc_read: (0x%llx) invalid buffer/size\n",
3887 ldcp->id);
3888 return (EINVAL);
3889 }
3890
3891 /* Set q_va and compute increment mask for the appropriate queue */
3892 if (ldcp->mode == LDC_MODE_RELIABLE) {
3893 q_va = ldcp->rx_dq_va;
3894 q_size_mask = (ldcp->rx_dq_entries-1)<<LDC_PACKET_SHIFT;
3895 } else {
3896 q_va = ldcp->rx_q_va;
3897 q_size_mask = (ldcp->rx_q_entries-1)<<LDC_PACKET_SHIFT;
3898 }
3899
3900 /*
3901 * Read packet(s) from the queue
3902 */
3903 rv = ldcp->readq_get_state(ldcp, &curr_head, &rx_tail,
3904 &ldcp->link_state);
3905 if (rv != 0) {
3906 cmn_err(CE_WARN, "ldc_read: (0x%lx) unable to read queue ptrs",
3907 ldcp->id);
3908 mutex_enter(&ldcp->tx_lock);
3909 i_ldc_reset(ldcp, B_TRUE);
3910 mutex_exit(&ldcp->tx_lock);
3911 return (ECONNRESET);
3912 }
3913 D1(ldcp->id, "ldc_read: (0x%llx) chd=0x%llx, tl=0x%llx, st=0x%llx\n",
3914 ldcp->id, curr_head, rx_tail, ldcp->link_state);
3915
3916 /* reset the channel state if the channel went down */
3917 if (ldcp->link_state != LDC_CHANNEL_UP)
3918 goto channel_is_reset;
3919
3920 for (;;) {
3921
3922 if (curr_head == rx_tail) {
3923 /*
3924 * If a data queue is being used, check the Rx HV
3925 * queue. This will copy over any new data packets
3926 * that have arrived.
3927 */
3928 if (ldcp->mode == LDC_MODE_RELIABLE)
3929 (void) i_ldc_chkq(ldcp);
3930
3931 rv = ldcp->readq_get_state(ldcp,
3932 &rx_head, &rx_tail, &ldcp->link_state);
3933 if (rv != 0) {
3934 cmn_err(CE_WARN,
3935 "ldc_read: (0x%lx) cannot read queue ptrs",
3936 ldcp->id);
3937 mutex_enter(&ldcp->tx_lock);
3938 i_ldc_reset(ldcp, B_TRUE);
3939 mutex_exit(&ldcp->tx_lock);
3940 return (ECONNRESET);
3941 }
3942
3943 if (ldcp->link_state != LDC_CHANNEL_UP)
3944 goto channel_is_reset;
3945
3946 if (curr_head == rx_tail) {
3947
3948 /* If in the middle of a fragmented xfer */
3949 if (first_fragment != 0) {
3950
3951 /* wait for ldc_delay usecs */
3952 drv_usecwait(ldc_delay);
3953
3954 if (++retries < ldc_max_retries)
3955 continue;
3956
3957 *sizep = 0;
3958 if (ldcp->mode != LDC_MODE_RELIABLE)
3959 ldcp->last_msg_rcd =
3960 first_fragment - 1;
3961 DWARN(DBG_ALL_LDCS, "ldc_read: "
3962 "(0x%llx) read timeout", ldcp->id);
3963 return (EAGAIN);
3964 }
3965 *sizep = 0;
3966 break;
3967 }
3968 }
3969 retries = 0;
3970
3971 D2(ldcp->id,
3972 "ldc_read: (0x%llx) chd=0x%llx, rxhd=0x%llx, rxtl=0x%llx\n",
3973 ldcp->id, curr_head, rx_head, rx_tail);
3974
3975 /* get the message */
3976 msg = (ldc_msg_t *)(q_va + curr_head);
3977
3978 DUMP_LDC_PKT(ldcp, "ldc_read received pkt",
3979 ldcp->rx_q_va + curr_head);
3980
3981 /* Check the message ID for the message received */
3982 if (ldcp->mode != LDC_MODE_RELIABLE) {
3983 if ((rv = i_ldc_check_seqid(ldcp, msg)) != 0) {
3984
3985 DWARN(ldcp->id, "ldc_read: (0x%llx) seqid "
3986 "error, q_ptrs=0x%lx,0x%lx",
3987 ldcp->id, rx_head, rx_tail);
3988
3989 /* throw away data */
3990 bytes_read = 0;
3991
3992 /* Reset last_msg_rcd to start of message */
3993 if (first_fragment != 0) {
3994 ldcp->last_msg_rcd = first_fragment - 1;
3995 first_fragment = 0;
3996 }
3997 /*
3998 * Send a NACK -- invalid seqid
3999 * get the current tail for the response
4000 */
4001 rv = i_ldc_send_pkt(ldcp, msg->type, LDC_NACK,
4002 (msg->ctrl & LDC_CTRL_MASK));
4003 if (rv) {
4004 cmn_err(CE_NOTE,
4005 "ldc_read: (0x%lx) err sending "
4006 "NACK msg\n", ldcp->id);
4007
4008 /* if cannot send NACK - reset chan */
4009 mutex_enter(&ldcp->tx_lock);
4010 i_ldc_reset(ldcp, B_FALSE);
4011 mutex_exit(&ldcp->tx_lock);
4012 rv = ECONNRESET;
4013 break;
4014 }
4015
4016 /* purge receive queue */
4017 rv = i_ldc_set_rx_head(ldcp, rx_tail);
4018
4019 break;
4020 }
4021
4022 /*
4023 * Process any messages of type CTRL messages
4024 * Future implementations should try to pass these
4025 * to LDC link by resetting the intr state.
4026 *
4027 * NOTE: not done as a switch() as type can be
4028 * both ctrl+data
4029 */
4030 if (msg->type & LDC_CTRL) {
4031 if (rv = i_ldc_ctrlmsg(ldcp, msg)) {
4032 if (rv == EAGAIN)
4033 continue;
4034 rv = i_ldc_set_rx_head(ldcp, rx_tail);
4035 *sizep = 0;
4036 bytes_read = 0;
4037 break;
4038 }
4039 }
4040
4041 /* process data ACKs */
4042 if ((msg->type & LDC_DATA) && (msg->stype & LDC_ACK)) {
4043 if (rv = i_ldc_process_data_ACK(ldcp, msg)) {
4044 *sizep = 0;
4045 bytes_read = 0;
4046 break;
4047 }
4048 }
4049
4050 /* process data NACKs */
4051 if ((msg->type & LDC_DATA) && (msg->stype & LDC_NACK)) {
4052 DWARN(ldcp->id,
4053 "ldc_read: (0x%llx) received DATA/NACK",
4054 ldcp->id);
4055 mutex_enter(&ldcp->tx_lock);
4056 i_ldc_reset(ldcp, B_TRUE);
4057 mutex_exit(&ldcp->tx_lock);
4058 return (ECONNRESET);
4059 }
4060 }
4061
4062 /* process data messages */
4063 if ((msg->type & LDC_DATA) && (msg->stype & LDC_INFO)) {
4064
4065 uint8_t *msgbuf = (uint8_t *)(
4066 (ldcp->mode == LDC_MODE_RELIABLE) ?
4067 msg->rdata : msg->udata);
4068
4069 D2(ldcp->id,
4070 "ldc_read: (0x%llx) received data msg\n", ldcp->id);
4071
4072 /* get the packet length */
4073 len = (msg->env & LDC_LEN_MASK);
4074
4075 /*
4076 * FUTURE OPTIMIZATION:
4077 * dont need to set q head for every
4078 * packet we read just need to do this when
4079 * we are done or need to wait for more
4080 * mondos to make a full packet - this is
4081 * currently expensive.
4082 */
4083
4084 if (first_fragment == 0) {
4085
4086 /*
4087 * first packets should always have the start
4088 * bit set (even for a single packet). If not
4089 * throw away the packet
4090 */
4091 if (!(msg->env & LDC_FRAG_START)) {
4092
4093 DWARN(DBG_ALL_LDCS,
4094 "ldc_read: (0x%llx) not start - "
4095 "frag=%x\n", ldcp->id,
4096 (msg->env) & LDC_FRAG_MASK);
4097
4098 /* toss pkt, inc head, cont reading */
4099 bytes_read = 0;
4100 target = target_bufp;
4101 curr_head =
4102 (curr_head + LDC_PACKET_SIZE)
4103 & q_size_mask;
4104 if (rv = ldcp->readq_set_head(ldcp,
4105 curr_head))
4106 break;
4107
4108 continue;
4109 }
4110
4111 first_fragment = msg->seqid;
4112 } else {
4113 /* check to see if this is a pkt w/ START bit */
4114 if (msg->env & LDC_FRAG_START) {
4115 DWARN(DBG_ALL_LDCS,
4116 "ldc_read:(0x%llx) unexpected pkt"
4117 " env=0x%x discarding %d bytes,"
4118 " lastmsg=%d, currentmsg=%d\n",
4119 ldcp->id, msg->env&LDC_FRAG_MASK,
4120 bytes_read, ldcp->last_msg_rcd,
4121 msg->seqid);
4122
4123 /* throw data we have read so far */
4124 bytes_read = 0;
4125 target = target_bufp;
4126 first_fragment = msg->seqid;
4127
4128 if (rv = ldcp->readq_set_head(ldcp,
4129 curr_head))
4130 break;
4131 }
4132 }
4133
4134 /* copy (next) pkt into buffer */
4135 if (len <= (*sizep - bytes_read)) {
4136 bcopy(msgbuf, target, len);
4137 target += len;
4138 bytes_read += len;
4139 } else {
4140 /*
4141 * there is not enough space in the buffer to
4142 * read this pkt. throw message away & continue
4143 * reading data from queue
4144 */
4145 DWARN(DBG_ALL_LDCS,
4146 "ldc_read: (0x%llx) buffer too small, "
4147 "head=0x%lx, expect=%d, got=%d\n", ldcp->id,
4148 curr_head, *sizep, bytes_read+len);
4149
4150 first_fragment = 0;
4151 target = target_bufp;
4152 bytes_read = 0;
4153
4154 /* throw away everything received so far */
4155 if (rv = ldcp->readq_set_head(ldcp, curr_head))
4156 break;
4157
4158 /* continue reading remaining pkts */
4159 continue;
4160 }
4161 }
4162
4163 /* set the message id */
4164 if (ldcp->mode != LDC_MODE_RELIABLE)
4165 ldcp->last_msg_rcd = msg->seqid;
4166
4167 /* move the head one position */
4168 curr_head = (curr_head + LDC_PACKET_SIZE) & q_size_mask;
4169
4170 if (msg->env & LDC_FRAG_STOP) {
4171
4172 /*
4173 * All pkts that are part of this fragmented transfer
4174 * have been read or this was a single pkt read
4175 * or there was an error
4176 */
4177
4178 /* set the queue head */
4179 if (rv = ldcp->readq_set_head(ldcp, curr_head))
4180 bytes_read = 0;
4181
4182 *sizep = bytes_read;
4183
4184 break;
4185 }
4186
4187 /* advance head if it is a CTRL packet or a DATA ACK packet */
4188 if ((msg->type & LDC_CTRL) ||
4189 ((msg->type & LDC_DATA) && (msg->stype & LDC_ACK))) {
4190
4191 /* set the queue head */
4192 if (rv = ldcp->readq_set_head(ldcp, curr_head)) {
4193 bytes_read = 0;
4194 break;
4195 }
4196
4197 D2(ldcp->id, "ldc_read: (0x%llx) set ACK qhead 0x%llx",
4198 ldcp->id, curr_head);
4199 }
4200
4201 } /* for (;;) */
4202
4203 D2(ldcp->id, "ldc_read: (0x%llx) end size=%d", ldcp->id, *sizep);
4204
4205 return (rv);
4206
4207 channel_is_reset:
4208 mutex_enter(&ldcp->tx_lock);
4209 i_ldc_reset(ldcp, B_FALSE);
4210 mutex_exit(&ldcp->tx_lock);
4211 return (ECONNRESET);
4212 }
4213
4214 /*
4215 * Fetch and buffer incoming packets so we can hand them back as
4216 * a basic byte stream.
4217 *
4218 * Enter and exit with ldcp->lock held by caller
4219 */
4220 static int
i_ldc_read_stream(ldc_chan_t * ldcp,caddr_t target_bufp,size_t * sizep)4221 i_ldc_read_stream(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep)
4222 {
4223 int rv;
4224 size_t size;
4225
4226 ASSERT(mutex_owned(&ldcp->lock));
4227
4228 D2(ldcp->id, "i_ldc_read_stream: (0x%llx) buffer size=%d",
4229 ldcp->id, *sizep);
4230
4231 if (ldcp->stream_remains == 0) {
4232 size = ldcp->mtu;
4233 rv = i_ldc_read_packet(ldcp,
4234 (caddr_t)ldcp->stream_bufferp, &size);
4235 D2(ldcp->id, "i_ldc_read_stream: read packet (0x%llx) size=%d",
4236 ldcp->id, size);
4237
4238 if (rv != 0)
4239 return (rv);
4240
4241 ldcp->stream_remains = size;
4242 ldcp->stream_offset = 0;
4243 }
4244
4245 size = MIN(ldcp->stream_remains, *sizep);
4246
4247 bcopy(ldcp->stream_bufferp + ldcp->stream_offset, target_bufp, size);
4248 ldcp->stream_offset += size;
4249 ldcp->stream_remains -= size;
4250
4251 D2(ldcp->id, "i_ldc_read_stream: (0x%llx) fill from buffer size=%d",
4252 ldcp->id, size);
4253
4254 *sizep = size;
4255 return (0);
4256 }
4257
4258 /*
4259 * Write specified amount of bytes to the channel
4260 * in multiple pkts of pkt_payload size. Each
4261 * packet is tagged with an unique packet ID in
4262 * the case of a reliable link.
4263 *
4264 * On return, size contains the number of bytes written.
4265 */
4266 int
ldc_write(ldc_handle_t handle,caddr_t buf,size_t * sizep)4267 ldc_write(ldc_handle_t handle, caddr_t buf, size_t *sizep)
4268 {
4269 ldc_chan_t *ldcp;
4270 int rv = 0;
4271
4272 if (handle == 0) {
4273 DWARN(DBG_ALL_LDCS, "ldc_write: invalid channel handle\n");
4274 return (EINVAL);
4275 }
4276 ldcp = (ldc_chan_t *)handle;
4277
4278 mutex_enter(&ldcp->tx_lock);
4279
4280 /* check if non-zero data to write */
4281 if (buf == NULL || sizep == NULL) {
4282 DWARN(ldcp->id, "ldc_write: (0x%llx) invalid data write\n",
4283 ldcp->id);
4284 mutex_exit(&ldcp->tx_lock);
4285 return (EINVAL);
4286 }
4287
4288 if (*sizep == 0) {
4289 DWARN(ldcp->id, "ldc_write: (0x%llx) write size of zero\n",
4290 ldcp->id);
4291 mutex_exit(&ldcp->tx_lock);
4292 return (0);
4293 }
4294
4295 /* Check if channel is UP for data exchange */
4296 if (ldcp->tstate != TS_UP) {
4297 DWARN(ldcp->id,
4298 "ldc_write: (0x%llx) channel is not in UP state\n",
4299 ldcp->id);
4300 *sizep = 0;
4301 rv = ECONNRESET;
4302 } else {
4303 rv = ldcp->write_p(ldcp, buf, sizep);
4304 }
4305
4306 mutex_exit(&ldcp->tx_lock);
4307
4308 return (rv);
4309 }
4310
4311 /*
4312 * Write a raw packet to the channel
4313 * On return, size contains the number of bytes written.
4314 */
4315 static int
i_ldc_write_raw(ldc_chan_t * ldcp,caddr_t buf,size_t * sizep)4316 i_ldc_write_raw(ldc_chan_t *ldcp, caddr_t buf, size_t *sizep)
4317 {
4318 ldc_msg_t *ldcmsg;
4319 uint64_t tx_head, tx_tail, new_tail;
4320 int rv = 0;
4321 size_t size;
4322
4323 ASSERT(MUTEX_HELD(&ldcp->tx_lock));
4324 ASSERT(ldcp->mode == LDC_MODE_RAW);
4325
4326 size = *sizep;
4327
4328 /*
4329 * Check to see if the packet size is less than or
4330 * equal to packet size support in raw mode
4331 */
4332 if (size > ldcp->pkt_payload) {
4333 DWARN(ldcp->id,
4334 "ldc_write: (0x%llx) invalid size (0x%llx) for RAW mode\n",
4335 ldcp->id, *sizep);
4336 *sizep = 0;
4337 return (EMSGSIZE);
4338 }
4339
4340 /* get the qptrs for the tx queue */
4341 rv = hv_ldc_tx_get_state(ldcp->id,
4342 &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state);
4343 if (rv != 0) {
4344 cmn_err(CE_WARN,
4345 "ldc_write: (0x%lx) cannot read queue ptrs\n", ldcp->id);
4346 *sizep = 0;
4347 return (EIO);
4348 }
4349
4350 if (ldcp->link_state == LDC_CHANNEL_DOWN ||
4351 ldcp->link_state == LDC_CHANNEL_RESET) {
4352 DWARN(ldcp->id,
4353 "ldc_write: (0x%llx) channel down/reset\n", ldcp->id);
4354
4355 *sizep = 0;
4356 if (mutex_tryenter(&ldcp->lock)) {
4357 i_ldc_reset(ldcp, B_FALSE);
4358 mutex_exit(&ldcp->lock);
4359 } else {
4360 /*
4361 * Release Tx lock, and then reacquire channel
4362 * and Tx lock in correct order
4363 */
4364 mutex_exit(&ldcp->tx_lock);
4365 mutex_enter(&ldcp->lock);
4366 mutex_enter(&ldcp->tx_lock);
4367 i_ldc_reset(ldcp, B_FALSE);
4368 mutex_exit(&ldcp->lock);
4369 }
4370 return (ECONNRESET);
4371 }
4372
4373 tx_tail = ldcp->tx_tail;
4374 tx_head = ldcp->tx_head;
4375 new_tail = (tx_tail + LDC_PACKET_SIZE) &
4376 ((ldcp->tx_q_entries-1) << LDC_PACKET_SHIFT);
4377
4378 if (new_tail == tx_head) {
4379 DWARN(DBG_ALL_LDCS,
4380 "ldc_write: (0x%llx) TX queue is full\n", ldcp->id);
4381 *sizep = 0;
4382 return (EWOULDBLOCK);
4383 }
4384
4385 D2(ldcp->id, "ldc_write: (0x%llx) start xfer size=%d",
4386 ldcp->id, size);
4387
4388 /* Send the data now */
4389 ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail);
4390
4391 /* copy the data into pkt */
4392 bcopy((uint8_t *)buf, ldcmsg, size);
4393
4394 /* increment tail */
4395 tx_tail = new_tail;
4396
4397 /*
4398 * All packets have been copied into the TX queue
4399 * update the tail ptr in the HV
4400 */
4401 rv = i_ldc_set_tx_tail(ldcp, tx_tail);
4402 if (rv) {
4403 if (rv == EWOULDBLOCK) {
4404 DWARN(ldcp->id, "ldc_write: (0x%llx) write timed out\n",
4405 ldcp->id);
4406 *sizep = 0;
4407 return (EWOULDBLOCK);
4408 }
4409
4410 *sizep = 0;
4411 if (mutex_tryenter(&ldcp->lock)) {
4412 i_ldc_reset(ldcp, B_FALSE);
4413 mutex_exit(&ldcp->lock);
4414 } else {
4415 /*
4416 * Release Tx lock, and then reacquire channel
4417 * and Tx lock in correct order
4418 */
4419 mutex_exit(&ldcp->tx_lock);
4420 mutex_enter(&ldcp->lock);
4421 mutex_enter(&ldcp->tx_lock);
4422 i_ldc_reset(ldcp, B_FALSE);
4423 mutex_exit(&ldcp->lock);
4424 }
4425 return (ECONNRESET);
4426 }
4427
4428 ldcp->tx_tail = tx_tail;
4429 *sizep = size;
4430
4431 D2(ldcp->id, "ldc_write: (0x%llx) end xfer size=%d", ldcp->id, size);
4432
4433 return (rv);
4434 }
4435
4436
4437 /*
4438 * Write specified amount of bytes to the channel
4439 * in multiple pkts of pkt_payload size. Each
4440 * packet is tagged with an unique packet ID in
4441 * the case of a reliable link.
4442 *
4443 * On return, size contains the number of bytes written.
4444 * This function needs to ensure that the write size is < MTU size
4445 */
4446 static int
i_ldc_write_packet(ldc_chan_t * ldcp,caddr_t buf,size_t * size)4447 i_ldc_write_packet(ldc_chan_t *ldcp, caddr_t buf, size_t *size)
4448 {
4449 ldc_msg_t *ldcmsg;
4450 uint64_t tx_head, tx_tail, new_tail, start;
4451 uint64_t txq_size_mask, numavail;
4452 uint8_t *msgbuf, *source = (uint8_t *)buf;
4453 size_t len, bytes_written = 0, remaining;
4454 int rv;
4455 uint32_t curr_seqid;
4456
4457 ASSERT(MUTEX_HELD(&ldcp->tx_lock));
4458
4459 ASSERT(ldcp->mode == LDC_MODE_RELIABLE ||
4460 ldcp->mode == LDC_MODE_UNRELIABLE);
4461
4462 /* compute mask for increment */
4463 txq_size_mask = (ldcp->tx_q_entries - 1) << LDC_PACKET_SHIFT;
4464
4465 /* get the qptrs for the tx queue */
4466 rv = hv_ldc_tx_get_state(ldcp->id,
4467 &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state);
4468 if (rv != 0) {
4469 cmn_err(CE_WARN,
4470 "ldc_write: (0x%lx) cannot read queue ptrs\n", ldcp->id);
4471 *size = 0;
4472 return (EIO);
4473 }
4474
4475 if (ldcp->link_state == LDC_CHANNEL_DOWN ||
4476 ldcp->link_state == LDC_CHANNEL_RESET) {
4477 DWARN(ldcp->id,
4478 "ldc_write: (0x%llx) channel down/reset\n", ldcp->id);
4479 *size = 0;
4480 if (mutex_tryenter(&ldcp->lock)) {
4481 i_ldc_reset(ldcp, B_FALSE);
4482 mutex_exit(&ldcp->lock);
4483 } else {
4484 /*
4485 * Release Tx lock, and then reacquire channel
4486 * and Tx lock in correct order
4487 */
4488 mutex_exit(&ldcp->tx_lock);
4489 mutex_enter(&ldcp->lock);
4490 mutex_enter(&ldcp->tx_lock);
4491 i_ldc_reset(ldcp, B_FALSE);
4492 mutex_exit(&ldcp->lock);
4493 }
4494 return (ECONNRESET);
4495 }
4496
4497 tx_tail = ldcp->tx_tail;
4498 new_tail = (tx_tail + LDC_PACKET_SIZE) %
4499 (ldcp->tx_q_entries << LDC_PACKET_SHIFT);
4500
4501 /*
4502 * Check to see if the queue is full. The check is done using
4503 * the appropriate head based on the link mode.
4504 */
4505 i_ldc_get_tx_head(ldcp, &tx_head);
4506
4507 if (new_tail == tx_head) {
4508 DWARN(DBG_ALL_LDCS,
4509 "ldc_write: (0x%llx) TX queue is full\n", ldcp->id);
4510 *size = 0;
4511 return (EWOULDBLOCK);
4512 }
4513
4514 /*
4515 * Make sure that the LDC Tx queue has enough space
4516 */
4517 numavail = (tx_head >> LDC_PACKET_SHIFT) - (tx_tail >> LDC_PACKET_SHIFT)
4518 + ldcp->tx_q_entries - 1;
4519 numavail %= ldcp->tx_q_entries;
4520
4521 if (*size > (numavail * ldcp->pkt_payload)) {
4522 DWARN(DBG_ALL_LDCS,
4523 "ldc_write: (0x%llx) TX queue has no space\n", ldcp->id);
4524 return (EWOULDBLOCK);
4525 }
4526
4527 D2(ldcp->id, "ldc_write: (0x%llx) start xfer size=%d",
4528 ldcp->id, *size);
4529
4530 /* Send the data now */
4531 bytes_written = 0;
4532 curr_seqid = ldcp->last_msg_snt;
4533 start = tx_tail;
4534
4535 while (*size > bytes_written) {
4536
4537 ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail);
4538
4539 msgbuf = (uint8_t *)((ldcp->mode == LDC_MODE_RELIABLE) ?
4540 ldcmsg->rdata : ldcmsg->udata);
4541
4542 ldcmsg->type = LDC_DATA;
4543 ldcmsg->stype = LDC_INFO;
4544 ldcmsg->ctrl = 0;
4545
4546 remaining = *size - bytes_written;
4547 len = min(ldcp->pkt_payload, remaining);
4548 ldcmsg->env = (uint8_t)len;
4549
4550 curr_seqid++;
4551 ldcmsg->seqid = curr_seqid;
4552
4553 /* copy the data into pkt */
4554 bcopy(source, msgbuf, len);
4555
4556 source += len;
4557 bytes_written += len;
4558
4559 /* increment tail */
4560 tx_tail = (tx_tail + LDC_PACKET_SIZE) & txq_size_mask;
4561
4562 ASSERT(tx_tail != tx_head);
4563 }
4564
4565 /* Set the start and stop bits */
4566 ldcmsg->env |= LDC_FRAG_STOP;
4567 ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + start);
4568 ldcmsg->env |= LDC_FRAG_START;
4569
4570 /*
4571 * All packets have been copied into the TX queue
4572 * update the tail ptr in the HV
4573 */
4574 rv = i_ldc_set_tx_tail(ldcp, tx_tail);
4575 if (rv == 0) {
4576 ldcp->tx_tail = tx_tail;
4577 ldcp->last_msg_snt = curr_seqid;
4578 *size = bytes_written;
4579 } else {
4580 int rv2;
4581
4582 if (rv != EWOULDBLOCK) {
4583 *size = 0;
4584 if (mutex_tryenter(&ldcp->lock)) {
4585 i_ldc_reset(ldcp, B_FALSE);
4586 mutex_exit(&ldcp->lock);
4587 } else {
4588 /*
4589 * Release Tx lock, and then reacquire channel
4590 * and Tx lock in correct order
4591 */
4592 mutex_exit(&ldcp->tx_lock);
4593 mutex_enter(&ldcp->lock);
4594 mutex_enter(&ldcp->tx_lock);
4595 i_ldc_reset(ldcp, B_FALSE);
4596 mutex_exit(&ldcp->lock);
4597 }
4598 return (ECONNRESET);
4599 }
4600
4601 D1(ldcp->id, "hv_tx_set_tail returns 0x%x (head 0x%x, "
4602 "old tail 0x%x, new tail 0x%x, qsize=0x%x)\n",
4603 rv, ldcp->tx_head, ldcp->tx_tail, tx_tail,
4604 (ldcp->tx_q_entries << LDC_PACKET_SHIFT));
4605
4606 rv2 = hv_ldc_tx_get_state(ldcp->id,
4607 &tx_head, &tx_tail, &ldcp->link_state);
4608
4609 D1(ldcp->id, "hv_ldc_tx_get_state returns 0x%x "
4610 "(head 0x%x, tail 0x%x state 0x%x)\n",
4611 rv2, tx_head, tx_tail, ldcp->link_state);
4612
4613 *size = 0;
4614 }
4615
4616 D2(ldcp->id, "ldc_write: (0x%llx) end xfer size=%d", ldcp->id, *size);
4617
4618 return (rv);
4619 }
4620
4621 /*
4622 * Write specified amount of bytes to the channel
4623 * in multiple pkts of pkt_payload size. Each
4624 * packet is tagged with an unique packet ID in
4625 * the case of a reliable link.
4626 *
4627 * On return, size contains the number of bytes written.
4628 * This function needs to ensure that the write size is < MTU size
4629 */
4630 static int
i_ldc_write_stream(ldc_chan_t * ldcp,caddr_t buf,size_t * sizep)4631 i_ldc_write_stream(ldc_chan_t *ldcp, caddr_t buf, size_t *sizep)
4632 {
4633 ASSERT(MUTEX_HELD(&ldcp->tx_lock));
4634 ASSERT(ldcp->mode == LDC_MODE_RELIABLE);
4635
4636 /* Truncate packet to max of MTU size */
4637 if (*sizep > ldcp->mtu) *sizep = ldcp->mtu;
4638 return (i_ldc_write_packet(ldcp, buf, sizep));
4639 }
4640
4641
4642 /*
4643 * Interfaces for channel nexus to register/unregister with LDC module
4644 * The nexus will register functions to be used to register individual
4645 * channels with the nexus and enable interrupts for the channels
4646 */
4647 int
ldc_register(ldc_cnex_t * cinfo)4648 ldc_register(ldc_cnex_t *cinfo)
4649 {
4650 ldc_chan_t *ldcp;
4651
4652 if (cinfo == NULL || cinfo->dip == NULL ||
4653 cinfo->reg_chan == NULL || cinfo->unreg_chan == NULL ||
4654 cinfo->add_intr == NULL || cinfo->rem_intr == NULL ||
4655 cinfo->clr_intr == NULL) {
4656
4657 DWARN(DBG_ALL_LDCS, "ldc_register: invalid nexus info\n");
4658 return (EINVAL);
4659 }
4660
4661 mutex_enter(&ldcssp->lock);
4662
4663 /* nexus registration */
4664 ldcssp->cinfo.dip = cinfo->dip;
4665 ldcssp->cinfo.reg_chan = cinfo->reg_chan;
4666 ldcssp->cinfo.unreg_chan = cinfo->unreg_chan;
4667 ldcssp->cinfo.add_intr = cinfo->add_intr;
4668 ldcssp->cinfo.rem_intr = cinfo->rem_intr;
4669 ldcssp->cinfo.clr_intr = cinfo->clr_intr;
4670
4671 /* register any channels that might have been previously initialized */
4672 ldcp = ldcssp->chan_list;
4673 while (ldcp) {
4674 if ((ldcp->tstate & TS_QCONF_RDY) &&
4675 (ldcp->tstate & TS_CNEX_RDY) == 0)
4676 (void) i_ldc_register_channel(ldcp);
4677
4678 ldcp = ldcp->next;
4679 }
4680
4681 mutex_exit(&ldcssp->lock);
4682
4683 return (0);
4684 }
4685
4686 int
ldc_unregister(ldc_cnex_t * cinfo)4687 ldc_unregister(ldc_cnex_t *cinfo)
4688 {
4689 if (cinfo == NULL || cinfo->dip == NULL) {
4690 DWARN(DBG_ALL_LDCS, "ldc_unregister: invalid nexus info\n");
4691 return (EINVAL);
4692 }
4693
4694 mutex_enter(&ldcssp->lock);
4695
4696 if (cinfo->dip != ldcssp->cinfo.dip) {
4697 DWARN(DBG_ALL_LDCS, "ldc_unregister: invalid dip\n");
4698 mutex_exit(&ldcssp->lock);
4699 return (EINVAL);
4700 }
4701
4702 /* nexus unregister */
4703 ldcssp->cinfo.dip = NULL;
4704 ldcssp->cinfo.reg_chan = NULL;
4705 ldcssp->cinfo.unreg_chan = NULL;
4706 ldcssp->cinfo.add_intr = NULL;
4707 ldcssp->cinfo.rem_intr = NULL;
4708 ldcssp->cinfo.clr_intr = NULL;
4709
4710 mutex_exit(&ldcssp->lock);
4711
4712 return (0);
4713 }
4714
4715 int
ldc_info(ldc_handle_t handle,ldc_info_t * info)4716 ldc_info(ldc_handle_t handle, ldc_info_t *info)
4717 {
4718 ldc_chan_t *ldcp;
4719 uint64_t avail;
4720
4721 if (handle == 0 || info == NULL) {
4722 DWARN(DBG_ALL_LDCS, "ldc_get_info: invalid args\n");
4723 return (EINVAL);
4724 }
4725
4726 ldcp = (ldc_chan_t *)handle;
4727
4728 mutex_enter(&ldcp->lock);
4729
4730 /* check to see if channel is initalized */
4731 if ((ldcp->tstate & ~TS_IN_RESET) < TS_INIT) {
4732 DWARN(ldcp->id,
4733 "ldc_get_info: (0x%llx) channel not initialized\n",
4734 ldcp->id);
4735 mutex_exit(&ldcp->lock);
4736 return (EINVAL);
4737 }
4738
4739 mutex_exit(&ldcp->lock);
4740
4741 /*
4742 * ldcssp->mapin_size is the max amount of shared memory supported by
4743 * the Hypervisor per guest. e.g, legacy HV supports 64MB; latest HV
4744 * support 1GB. This size is read during ldc module initialization.
4745 *
4746 * ldc_dring_direct_map_rsvd is the amount of memory reserved for
4747 * mapping in descriptor rings. In the initial implementation, we use a
4748 * simple approach to determine the amount of mapin space available per
4749 * channel. In future, we may implement strict accounting of the actual
4750 * memory consumed to determine the exact amount available per channel.
4751 */
4752 if (ldcssp->mapin_size <= ldc_dring_direct_map_rsvd) {
4753 info->direct_map_size_max = 0;
4754 return (0);
4755 }
4756
4757 avail = ldcssp->mapin_size - ldc_dring_direct_map_rsvd;
4758 if (avail >= ldc_direct_map_size_max) {
4759 info->direct_map_size_max = ldc_direct_map_size_max;
4760 } else {
4761 info->direct_map_size_max = 0;
4762 }
4763
4764 return (0);
4765 }
4766