1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #include <sys/types.h>
28 #include <sys/kmem.h>
29 #include <sys/note.h>
30
31 #include "ghd.h"
32
33
34
35 /*ARGSUSED*/
36 gtgt_t *
ghd_target_init(dev_info_t * hba_dip,dev_info_t * tgt_dip,ccc_t * cccp,size_t tgt_private_size,void * hba_private,ushort_t target,uchar_t lun)37 ghd_target_init(dev_info_t *hba_dip,
38 dev_info_t *tgt_dip,
39 ccc_t *cccp,
40 size_t tgt_private_size,
41 void *hba_private,
42 ushort_t target,
43 uchar_t lun)
44 {
45 _NOTE(ARGUNUSED(hba_dip))
46 gtgt_t *gtgtp;
47 size_t size = sizeof (*gtgtp) + tgt_private_size;
48 gdev_t *gdevp;
49 ulong_t maxactive;
50
51 gtgtp = kmem_zalloc(size, KM_SLEEP);
52
53 /*
54 * initialize the per instance structure
55 */
56
57 gtgtp->gt_tgt_private = (void *)(gtgtp + 1);
58 gtgtp->gt_size = size;
59 gtgtp->gt_hba_private = hba_private;
60 gtgtp->gt_target = target;
61 gtgtp->gt_lun = lun;
62 gtgtp->gt_ccc = cccp;
63
64 /*
65 * set the queue's maxactive to 1 if
66 * property not specified on target or hba devinfo node
67 */
68 maxactive = ddi_getprop(DDI_DEV_T_ANY, tgt_dip, 0, "ghd-maxactive", 1);
69 gtgtp->gt_maxactive = maxactive;
70
71 /* initialize the linked list pointers */
72 GTGT_INIT(gtgtp);
73
74 /*
75 * grab both mutexes so the queue structures
76 * stay stable while adding this instance to the linked lists
77 */
78 mutex_enter(&cccp->ccc_hba_mutex);
79 mutex_enter(&cccp->ccc_waitq_mutex);
80
81 /*
82 * Search the HBA's linked list of device structures.
83 *
84 * If this device is already attached then link this instance
85 * to the existing per-device-structure on the ccc_devs list.
86 *
87 */
88 gdevp = CCCP2GDEVP(cccp);
89 while (gdevp != NULL) {
90 if (gdevp->gd_target == target && gdevp->gd_lun == lun) {
91 GDBG_WAITQ(("ghd_target_init(%d,%d) found gdevp 0x%p"
92 " gtgtp 0x%p max %lu\n", target, lun,
93 (void *)gdevp, (void *)gtgtp, maxactive));
94
95 goto foundit;
96 }
97 gdevp = GDEV_NEXTP(gdevp);
98 }
99
100 /*
101 * Not found. This is the first instance for this device.
102 */
103
104
105 /* allocate the per-device-structure */
106
107 gdevp = kmem_zalloc(sizeof (*gdevp), KM_SLEEP);
108 gdevp->gd_target = target;
109 gdevp->gd_lun = lun;
110
111 /*
112 * link this second level queue to the HBA's first
113 * level queue
114 */
115 GDEV_QATTACH(gdevp, cccp, maxactive);
116
117 GDBG_WAITQ(("ghd_target_init(%d,%d) new gdevp 0x%p gtgtp 0x%p"
118 " max %lu\n", target, lun, (void *)gdevp, (void *)gtgtp,
119 maxactive));
120
121 foundit:
122
123 /* save the ptr to the per device structure */
124 gtgtp->gt_gdevp = gdevp;
125
126 /* Add the per instance structure to the per device list */
127 GTGT_ATTACH(gtgtp, gdevp);
128
129 ghd_waitq_process_and_mutex_exit(cccp);
130
131 return (gtgtp);
132 }
133
134 /*ARGSUSED*/
135 void
ghd_target_free(dev_info_t * hba_dip,dev_info_t * tgt_dip,ccc_t * cccp,gtgt_t * gtgtp)136 ghd_target_free(dev_info_t *hba_dip,
137 dev_info_t *tgt_dip,
138 ccc_t *cccp,
139 gtgt_t *gtgtp)
140 {
141 _NOTE(ARGUNUSED(hba_dip,tgt_dip))
142
143 gdev_t *gdevp = gtgtp->gt_gdevp;
144
145 GDBG_WAITQ(("ghd_target_free(%d,%d) gdevp-0x%p gtgtp 0x%p\n",
146 gtgtp->gt_target, gtgtp->gt_lun, (void *)gdevp, (void *)gtgtp));
147
148 /*
149 * grab both mutexes so the queue structures
150 * stay stable while deleting this instance
151 */
152 mutex_enter(&cccp->ccc_hba_mutex);
153 mutex_enter(&cccp->ccc_waitq_mutex);
154
155 ASSERT(gdevp->gd_ninstances > 0);
156
157 /*
158 * remove this per-instance structure from the device list and
159 * free the memory
160 */
161 GTGT_DEATTACH(gtgtp, gdevp);
162 kmem_free((caddr_t)gtgtp, gtgtp->gt_size);
163
164 if (gdevp->gd_ninstances == 1) {
165 GDBG_WAITQ(("ghd_target_free: N=1 gdevp 0x%p\n",
166 (void *)gdevp));
167 /*
168 * If there's now just one instance left attached to this
169 * device then reset the queue's max active value
170 * from that instance's saved value.
171 */
172 gtgtp = GDEVP2GTGTP(gdevp);
173 GDEV_MAXACTIVE(gdevp) = gtgtp->gt_maxactive;
174
175 } else if (gdevp->gd_ninstances == 0) {
176 /* else no instances left */
177 GDBG_WAITQ(("ghd_target_free: N=0 gdevp 0x%p\n",
178 (void *)gdevp));
179
180 /* detach this per-dev-structure from the HBA's dev list */
181 GDEV_QDETACH(gdevp, cccp);
182 kmem_free(gdevp, sizeof (*gdevp));
183
184 }
185 #if defined(GHD_DEBUG) || defined(__lint)
186 else {
187 /* leave maxactive set to 1 */
188 GDBG_WAITQ(("ghd_target_free: N>1 gdevp 0x%p\n",
189 (void *)gdevp));
190 }
191 #endif
192
193 ghd_waitq_process_and_mutex_exit(cccp);
194 }
195
196 void
ghd_waitq_shuffle_up(ccc_t * cccp,gdev_t * gdevp)197 ghd_waitq_shuffle_up(ccc_t *cccp, gdev_t *gdevp)
198 {
199 gcmd_t *gcmdp;
200
201 ASSERT(mutex_owned(&cccp->ccc_waitq_mutex));
202
203 GDBG_WAITQ(("ghd_waitq_shuffle_up: cccp 0x%p gdevp 0x%p N %ld "
204 "max %ld\n", (void *)cccp, (void *)gdevp, GDEV_NACTIVE(gdevp),
205 GDEV_MAXACTIVE(gdevp)));
206 for (;;) {
207 /*
208 * Now check the device wait queue throttle to see if I can
209 * shuffle up a request to the HBA wait queue.
210 */
211 if (GDEV_NACTIVE(gdevp) >= GDEV_MAXACTIVE(gdevp)) {
212 GDBG_WAITQ(("ghd_waitq_shuffle_up: N>MAX gdevp 0x%p\n",
213 (void *)gdevp));
214 return;
215 }
216
217 /*
218 * single thread requests while multiple instances
219 * because the different target drives might have
220 * conflicting maxactive throttles.
221 */
222 if (gdevp->gd_ninstances > 1 && GDEV_NACTIVE(gdevp) > 0) {
223 GDBG_WAITQ(("ghd_waitq_shuffle_up: multi gdevp 0x%p\n",
224 (void *)gdevp));
225 return;
226 }
227
228 /*
229 * promote the topmost request from the device queue to
230 * the HBA queue.
231 */
232 if ((gcmdp = L2_remove_head(&GDEV_QHEAD(gdevp))) == NULL) {
233 /* the device is empty so we're done */
234 GDBG_WAITQ(("ghd_waitq_shuffle_up: MT gdevp 0x%p\n",
235 (void *)gdevp));
236 return;
237 }
238 L2_add(&GHBA_QHEAD(cccp), &gcmdp->cmd_q, gcmdp);
239 GDEV_NACTIVE(gdevp)++;
240 gcmdp->cmd_waitq_level++;
241 GDBG_WAITQ(("ghd_waitq_shuffle_up: gdevp 0x%p gcmdp 0x%p\n",
242 (void *)gdevp, (void *)gcmdp));
243 }
244 }
245
246
247 void
ghd_waitq_delete(ccc_t * cccp,gcmd_t * gcmdp)248 ghd_waitq_delete(ccc_t *cccp, gcmd_t *gcmdp)
249 {
250 gtgt_t *gtgtp = GCMDP2GTGTP(gcmdp);
251 gdev_t *gdevp = gtgtp->gt_gdevp;
252 #if defined(GHD_DEBUG) || defined(__lint)
253 Q_t *qp = &gdevp->gd_waitq;
254 #endif
255
256 ASSERT(mutex_owned(&cccp->ccc_hba_mutex));
257 mutex_enter(&cccp->ccc_waitq_mutex);
258
259 /*
260 * Adjust all queue counters. If this request is being aborted
261 * it might only have made it to the target queue. Otherwise,
262 * both the target and hba queue have to be adjusted when a
263 * request is completed normally. The cmd_waitq_level value
264 * indicates which queue counters need to be adjusted. It's
265 * incremented as the request progresses up the queues.
266 */
267 switch (gcmdp->cmd_waitq_level) {
268 case 0:
269 break;
270 case 1:
271 /*
272 * If this is an early-timeout, or early-abort, the request
273 * is still linked onto a waitq. Remove it now. If it's
274 * an active request and no longer on the waitq then calling
275 * L2_delete a second time does no harm.
276 */
277 L2_delete(&gcmdp->cmd_q);
278 break;
279
280 case 2:
281 L2_delete(&gcmdp->cmd_q);
282 #if defined(GHD_DEBUG) || defined(__lint)
283 if (GDEV_NACTIVE(gdevp) == 0)
284 debug_enter("\n\nGHD WAITQ DELETE\n\n");
285 #endif
286 GDEV_NACTIVE(gdevp)--;
287 break;
288
289 case 3:
290 /* it's an active or completed command */
291 #if defined(GHD_DEBUG) || defined(__lint)
292 if (GDEV_NACTIVE(gdevp) == 0 || GHBA_NACTIVE(cccp) == 0)
293 debug_enter("\n\nGHD WAITQ DELETE\n\n");
294 #endif
295 GDEV_NACTIVE(gdevp)--;
296 GHBA_NACTIVE(cccp)--;
297 break;
298
299 default:
300 /* this shouldn't happen */
301 #if defined(GHD_DEBUG) || defined(__lint)
302 debug_enter("\n\nGHD WAITQ LEVEL > 3\n\n");
303 #endif
304 break;
305 }
306
307 GDBG_WAITQ(("ghd_waitq_delete: gcmdp 0x%p qp 0x%p level %ld\n",
308 (void *)gcmdp, (void *)qp, gcmdp->cmd_waitq_level));
309
310
311 /*
312 * There's probably now more room in the HBA queue. Move
313 * up as many requests as possible.
314 */
315 ghd_waitq_shuffle_up(cccp, gdevp);
316
317 mutex_exit(&cccp->ccc_waitq_mutex);
318 }
319
320
321 int
ghd_waitq_process_and_mutex_hold(ccc_t * cccp)322 ghd_waitq_process_and_mutex_hold(ccc_t *cccp)
323 {
324 gcmd_t *gcmdp;
325 int rc = FALSE;
326
327 ASSERT(mutex_owned(&cccp->ccc_hba_mutex));
328 ASSERT(mutex_owned(&cccp->ccc_waitq_mutex));
329
330 for (;;) {
331 if (L2_EMPTY(&GHBA_QHEAD(cccp))) {
332 /* return if the list is empty */
333 GDBG_WAITQ(("ghd_waitq_proc: MT cccp 0x%p qp 0x%p\n",
334 (void *)cccp, (void *)&cccp->ccc_waitq));
335 break;
336 }
337 if (GHBA_NACTIVE(cccp) >= GHBA_MAXACTIVE(cccp)) {
338 /* return if the HBA is too active */
339 GDBG_WAITQ(("ghd_waitq_proc: N>M cccp 0x%p qp 0x%p"
340 " N %ld max %ld\n", (void *)cccp,
341 (void *)&cccp->ccc_waitq,
342 GHBA_NACTIVE(cccp),
343 GHBA_MAXACTIVE(cccp)));
344 break;
345 }
346
347 /*
348 * bail out if the wait queue has been
349 * "held" by the HBA driver
350 */
351 if (cccp->ccc_waitq_held) {
352 GDBG_WAITQ(("ghd_waitq_proc: held"));
353 return (rc);
354 }
355
356 if (cccp->ccc_waitq_frozen) {
357
358 clock_t lbolt, delay_in_hz, time_to_wait;
359
360 delay_in_hz =
361 drv_usectohz(cccp->ccc_waitq_freezedelay * 1000);
362
363 lbolt = ddi_get_lbolt();
364 time_to_wait = delay_in_hz -
365 (lbolt - cccp->ccc_waitq_freezetime);
366
367 if (time_to_wait > 0) {
368 /*
369 * stay frozen; we'll be called again
370 * by ghd_timeout_softintr()
371 */
372 GDBG_WAITQ(("ghd_waitq_proc: frozen"));
373 return (rc);
374 } else {
375 /* unfreeze and continue */
376 GDBG_WAITQ(("ghd_waitq_proc: unfreezing"));
377 cccp->ccc_waitq_freezetime = 0;
378 cccp->ccc_waitq_freezedelay = 0;
379 cccp->ccc_waitq_frozen = 0;
380 }
381 }
382
383 gcmdp = (gcmd_t *)L2_remove_head(&GHBA_QHEAD(cccp));
384 GHBA_NACTIVE(cccp)++;
385 gcmdp->cmd_waitq_level++;
386 mutex_exit(&cccp->ccc_waitq_mutex);
387
388 /*
389 * Start up the next I/O request
390 */
391 ASSERT(gcmdp != NULL);
392 gcmdp->cmd_state = GCMD_STATE_ACTIVE;
393 if (!(*cccp->ccc_hba_start)(cccp->ccc_hba_handle, gcmdp)) {
394 /* if the HBA rejected the request, requeue it */
395 gcmdp->cmd_state = GCMD_STATE_WAITQ;
396 mutex_enter(&cccp->ccc_waitq_mutex);
397 GHBA_NACTIVE(cccp)--;
398 gcmdp->cmd_waitq_level--;
399 L2_add_head(&GHBA_QHEAD(cccp), &gcmdp->cmd_q, gcmdp);
400 GDBG_WAITQ(("ghd_waitq_proc: busy cccp 0x%p gcmdp 0x%p"
401 " handle 0x%p\n", (void *)cccp, (void *)gcmdp,
402 cccp->ccc_hba_handle));
403 break;
404 }
405 rc = TRUE;
406 mutex_enter(&cccp->ccc_waitq_mutex);
407 GDBG_WAITQ(("ghd_waitq_proc: ++ cccp 0x%p gcmdp 0x%p N %ld\n",
408 (void *)cccp, (void *)gcmdp, GHBA_NACTIVE(cccp)));
409 }
410 ASSERT(mutex_owned(&cccp->ccc_hba_mutex));
411 ASSERT(mutex_owned(&cccp->ccc_waitq_mutex));
412 return (rc);
413 }
414
415 void
ghd_waitq_process_and_mutex_exit(ccc_t * cccp)416 ghd_waitq_process_and_mutex_exit(ccc_t *cccp)
417 {
418 ASSERT(mutex_owned(&cccp->ccc_hba_mutex));
419 ASSERT(mutex_owned(&cccp->ccc_waitq_mutex));
420
421 GDBG_WAITQ(("ghd_waitq_process_and_mutex_exit: cccp 0x%p\n",
422 (void *)cccp));
423
424 (void) ghd_waitq_process_and_mutex_hold(cccp);
425
426 /*
427 * Release the mutexes in the opposite order that they
428 * were acquired to prevent requests queued by
429 * ghd_transport() from getting hung up in the wait queue.
430 */
431 mutex_exit(&cccp->ccc_hba_mutex);
432 mutex_exit(&cccp->ccc_waitq_mutex);
433 }
434