xref: /illumos-gate/usr/src/uts/common/os/ddi_intr.c (revision 71222654)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 
25 #include <sys/note.h>
26 #include <sys/sysmacros.h>
27 #include <sys/types.h>
28 #include <sys/param.h>
29 #include <sys/systm.h>
30 #include <sys/kmem.h>
31 #include <sys/cmn_err.h>
32 #include <sys/debug.h>
33 #include <sys/avintr.h>
34 #include <sys/autoconf.h>
35 #include <sys/sunndi.h>
36 #include <sys/ndi_impldefs.h>	/* include prototypes */
37 #include <sys/atomic.h>
38 
39 /*
40  * New DDI interrupt framework
41  */
42 
43 /*
44  * ddi_intr_get_supported_types:
45  *	Return, as a bit mask, the hardware interrupt types supported by
46  *	both the device and by the host in the integer pointed
47  *	to be the 'typesp' argument.
48  */
49 int
ddi_intr_get_supported_types(dev_info_t * dip,int * typesp)50 ddi_intr_get_supported_types(dev_info_t *dip, int *typesp)
51 {
52 	int			ret;
53 	ddi_intr_handle_impl_t	hdl;
54 
55 	if (dip == NULL)
56 		return (DDI_EINVAL);
57 
58 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_get_supported_types: dip %p\n",
59 	    (void *)dip));
60 
61 	if (*typesp = i_ddi_intr_get_supported_types(dip))
62 		return (DDI_SUCCESS);
63 
64 	bzero(&hdl, sizeof (ddi_intr_handle_impl_t));
65 	hdl.ih_dip = dip;
66 
67 	ret = i_ddi_intr_ops(dip, dip, DDI_INTROP_SUPPORTED_TYPES, &hdl,
68 	    (void *)typesp);
69 
70 	if (ret != DDI_SUCCESS)
71 		return (DDI_INTR_NOTFOUND);
72 
73 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_get_supported_types: types %x\n",
74 	    *typesp));
75 
76 	return (ret);
77 }
78 
79 /*
80  * ddi_intr_get_nintrs:
81  * 	Return as an integer in the integer pointed to by the argument
82  * 	*nintrsp*, the number of interrupts the device supports for the
83  *	given interrupt type.
84  */
85 int
ddi_intr_get_nintrs(dev_info_t * dip,int type,int * nintrsp)86 ddi_intr_get_nintrs(dev_info_t *dip, int type, int *nintrsp)
87 {
88 	int			ret;
89 	ddi_intr_handle_impl_t	hdl;
90 
91 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_get_nintrs: dip %p, type: %d\n",
92 	    (void *)dip, type));
93 
94 	if ((dip == NULL) || (nintrsp == NULL) ||
95 	    !DDI_INTR_TYPE_FLAG_VALID(type) ||
96 	    !(i_ddi_intr_get_supported_types(dip) & type)) {
97 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_get_nintrs: "
98 		    "Invalid input args\n"));
99 		return (DDI_EINVAL);
100 	}
101 
102 	if (*nintrsp = i_ddi_intr_get_supported_nintrs(dip, type))
103 		return (DDI_SUCCESS);
104 
105 	bzero(&hdl, sizeof (ddi_intr_handle_impl_t));
106 	hdl.ih_dip = dip;
107 	hdl.ih_type = type;
108 
109 	ret = i_ddi_intr_ops(dip, dip, DDI_INTROP_NINTRS, &hdl,
110 	    (void *)nintrsp);
111 
112 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_get_nintrs:: nintrs %x\n",
113 	    *nintrsp));
114 
115 	return (ret);
116 }
117 
118 /*
119  * ddi_intr_get_navail:
120  *	Bus nexus driver will return availble interrupt count value for
121  *	a given interrupt type.
122  *
123  * 	Return as an integer in the integer pointed to by the argument
124  * 	*navailp*, the number of interrupts currently available for the
125  *	given interrupt type.
126  */
127 int
ddi_intr_get_navail(dev_info_t * dip,int type,int * navailp)128 ddi_intr_get_navail(dev_info_t *dip, int type, int *navailp)
129 {
130 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_get_navail: dip %p, type: %d\n",
131 	    (void *)dip, type));
132 
133 	if ((dip == NULL) || (navailp == NULL) ||
134 	    !DDI_INTR_TYPE_FLAG_VALID(type) ||
135 	    !(i_ddi_intr_get_supported_types(dip) & type)) {
136 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_get_navail: "
137 		    "Invalid input args\n"));
138 		return (DDI_EINVAL);
139 	}
140 
141 	if ((*navailp = i_ddi_intr_get_current_navail(dip, type)) == 0)
142 		return (DDI_INTR_NOTFOUND);
143 
144 	return (DDI_SUCCESS);
145 }
146 
147 /*
148  * Interrupt allocate/free functions
149  */
150 int
ddi_intr_alloc(dev_info_t * dip,ddi_intr_handle_t * h_array,int type,int inum,int count,int * actualp,int behavior)151 ddi_intr_alloc(dev_info_t *dip, ddi_intr_handle_t *h_array, int type, int inum,
152     int count, int *actualp, int behavior)
153 {
154 	ddi_intr_handle_impl_t	*hdlp, tmp_hdl;
155 	int			i, ret, cap = 0, curr_type, nintrs;
156 	uint_t			pri, navail, curr_nintrs = 0;
157 
158 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_alloc: name %s dip 0x%p "
159 	    "type %x inum %x count %x behavior %x\n", ddi_driver_name(dip),
160 	    (void *)dip, type, inum, count, behavior));
161 
162 	/* Validate parameters */
163 	if ((dip == NULL) || (h_array == NULL) || (inum < 0) || (count < 1) ||
164 	    (actualp == NULL) || !DDI_INTR_BEHAVIOR_FLAG_VALID(behavior)) {
165 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_alloc: "
166 		    "Invalid input args\n"));
167 		return (DDI_EINVAL);
168 	}
169 
170 	/* Validate interrupt type */
171 	if (!DDI_INTR_TYPE_FLAG_VALID(type) ||
172 	    !(i_ddi_intr_get_supported_types(dip) & type)) {
173 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_alloc: type %x not "
174 		    "supported\n", type));
175 		return (DDI_EINVAL);
176 	}
177 
178 	/* Validate inum not previously allocated */
179 	if ((type == DDI_INTR_TYPE_FIXED) &&
180 	    (i_ddi_get_intr_handle(dip, inum) != NULL)) {
181 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_alloc: inum %d is already "
182 		    "in use, cannot allocate again!!\n", inum));
183 		return (DDI_EINVAL);
184 	}
185 
186 	/* Get how many interrupts the device supports */
187 	if ((nintrs = i_ddi_intr_get_supported_nintrs(dip, type)) == 0) {
188 		if (ddi_intr_get_nintrs(dip, type, &nintrs) != DDI_SUCCESS) {
189 			DDI_INTR_APIDBG((CE_CONT, "ddi_intr_alloc: no "
190 			    "interrupts found of type %d\n", type));
191 			return (DDI_INTR_NOTFOUND);
192 		}
193 	}
194 
195 	/* Get how many interrupts the device is already using */
196 	if ((curr_type = i_ddi_intr_get_current_type(dip)) != 0) {
197 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_alloc: type %x "
198 		    "is already being used\n", curr_type));
199 		curr_nintrs = i_ddi_intr_get_current_nintrs(dip);
200 	}
201 
202 	/* Validate interrupt type consistency */
203 	if ((curr_type != 0) && (type != curr_type)) {
204 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_alloc: Requested "
205 		    "interrupt type %x is different from interrupt type %x"
206 		    "already in use\n", type, curr_type));
207 		return (DDI_EINVAL);
208 	}
209 
210 	/* Validate count does not exceed what device supports */
211 	if (count > nintrs) {
212 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_alloc: no of interrupts "
213 		    "requested %d is more than supported %d\n", count, nintrs));
214 		return (DDI_EINVAL);
215 	} else if ((count + curr_nintrs) > nintrs) {
216 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_alloc: count %d "
217 		    "+ intrs in use %d exceeds supported %d intrs\n",
218 		    count, curr_nintrs, nintrs));
219 		return (DDI_EINVAL);
220 	}
221 
222 	/* Validate power of 2 requirements for MSI */
223 	if ((type == DDI_INTR_TYPE_MSI) && !ISP2(curr_nintrs + count)) {
224 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_alloc: "
225 		    "MSI count %d is not a power of two\n", count));
226 		return (DDI_EINVAL);
227 	}
228 
229 	/*
230 	 * Initialize the device's interrupt information structure,
231 	 * and establish an association with IRM if it is supported.
232 	 *
233 	 * NOTE: IRM checks minimum support, and can return DDI_EAGAIN.
234 	 */
235 	if (curr_nintrs == 0) {
236 		i_ddi_intr_devi_init(dip);
237 		if (i_ddi_irm_insert(dip, type, count) == DDI_EAGAIN) {
238 			cmn_err(CE_WARN, "ddi_intr_alloc: "
239 			    "cannot fit into interrupt pool\n");
240 			return (DDI_EAGAIN);
241 		}
242 	}
243 
244 	/* Synchronously adjust IRM associations for non-IRM aware drivers */
245 	if (curr_nintrs && (i_ddi_irm_supported(dip, type) != DDI_SUCCESS))
246 		(void) i_ddi_irm_modify(dip, count + curr_nintrs);
247 
248 	/* Get how many interrupts are currently available */
249 	navail = i_ddi_intr_get_current_navail(dip, type);
250 
251 	/* Validate that requested number of interrupts are available */
252 	if (curr_nintrs == navail) {
253 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_alloc: max # of intrs %d "
254 		    "already allocated\n", navail));
255 		return (DDI_EAGAIN);
256 	}
257 	if ((count + curr_nintrs) > navail) {
258 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_alloc: requested # of "
259 		    "intrs %d exceeds # of available intrs %d\n", count,
260 		    navail - curr_nintrs));
261 		if (behavior == DDI_INTR_ALLOC_STRICT) {
262 			DDI_INTR_APIDBG((CE_CONT, "ddi_intr_alloc: "
263 			    "DDI_INTR_ALLOC_STRICT flag is passed, "
264 			    "return failure\n"));
265 			if (curr_nintrs == 0)
266 				i_ddi_intr_devi_fini(dip);
267 			else if (i_ddi_irm_supported(dip, type) != DDI_SUCCESS)
268 				(void) i_ddi_irm_modify(dip, curr_nintrs);
269 			return (DDI_EAGAIN);
270 		}
271 		count = navail - curr_nintrs;
272 	}
273 
274 	/* Now allocate required number of interrupts */
275 	bzero(&tmp_hdl, sizeof (ddi_intr_handle_impl_t));
276 	tmp_hdl.ih_type = type;
277 	tmp_hdl.ih_inum = inum;
278 	tmp_hdl.ih_scratch1 = count;
279 	tmp_hdl.ih_scratch2 = (void *)(uintptr_t)behavior;
280 	tmp_hdl.ih_dip = dip;
281 
282 	if (i_ddi_intr_ops(dip, dip, DDI_INTROP_ALLOC,
283 	    &tmp_hdl, (void *)actualp) != DDI_SUCCESS) {
284 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_alloc: allocation "
285 		    "failed\n"));
286 		i_ddi_intr_devi_fini(dip);
287 		return (*actualp ? DDI_EAGAIN : DDI_INTR_NOTFOUND);
288 	}
289 
290 	if ((ret = i_ddi_intr_ops(dip, dip, DDI_INTROP_GETPRI,
291 	    &tmp_hdl, (void *)&pri)) != DDI_SUCCESS) {
292 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_alloc: get priority "
293 		    "failed\n"));
294 		goto fail;
295 	}
296 
297 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_alloc: getting capability\n"));
298 
299 	if ((ret = i_ddi_intr_ops(dip, dip, DDI_INTROP_GETCAP,
300 	    &tmp_hdl, (void *)&cap)) != DDI_SUCCESS) {
301 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_alloc: get capability "
302 		    "failed\n"));
303 		goto fail;
304 	}
305 
306 	/*
307 	 * Save current interrupt type, supported and current intr count.
308 	 */
309 	i_ddi_intr_set_current_type(dip, type);
310 	i_ddi_intr_set_supported_nintrs(dip, nintrs);
311 	i_ddi_intr_set_current_nintrs(dip,
312 	    i_ddi_intr_get_current_nintrs(dip) + *actualp);
313 
314 	/* Now, go and handle each "handle" */
315 	for (i = inum; i < (inum + *actualp); i++) {
316 		hdlp = (ddi_intr_handle_impl_t *)kmem_zalloc(
317 		    (sizeof (ddi_intr_handle_impl_t)), KM_SLEEP);
318 		rw_init(&hdlp->ih_rwlock, NULL, RW_DRIVER, NULL);
319 		h_array[i] = (struct __ddi_intr_handle *)hdlp;
320 		hdlp->ih_type = type;
321 		hdlp->ih_pri = pri;
322 		hdlp->ih_cap = cap;
323 		hdlp->ih_ver = DDI_INTR_VERSION;
324 		hdlp->ih_state = DDI_IHDL_STATE_ALLOC;
325 		hdlp->ih_dip = dip;
326 		hdlp->ih_inum = i;
327 		i_ddi_alloc_intr_phdl(hdlp);
328 		if (type & DDI_INTR_TYPE_FIXED)
329 			i_ddi_set_intr_handle(dip, hdlp->ih_inum,
330 			    (ddi_intr_handle_t)hdlp);
331 
332 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_alloc: hdlp = 0x%p\n",
333 		    (void *)h_array[i]));
334 	}
335 
336 	return (DDI_SUCCESS);
337 
338 fail:
339 	(void) i_ddi_intr_ops(tmp_hdl.ih_dip, tmp_hdl.ih_dip,
340 	    DDI_INTROP_FREE, &tmp_hdl, NULL);
341 	i_ddi_intr_devi_fini(dip);
342 
343 	return (ret);
344 }
345 
346 int
ddi_intr_free(ddi_intr_handle_t h)347 ddi_intr_free(ddi_intr_handle_t h)
348 {
349 	ddi_intr_handle_impl_t	*hdlp = (ddi_intr_handle_impl_t *)h;
350 	int			ret;
351 
352 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_free: hdlp = %p\n", (void *)hdlp));
353 
354 	if (hdlp == NULL)
355 		return (DDI_EINVAL);
356 
357 	rw_enter(&hdlp->ih_rwlock, RW_WRITER);
358 	if (((hdlp->ih_flags & DDI_INTR_MSIX_DUP) &&
359 	    (hdlp->ih_state != DDI_IHDL_STATE_ADDED)) ||
360 	    ((hdlp->ih_state != DDI_IHDL_STATE_ALLOC) &&
361 	    (!(hdlp->ih_flags & DDI_INTR_MSIX_DUP)))) {
362 		rw_exit(&hdlp->ih_rwlock);
363 		return (DDI_EINVAL);
364 	}
365 
366 	/* Set the number of interrupts to free */
367 	hdlp->ih_scratch1 = 1;
368 
369 	ret = i_ddi_intr_ops(hdlp->ih_dip, hdlp->ih_dip,
370 	    DDI_INTROP_FREE, hdlp, NULL);
371 
372 	rw_exit(&hdlp->ih_rwlock);
373 	if (ret == DDI_SUCCESS) {
374 		/* This would be the dup vector */
375 		if (hdlp->ih_flags & DDI_INTR_MSIX_DUP)
376 			atomic_dec_32(&hdlp->ih_main->ih_dup_cnt);
377 		else {
378 			int	n, curr_type;
379 
380 			n = i_ddi_intr_get_current_nintrs(hdlp->ih_dip) - 1;
381 			curr_type = i_ddi_intr_get_current_type(hdlp->ih_dip);
382 
383 			i_ddi_intr_set_current_nintrs(hdlp->ih_dip, n);
384 
385 			if ((i_ddi_irm_supported(hdlp->ih_dip, curr_type)
386 			    != DDI_SUCCESS) && (n > 0))
387 				(void) i_ddi_irm_modify(hdlp->ih_dip, n);
388 
389 			if (hdlp->ih_type & DDI_INTR_TYPE_FIXED)
390 				i_ddi_set_intr_handle(hdlp->ih_dip,
391 				    hdlp->ih_inum, NULL);
392 
393 			i_ddi_intr_devi_fini(hdlp->ih_dip);
394 			i_ddi_free_intr_phdl(hdlp);
395 		}
396 		rw_destroy(&hdlp->ih_rwlock);
397 		kmem_free(hdlp, sizeof (ddi_intr_handle_impl_t));
398 	}
399 
400 	return (ret);
401 }
402 
403 /*
404  * Interrupt get/set capacity functions
405  *
406  * The logic used to figure this out is shown here:
407  *
408  *			Device level		Platform level	    Intr source
409  * 1. Fixed interrupts
410  * (non-PCI)
411  * o Flags supported	N/A			Maskable/Pending/    rootnex
412  *						No Block Enable
413  * o navail					1
414  *
415  * 2. PCI Fixed interrupts
416  * o Flags supported	pending/Maskable	Maskable/pending/    pci
417  *						No Block enable
418  * o navail		N/A			1
419  *
420  * 3. PCI MSI
421  * o Flags supported	Maskable/Pending	Maskable/Pending    pci
422  *			Block Enable		(if drvr doesn't)   Block Enable
423  * o navail		N/A			#vectors - #used    N/A
424  *
425  * 4. PCI MSI-X
426  * o Flags supported	Maskable/Pending	Maskable/Pending    pci
427  *			Block Enable				    Block Enable
428  * o navail		N/A			#vectors - #used    N/A
429  *
430  * where:
431  *	#vectors	- Total numbers of vectors available
432  *	#used		- Total numbers of vectors currently being used
433  *
434  * For devices complying to PCI2.3 or greater, see bit10 of Command Register
435  * 0 - enables assertion of INTx
436  * 1 - disables assertion of INTx
437  *
438  * For non MSI/X interrupts; if the IRQ is shared then all ddi_intr_set_*()
439  * operations return failure.
440  */
441 int
ddi_intr_get_cap(ddi_intr_handle_t h,int * flagsp)442 ddi_intr_get_cap(ddi_intr_handle_t h, int *flagsp)
443 {
444 	ddi_intr_handle_impl_t	*hdlp = (ddi_intr_handle_impl_t *)h;
445 	int			ret;
446 
447 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_get_cap: hdlp = %p\n",
448 	    (void *)hdlp));
449 
450 	*flagsp = 0;
451 	if (hdlp == NULL)
452 		return (DDI_EINVAL);
453 
454 	rw_enter(&hdlp->ih_rwlock, RW_READER);
455 
456 	if (hdlp->ih_cap) {
457 		*flagsp = hdlp->ih_cap & ~DDI_INTR_FLAG_MSI64;
458 		rw_exit(&hdlp->ih_rwlock);
459 		return (DDI_SUCCESS);
460 	}
461 
462 	ret = i_ddi_intr_ops(hdlp->ih_dip, hdlp->ih_dip,
463 	    DDI_INTROP_GETCAP, hdlp, (void *)flagsp);
464 
465 	if (ret == DDI_SUCCESS) {
466 		hdlp->ih_cap = *flagsp;
467 
468 		/* Mask out MSI/X 64-bit support to the consumer */
469 		*flagsp &= ~DDI_INTR_FLAG_MSI64;
470 	}
471 
472 	rw_exit(&hdlp->ih_rwlock);
473 	return (ret);
474 }
475 
476 int
ddi_intr_set_cap(ddi_intr_handle_t h,int flags)477 ddi_intr_set_cap(ddi_intr_handle_t h, int flags)
478 {
479 	ddi_intr_handle_impl_t	*hdlp = (ddi_intr_handle_impl_t *)h;
480 	int			ret;
481 
482 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_set_cap: hdlp = %p", (void *)hdlp));
483 
484 	if (hdlp == NULL)
485 		return (DDI_EINVAL);
486 
487 	rw_enter(&hdlp->ih_rwlock, RW_WRITER);
488 	if (hdlp->ih_state != DDI_IHDL_STATE_ALLOC) {
489 		rw_exit(&hdlp->ih_rwlock);
490 		return (DDI_EINVAL);
491 	}
492 
493 	/* Only DDI_INTR_FLAG_LEVEL or DDI_INTR_FLAG_EDGE are allowed */
494 	if (!(flags & (DDI_INTR_FLAG_EDGE | DDI_INTR_FLAG_LEVEL))) {
495 		DDI_INTR_APIDBG((CE_CONT, "%s%d: only LEVEL or EDGE capability "
496 		    "can be set\n", ddi_driver_name(hdlp->ih_dip),
497 		    ddi_get_instance(hdlp->ih_dip)));
498 		rw_exit(&hdlp->ih_rwlock);
499 		return (DDI_EINVAL);
500 	}
501 
502 	/* Both level/edge flags must be currently supported */
503 	if (!(hdlp->ih_cap & (DDI_INTR_FLAG_EDGE | DDI_INTR_FLAG_LEVEL))) {
504 		DDI_INTR_APIDBG((CE_CONT, "%s%d: Both LEVEL and EDGE capability"
505 		    " must be supported\n", ddi_driver_name(hdlp->ih_dip),
506 		    ddi_get_instance(hdlp->ih_dip)));
507 		rw_exit(&hdlp->ih_rwlock);
508 		return (DDI_ENOTSUP);
509 	}
510 
511 	ret = i_ddi_intr_ops(hdlp->ih_dip, hdlp->ih_dip,
512 	    DDI_INTROP_SETCAP, hdlp, &flags);
513 
514 	rw_exit(&hdlp->ih_rwlock);
515 	return (ret);
516 }
517 
518 /*
519  * Priority related functions
520  */
521 
522 /*
523  * ddi_intr_get_hilevel_pri:
524  *	Returns the minimum priority level for a
525  *	high-level interrupt on a platform.
526  */
527 uint_t
ddi_intr_get_hilevel_pri(void)528 ddi_intr_get_hilevel_pri(void)
529 {
530 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_get_hilevel_pri:\n"));
531 	return (LOCK_LEVEL + 1);
532 }
533 
534 int
ddi_intr_get_pri(ddi_intr_handle_t h,uint_t * prip)535 ddi_intr_get_pri(ddi_intr_handle_t h, uint_t *prip)
536 {
537 	ddi_intr_handle_impl_t	*hdlp = (ddi_intr_handle_impl_t *)h;
538 	int			ret;
539 
540 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_get_pri: hdlp = %p\n",
541 	    (void *)hdlp));
542 
543 	*prip = 0;
544 	if (hdlp == NULL)
545 		return (DDI_EINVAL);
546 
547 	rw_enter(&hdlp->ih_rwlock, RW_READER);
548 	/* Already initialized, just return that */
549 	if (hdlp->ih_pri) {
550 		*prip = hdlp->ih_pri;
551 		rw_exit(&hdlp->ih_rwlock);
552 		return (DDI_SUCCESS);
553 	}
554 
555 	ret = i_ddi_intr_ops(hdlp->ih_dip, hdlp->ih_dip,
556 	    DDI_INTROP_GETPRI, hdlp, (void *)prip);
557 
558 	if (ret == DDI_SUCCESS)
559 		hdlp->ih_pri = *prip;
560 
561 	rw_exit(&hdlp->ih_rwlock);
562 	return (ret);
563 }
564 
565 int
ddi_intr_set_pri(ddi_intr_handle_t h,uint_t pri)566 ddi_intr_set_pri(ddi_intr_handle_t h, uint_t pri)
567 {
568 	ddi_intr_handle_impl_t	*hdlp = (ddi_intr_handle_impl_t *)h;
569 	int			ret;
570 
571 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_set_pri: hdlp = %p", (void *)hdlp));
572 
573 	if (hdlp == NULL)
574 		return (DDI_EINVAL);
575 
576 	/* Validate priority argument */
577 	if (pri < DDI_INTR_PRI_MIN || pri > DDI_INTR_PRI_MAX) {
578 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_set_pri: invalid priority "
579 		    "specified  = %x\n", pri));
580 		return (DDI_EINVAL);
581 	}
582 
583 	rw_enter(&hdlp->ih_rwlock, RW_WRITER);
584 	if (hdlp->ih_state != DDI_IHDL_STATE_ALLOC) {
585 		rw_exit(&hdlp->ih_rwlock);
586 		return (DDI_EINVAL);
587 	}
588 
589 	/* If the passed priority is same as existing priority; do nothing */
590 	if (pri == hdlp->ih_pri) {
591 		rw_exit(&hdlp->ih_rwlock);
592 		return (DDI_SUCCESS);
593 	}
594 
595 	ret = i_ddi_intr_ops(hdlp->ih_dip, hdlp->ih_dip,
596 	    DDI_INTROP_SETPRI, hdlp, &pri);
597 
598 	if (ret == DDI_SUCCESS)
599 		hdlp->ih_pri = pri;
600 
601 	rw_exit(&hdlp->ih_rwlock);
602 	return (ret);
603 }
604 
605 /*
606  * Interrupt add/duplicate/remove handlers
607  */
608 int
ddi_intr_add_handler(ddi_intr_handle_t h,ddi_intr_handler_t inthandler,void * arg1,void * arg2)609 ddi_intr_add_handler(ddi_intr_handle_t h, ddi_intr_handler_t inthandler,
610     void *arg1, void *arg2)
611 {
612 	ddi_intr_handle_impl_t	*hdlp = (ddi_intr_handle_impl_t *)h;
613 	int			ret;
614 
615 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_add_handler: hdlp = 0x%p\n",
616 	    (void *)hdlp));
617 
618 	if ((hdlp == NULL) || (inthandler == NULL))
619 		return (DDI_EINVAL);
620 
621 	rw_enter(&hdlp->ih_rwlock, RW_WRITER);
622 	if (hdlp->ih_state != DDI_IHDL_STATE_ALLOC) {
623 		rw_exit(&hdlp->ih_rwlock);
624 		return (DDI_EINVAL);
625 	}
626 
627 	hdlp->ih_cb_func = inthandler;
628 	hdlp->ih_cb_arg1 = arg1;
629 	hdlp->ih_cb_arg2 = arg2;
630 
631 	ret = i_ddi_intr_ops(hdlp->ih_dip, hdlp->ih_dip,
632 	    DDI_INTROP_ADDISR, hdlp, NULL);
633 
634 	if (ret != DDI_SUCCESS) {
635 		hdlp->ih_cb_func = NULL;
636 		hdlp->ih_cb_arg1 = NULL;
637 		hdlp->ih_cb_arg2 = NULL;
638 	} else
639 		hdlp->ih_state = DDI_IHDL_STATE_ADDED;
640 
641 	rw_exit(&hdlp->ih_rwlock);
642 	return (ret);
643 }
644 
645 int
ddi_intr_dup_handler(ddi_intr_handle_t org,int dup_inum,ddi_intr_handle_t * dup)646 ddi_intr_dup_handler(ddi_intr_handle_t org, int dup_inum,
647     ddi_intr_handle_t *dup)
648 {
649 	ddi_intr_handle_impl_t	*hdlp = (ddi_intr_handle_impl_t *)org;
650 	ddi_intr_handle_impl_t	*dup_hdlp;
651 	int			ret;
652 
653 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_dup_handler: hdlp = 0x%p\n",
654 	    (void *)hdlp));
655 
656 	/* Do some input argument checking ("dup" handle is not allocated) */
657 	if ((hdlp == NULL) || (*dup != NULL) || (dup_inum < 0)) {
658 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_dup_handler: Invalid "
659 		    "input args\n"));
660 		return (DDI_EINVAL);
661 	}
662 
663 	rw_enter(&hdlp->ih_rwlock, RW_READER);
664 
665 	/* Do some input argument checking */
666 	if ((hdlp->ih_state == DDI_IHDL_STATE_ALLOC) ||	/* intr handle alloc? */
667 	    (hdlp->ih_type != DDI_INTR_TYPE_MSIX) ||	/* only MSI-X allowed */
668 	    (hdlp->ih_flags & DDI_INTR_MSIX_DUP)) {	/* only dup original */
669 		rw_exit(&hdlp->ih_rwlock);
670 		return (DDI_EINVAL);
671 	}
672 
673 	hdlp->ih_scratch1 = dup_inum;
674 	ret = i_ddi_intr_ops(hdlp->ih_dip, hdlp->ih_dip,
675 	    DDI_INTROP_DUPVEC, hdlp, NULL);
676 
677 	if (ret == DDI_SUCCESS) {
678 		dup_hdlp = (ddi_intr_handle_impl_t *)
679 		    kmem_alloc(sizeof (ddi_intr_handle_impl_t), KM_SLEEP);
680 
681 		atomic_inc_32(&hdlp->ih_dup_cnt);
682 
683 		*dup = (ddi_intr_handle_t)dup_hdlp;
684 		bcopy(hdlp, dup_hdlp, sizeof (ddi_intr_handle_impl_t));
685 
686 		/* These fields are unique to each dupped msi-x vector */
687 		rw_init(&dup_hdlp->ih_rwlock, NULL, RW_DRIVER, NULL);
688 		dup_hdlp->ih_state = DDI_IHDL_STATE_ADDED;
689 		dup_hdlp->ih_inum = dup_inum;
690 		dup_hdlp->ih_flags |= DDI_INTR_MSIX_DUP;
691 		dup_hdlp->ih_dup_cnt = 0;
692 
693 		/* Point back to original vector */
694 		dup_hdlp->ih_main = hdlp;
695 	}
696 
697 	rw_exit(&hdlp->ih_rwlock);
698 	return (ret);
699 }
700 
701 int
ddi_intr_remove_handler(ddi_intr_handle_t h)702 ddi_intr_remove_handler(ddi_intr_handle_t h)
703 {
704 	ddi_intr_handle_impl_t	*hdlp = (ddi_intr_handle_impl_t *)h;
705 	int			ret = DDI_SUCCESS;
706 
707 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_remove_handler: hdlp = %p\n",
708 	    (void *)hdlp));
709 
710 	if (hdlp == NULL)
711 		return (DDI_EINVAL);
712 
713 	rw_enter(&hdlp->ih_rwlock, RW_WRITER);
714 
715 	if (hdlp->ih_state != DDI_IHDL_STATE_ADDED) {
716 		ret = DDI_EINVAL;
717 		goto done;
718 	} else if (hdlp->ih_flags & DDI_INTR_MSIX_DUP)
719 		goto done;
720 
721 	ASSERT(hdlp->ih_dup_cnt == 0);
722 	if (hdlp->ih_dup_cnt > 0) {
723 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_remove_handler: MSI-X "
724 		    "dup_cnt %d is not 0\n", hdlp->ih_dup_cnt));
725 		ret = DDI_FAILURE;
726 		goto done;
727 	}
728 
729 	ret = i_ddi_intr_ops(hdlp->ih_dip, hdlp->ih_dip,
730 	    DDI_INTROP_REMISR, hdlp, NULL);
731 
732 	if (ret == DDI_SUCCESS) {
733 		hdlp->ih_state = DDI_IHDL_STATE_ALLOC;
734 		hdlp->ih_cb_func = NULL;
735 		hdlp->ih_cb_arg1 = NULL;
736 		hdlp->ih_cb_arg2 = NULL;
737 	}
738 
739 done:
740 	rw_exit(&hdlp->ih_rwlock);
741 	return (ret);
742 }
743 
744 
745 /*
746  * Interrupt enable/disable/block_enable/block_disable handlers
747  */
748 int
ddi_intr_enable(ddi_intr_handle_t h)749 ddi_intr_enable(ddi_intr_handle_t h)
750 {
751 	ddi_intr_handle_impl_t	*hdlp = (ddi_intr_handle_impl_t *)h;
752 	int			ret;
753 
754 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_enable: hdlp = %p\n",
755 	    (void *)hdlp));
756 
757 	if (hdlp == NULL)
758 		return (DDI_EINVAL);
759 
760 	rw_enter(&hdlp->ih_rwlock, RW_WRITER);
761 	if ((hdlp->ih_state != DDI_IHDL_STATE_ADDED) ||
762 	    ((hdlp->ih_type == DDI_INTR_TYPE_MSI) &&
763 	    (hdlp->ih_cap & DDI_INTR_FLAG_BLOCK))) {
764 		rw_exit(&hdlp->ih_rwlock);
765 		return (DDI_EINVAL);
766 	}
767 
768 	I_DDI_VERIFY_MSIX_HANDLE(hdlp);
769 
770 	ret = i_ddi_intr_ops(hdlp->ih_dip, hdlp->ih_dip,
771 	    DDI_INTROP_ENABLE, hdlp, NULL);
772 
773 	if (ret == DDI_SUCCESS) {
774 		hdlp->ih_state = DDI_IHDL_STATE_ENABLE;
775 		i_ddi_intr_set_current_nenables(hdlp->ih_dip,
776 		    i_ddi_intr_get_current_nenables(hdlp->ih_dip) + 1);
777 	}
778 
779 	rw_exit(&hdlp->ih_rwlock);
780 	return (ret);
781 }
782 
783 int
ddi_intr_disable(ddi_intr_handle_t h)784 ddi_intr_disable(ddi_intr_handle_t h)
785 {
786 	ddi_intr_handle_impl_t	*hdlp = (ddi_intr_handle_impl_t *)h;
787 	int			ret;
788 
789 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_disable: hdlp = %p\n",
790 	    (void *)hdlp));
791 
792 	if (hdlp == NULL)
793 		return (DDI_EINVAL);
794 
795 	rw_enter(&hdlp->ih_rwlock, RW_WRITER);
796 	if ((hdlp->ih_state != DDI_IHDL_STATE_ENABLE) ||
797 	    ((hdlp->ih_type == DDI_INTR_TYPE_MSI) &&
798 	    (hdlp->ih_cap & DDI_INTR_FLAG_BLOCK))) {
799 		rw_exit(&hdlp->ih_rwlock);
800 		return (DDI_EINVAL);
801 	}
802 
803 	I_DDI_VERIFY_MSIX_HANDLE(hdlp);
804 
805 	ret = i_ddi_intr_ops(hdlp->ih_dip, hdlp->ih_dip,
806 	    DDI_INTROP_DISABLE, hdlp, NULL);
807 
808 	if (ret == DDI_SUCCESS) {
809 		hdlp->ih_state = DDI_IHDL_STATE_ADDED;
810 		i_ddi_intr_set_current_nenables(hdlp->ih_dip,
811 		    i_ddi_intr_get_current_nenables(hdlp->ih_dip) - 1);
812 	}
813 
814 	rw_exit(&hdlp->ih_rwlock);
815 	return (ret);
816 }
817 
818 int
ddi_intr_block_enable(ddi_intr_handle_t * h_array,int count)819 ddi_intr_block_enable(ddi_intr_handle_t *h_array, int count)
820 {
821 	ddi_intr_handle_impl_t	*hdlp;
822 	int			i, ret;
823 
824 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_block_enable: h_array = %p\n",
825 	    (void *)h_array));
826 
827 	if (h_array == NULL)
828 		return (DDI_EINVAL);
829 
830 	for (i = 0; i < count; i++) {
831 		hdlp = (ddi_intr_handle_impl_t *)h_array[i];
832 		rw_enter(&hdlp->ih_rwlock, RW_READER);
833 
834 		if (hdlp->ih_state != DDI_IHDL_STATE_ADDED ||
835 		    hdlp->ih_type != DDI_INTR_TYPE_MSI ||
836 		    !(hdlp->ih_cap & DDI_INTR_FLAG_BLOCK)) {
837 			rw_exit(&hdlp->ih_rwlock);
838 			return (DDI_EINVAL);
839 		}
840 		rw_exit(&hdlp->ih_rwlock);
841 	}
842 
843 	hdlp = (ddi_intr_handle_impl_t *)h_array[0];
844 	rw_enter(&hdlp->ih_rwlock, RW_WRITER);
845 	hdlp->ih_scratch1 = count;
846 	hdlp->ih_scratch2 = (void *)h_array;
847 
848 	ret = i_ddi_intr_ops(hdlp->ih_dip, hdlp->ih_dip,
849 	    DDI_INTROP_BLOCKENABLE, hdlp, NULL);
850 
851 	rw_exit(&hdlp->ih_rwlock);
852 
853 	if (ret == DDI_SUCCESS) {
854 		for (i = 0; i < count; i++) {
855 			hdlp = (ddi_intr_handle_impl_t *)h_array[i];
856 			rw_enter(&hdlp->ih_rwlock, RW_WRITER);
857 			hdlp->ih_state = DDI_IHDL_STATE_ENABLE;
858 			rw_exit(&hdlp->ih_rwlock);
859 		}
860 		i_ddi_intr_set_current_nenables(hdlp->ih_dip, 1);
861 	}
862 
863 	return (ret);
864 }
865 
866 int
ddi_intr_block_disable(ddi_intr_handle_t * h_array,int count)867 ddi_intr_block_disable(ddi_intr_handle_t *h_array, int count)
868 {
869 	ddi_intr_handle_impl_t	*hdlp;
870 	int			i, ret;
871 
872 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_block_disable: h_array = %p\n",
873 	    (void *)h_array));
874 
875 	if (h_array == NULL)
876 		return (DDI_EINVAL);
877 
878 	for (i = 0; i < count; i++) {
879 		hdlp = (ddi_intr_handle_impl_t *)h_array[i];
880 		rw_enter(&hdlp->ih_rwlock, RW_READER);
881 		if (hdlp->ih_state != DDI_IHDL_STATE_ENABLE ||
882 		    hdlp->ih_type != DDI_INTR_TYPE_MSI ||
883 		    !(hdlp->ih_cap & DDI_INTR_FLAG_BLOCK)) {
884 			rw_exit(&hdlp->ih_rwlock);
885 			return (DDI_EINVAL);
886 		}
887 		rw_exit(&hdlp->ih_rwlock);
888 	}
889 
890 	hdlp = (ddi_intr_handle_impl_t *)h_array[0];
891 	rw_enter(&hdlp->ih_rwlock, RW_WRITER);
892 	hdlp->ih_scratch1 = count;
893 	hdlp->ih_scratch2 = (void *)h_array;
894 
895 	ret = i_ddi_intr_ops(hdlp->ih_dip, hdlp->ih_dip,
896 	    DDI_INTROP_BLOCKDISABLE, hdlp, NULL);
897 
898 	rw_exit(&hdlp->ih_rwlock);
899 
900 	if (ret == DDI_SUCCESS) {
901 		for (i = 0; i < count; i++) {
902 			hdlp = (ddi_intr_handle_impl_t *)h_array[i];
903 			rw_enter(&hdlp->ih_rwlock, RW_WRITER);
904 			hdlp->ih_state = DDI_IHDL_STATE_ADDED;
905 			rw_exit(&hdlp->ih_rwlock);
906 		}
907 		i_ddi_intr_set_current_nenables(hdlp->ih_dip, 0);
908 	}
909 
910 	return (ret);
911 }
912 
913 /*
914  * Interrupt set/clr mask handlers
915  */
916 int
ddi_intr_set_mask(ddi_intr_handle_t h)917 ddi_intr_set_mask(ddi_intr_handle_t h)
918 {
919 	ddi_intr_handle_impl_t	*hdlp = (ddi_intr_handle_impl_t *)h;
920 	int			ret;
921 
922 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_set_mask: hdlp = %p\n",
923 	    (void *)hdlp));
924 
925 	if (hdlp == NULL)
926 		return (DDI_EINVAL);
927 
928 	rw_enter(&hdlp->ih_rwlock, RW_WRITER);
929 	if ((hdlp->ih_state != DDI_IHDL_STATE_ENABLE) ||
930 	    (!(hdlp->ih_cap & DDI_INTR_FLAG_MASKABLE))) {
931 		rw_exit(&hdlp->ih_rwlock);
932 		return (DDI_EINVAL);
933 	}
934 
935 	ret =  i_ddi_intr_ops(hdlp->ih_dip, hdlp->ih_dip,
936 	    DDI_INTROP_SETMASK, hdlp, NULL);
937 
938 	rw_exit(&hdlp->ih_rwlock);
939 	return (ret);
940 }
941 
942 int
ddi_intr_clr_mask(ddi_intr_handle_t h)943 ddi_intr_clr_mask(ddi_intr_handle_t h)
944 {
945 	ddi_intr_handle_impl_t	*hdlp = (ddi_intr_handle_impl_t *)h;
946 	int			ret;
947 
948 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_clr_mask: hdlp = %p\n",
949 	    (void *)hdlp));
950 
951 	if (hdlp == NULL)
952 		return (DDI_EINVAL);
953 
954 	rw_enter(&hdlp->ih_rwlock, RW_WRITER);
955 	if ((hdlp->ih_state != DDI_IHDL_STATE_ENABLE) ||
956 	    (!(hdlp->ih_cap & DDI_INTR_FLAG_MASKABLE))) {
957 		rw_exit(&hdlp->ih_rwlock);
958 		return (DDI_EINVAL);
959 	}
960 
961 	ret = i_ddi_intr_ops(hdlp->ih_dip, hdlp->ih_dip,
962 	    DDI_INTROP_CLRMASK, hdlp, NULL);
963 
964 	rw_exit(&hdlp->ih_rwlock);
965 	return (ret);
966 }
967 
968 /*
969  * Interrupt get_pending handler
970  */
971 int
ddi_intr_get_pending(ddi_intr_handle_t h,int * pendingp)972 ddi_intr_get_pending(ddi_intr_handle_t h, int *pendingp)
973 {
974 	ddi_intr_handle_impl_t	*hdlp = (ddi_intr_handle_impl_t *)h;
975 	int			ret;
976 
977 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_get_pending: hdlp = %p\n",
978 	    (void *)hdlp));
979 
980 	if (hdlp == NULL)
981 		return (DDI_EINVAL);
982 
983 	rw_enter(&hdlp->ih_rwlock, RW_READER);
984 	if (!(hdlp->ih_cap & DDI_INTR_FLAG_PENDING)) {
985 		rw_exit(&hdlp->ih_rwlock);
986 		return (DDI_EINVAL);
987 	}
988 
989 	ret = i_ddi_intr_ops(hdlp->ih_dip, hdlp->ih_dip,
990 	    DDI_INTROP_GETPENDING, hdlp, (void *)pendingp);
991 
992 	rw_exit(&hdlp->ih_rwlock);
993 	return (ret);
994 }
995 
996 /*
997  * Set the number of interrupts requested from IRM
998  */
999 int
ddi_intr_set_nreq(dev_info_t * dip,int nreq)1000 ddi_intr_set_nreq(dev_info_t *dip, int nreq)
1001 {
1002 	int	curr_type, nintrs;
1003 
1004 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_set_nreq: dip %p, nreq %d\n",
1005 	    (void *)dip, nreq));
1006 
1007 	ASSERT(dip != NULL);
1008 	ASSERT(nreq > 0);
1009 
1010 	/* Sanity check inputs */
1011 	if ((dip == NULL) || (nreq < 1))
1012 		return (DDI_EINVAL);
1013 
1014 	curr_type = i_ddi_intr_get_current_type(dip);
1015 
1016 	/* Only valid for IRM drivers actively using interrupts */
1017 	if ((curr_type == 0) ||
1018 	    (i_ddi_irm_supported(dip, curr_type) != DDI_SUCCESS))
1019 		return (DDI_ENOTSUP);
1020 
1021 	/* Range check */
1022 	if (ddi_intr_get_nintrs(dip, curr_type, &nintrs) != DDI_SUCCESS)
1023 		return (DDI_FAILURE);
1024 	if (nreq > nintrs)
1025 		return (DDI_EINVAL);
1026 
1027 	return (i_ddi_irm_modify(dip, nreq));
1028 }
1029 
1030 /*
1031  * Soft interrupt handlers
1032  */
1033 /*
1034  * Add a soft interrupt and register its handler
1035  */
1036 /* ARGSUSED */
1037 int
ddi_intr_add_softint(dev_info_t * dip,ddi_softint_handle_t * h_p,int soft_pri,ddi_intr_handler_t handler,void * arg1)1038 ddi_intr_add_softint(dev_info_t *dip, ddi_softint_handle_t *h_p, int soft_pri,
1039     ddi_intr_handler_t handler, void *arg1)
1040 {
1041 	ddi_softint_hdl_impl_t	*hdlp;
1042 	int			ret;
1043 
1044 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_add_softint: dip = %p, "
1045 	    "softpri = 0x%x\n", (void *)dip, soft_pri));
1046 
1047 	if ((dip == NULL) || (h_p == NULL) || (handler == NULL)) {
1048 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_add_softint: "
1049 		    "invalid arguments"));
1050 
1051 		return (DDI_EINVAL);
1052 	}
1053 
1054 	/* Validate input arguments */
1055 	if (soft_pri < DDI_INTR_SOFTPRI_MIN ||
1056 	    soft_pri > DDI_INTR_SOFTPRI_MAX) {
1057 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_add_softint: invalid "
1058 		    "soft_pri input given  = %x\n", soft_pri));
1059 		return (DDI_EINVAL);
1060 	}
1061 
1062 	hdlp = (ddi_softint_hdl_impl_t *)kmem_zalloc(
1063 	    sizeof (ddi_softint_hdl_impl_t), KM_SLEEP);
1064 
1065 	/* fill up internally */
1066 	rw_init(&hdlp->ih_rwlock, NULL, RW_DRIVER, NULL);
1067 	rw_enter(&hdlp->ih_rwlock, RW_WRITER);
1068 	hdlp->ih_pri = soft_pri;
1069 	hdlp->ih_dip = dip;
1070 	hdlp->ih_cb_func = handler;
1071 	hdlp->ih_cb_arg1 = arg1;
1072 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_add_softint: hdlp = %p\n",
1073 	    (void *)hdlp));
1074 
1075 	/* do the platform specific calls */
1076 	if ((ret = i_ddi_add_softint(hdlp)) != DDI_SUCCESS) {
1077 		rw_exit(&hdlp->ih_rwlock);
1078 		rw_destroy(&hdlp->ih_rwlock);
1079 		kmem_free(hdlp, sizeof (ddi_softint_hdl_impl_t));
1080 		return (ret);
1081 	}
1082 
1083 	*h_p = (ddi_softint_handle_t)hdlp;
1084 	rw_exit(&hdlp->ih_rwlock);
1085 	return (ret);
1086 }
1087 
1088 /*
1089  * Remove the soft interrupt
1090  */
1091 int
ddi_intr_remove_softint(ddi_softint_handle_t h)1092 ddi_intr_remove_softint(ddi_softint_handle_t h)
1093 {
1094 	ddi_softint_hdl_impl_t	*hdlp = (ddi_softint_hdl_impl_t *)h;
1095 
1096 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_remove_softint: hdlp = %p\n",
1097 	    (void *)hdlp));
1098 
1099 	if (hdlp == NULL)
1100 		return (DDI_EINVAL);
1101 
1102 	rw_enter(&hdlp->ih_rwlock, RW_WRITER);
1103 	i_ddi_remove_softint(hdlp);
1104 	rw_exit(&hdlp->ih_rwlock);
1105 	rw_destroy(&hdlp->ih_rwlock);
1106 
1107 	/* kmem_free the hdl impl_t structure allocated earlier */
1108 	kmem_free(hdlp, sizeof (ddi_softint_hdl_impl_t));
1109 	return (DDI_SUCCESS);
1110 }
1111 
1112 /*
1113  * Trigger a soft interrupt
1114  */
1115 int
ddi_intr_trigger_softint(ddi_softint_handle_t h,void * arg2)1116 ddi_intr_trigger_softint(ddi_softint_handle_t h, void *arg2)
1117 {
1118 	ddi_softint_hdl_impl_t	*hdlp = (ddi_softint_hdl_impl_t *)h;
1119 	int			ret;
1120 
1121 	if (hdlp == NULL)
1122 		return (DDI_EINVAL);
1123 
1124 	if ((ret = i_ddi_trigger_softint(hdlp, arg2)) != DDI_SUCCESS) {
1125 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_trigger_softint: failed, "
1126 		    " ret 0%x\n", ret));
1127 
1128 		return (ret);
1129 	}
1130 
1131 	hdlp->ih_cb_arg2 = arg2;
1132 	return (DDI_SUCCESS);
1133 }
1134 
1135 /*
1136  * Get the soft interrupt priority
1137  */
1138 int
ddi_intr_get_softint_pri(ddi_softint_handle_t h,uint_t * soft_prip)1139 ddi_intr_get_softint_pri(ddi_softint_handle_t h, uint_t *soft_prip)
1140 {
1141 	ddi_softint_hdl_impl_t	*hdlp = (ddi_softint_hdl_impl_t *)h;
1142 
1143 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_get_softint_pri: h = %p\n",
1144 	    (void *)h));
1145 
1146 	if (hdlp == NULL)
1147 		return (DDI_EINVAL);
1148 
1149 	rw_enter(&hdlp->ih_rwlock, RW_READER);
1150 	*soft_prip = hdlp->ih_pri;
1151 	rw_exit(&hdlp->ih_rwlock);
1152 	return (DDI_SUCCESS);
1153 }
1154 
1155 /*
1156  * Set the soft interrupt priority
1157  */
1158 int
ddi_intr_set_softint_pri(ddi_softint_handle_t h,uint_t soft_pri)1159 ddi_intr_set_softint_pri(ddi_softint_handle_t h, uint_t soft_pri)
1160 {
1161 	ddi_softint_hdl_impl_t	*hdlp = (ddi_softint_hdl_impl_t *)h;
1162 	int			ret;
1163 	uint_t			orig_soft_pri;
1164 
1165 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_set_softint_pri: h = %p\n",
1166 	    (void *)h));
1167 
1168 	if (hdlp == NULL)
1169 		return (DDI_EINVAL);
1170 
1171 	/* Validate priority argument */
1172 	if (soft_pri < DDI_INTR_SOFTPRI_MIN ||
1173 	    soft_pri > DDI_INTR_SOFTPRI_MAX) {
1174 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_set_softint_pri: invalid "
1175 		    "soft_pri input given  = %x\n", soft_pri));
1176 		return (DDI_EINVAL);
1177 	}
1178 
1179 	rw_enter(&hdlp->ih_rwlock, RW_WRITER);
1180 	orig_soft_pri = hdlp->ih_pri;
1181 	hdlp->ih_pri = soft_pri;
1182 
1183 	if ((ret = i_ddi_set_softint_pri(hdlp, orig_soft_pri)) != DDI_SUCCESS) {
1184 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_set_softint_pri: failed, "
1185 		    " ret 0%x\n", ret));
1186 		hdlp->ih_pri = orig_soft_pri;
1187 	}
1188 
1189 	rw_exit(&hdlp->ih_rwlock);
1190 	return (ret);
1191 }
1192 
1193 /*
1194  * Old DDI interrupt framework
1195  *
1196  * The following DDI interrupt interfaces are obsolete.
1197  * Use the above new DDI interrupt interfaces instead.
1198  */
1199 
1200 int
ddi_intr_hilevel(dev_info_t * dip,uint_t inumber)1201 ddi_intr_hilevel(dev_info_t *dip, uint_t inumber)
1202 {
1203 	ddi_intr_handle_t	hdl;
1204 	ddi_intr_handle_t	*hdl_p;
1205 	size_t			hdl_sz = 0;
1206 	int			actual, ret;
1207 	uint_t			high_pri, pri;
1208 
1209 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_hilevel: name=%s%d dip=0x%p "
1210 	    "inum=0x%x\n", ddi_driver_name(dip), ddi_get_instance(dip),
1211 	    (void *)dip, inumber));
1212 
1213 	/*
1214 	 * The device driver may have already registed with the
1215 	 * framework. If so, first try to get the existing interrupt handle
1216 	 * for that given inumber and use that handle.
1217 	 */
1218 	if ((hdl = i_ddi_get_intr_handle(dip, inumber)) == NULL) {
1219 		hdl_sz = sizeof (ddi_intr_handle_t) * (inumber + 1);
1220 		hdl_p = kmem_zalloc(hdl_sz, KM_SLEEP);
1221 		if ((ret = ddi_intr_alloc(dip, hdl_p, DDI_INTR_TYPE_FIXED,
1222 		    inumber, 1, &actual,
1223 		    DDI_INTR_ALLOC_NORMAL)) != DDI_SUCCESS) {
1224 			DDI_INTR_APIDBG((CE_CONT, "ddi_intr_hilevel: "
1225 			    "ddi_intr_alloc failed, ret 0x%x\n", ret));
1226 			kmem_free(hdl_p, hdl_sz);
1227 			return (0);
1228 		}
1229 		hdl = hdl_p[inumber];
1230 	}
1231 
1232 	if ((ret = ddi_intr_get_pri(hdl, &pri)) != DDI_SUCCESS) {
1233 		DDI_INTR_APIDBG((CE_CONT, "ddi_intr_hilevel: "
1234 		    "ddi_intr_get_pri failed, ret 0x%x\n", ret));
1235 		(void) ddi_intr_free(hdl);
1236 		if (hdl_sz)
1237 			kmem_free(hdl_p, hdl_sz);
1238 		return (0);
1239 	}
1240 
1241 	high_pri = ddi_intr_get_hilevel_pri();
1242 
1243 	DDI_INTR_APIDBG((CE_CONT, "ddi_intr_hilevel: pri = %x, "
1244 	    "high_pri = %x\n", pri, high_pri));
1245 
1246 	/* Free the handle allocated here only if no existing handle exists */
1247 	if (hdl_sz) {
1248 		(void) ddi_intr_free(hdl);
1249 		kmem_free(hdl_p, hdl_sz);
1250 	}
1251 
1252 	return (pri >= high_pri);
1253 }
1254 
1255 int
ddi_dev_nintrs(dev_info_t * dip,int * result)1256 ddi_dev_nintrs(dev_info_t *dip, int *result)
1257 {
1258 	DDI_INTR_APIDBG((CE_CONT, "ddi_dev_nintrs: name=%s%d dip=0x%p\n",
1259 	    ddi_driver_name(dip), ddi_get_instance(dip), (void *)dip));
1260 
1261 	if (ddi_intr_get_nintrs(dip, DDI_INTR_TYPE_FIXED,
1262 	    result) != DDI_SUCCESS) {
1263 		DDI_INTR_APIDBG((CE_CONT, "ddi_dev_nintrs: "
1264 		    "ddi_intr_get_nintrs failed\n"));
1265 		*result = 0;
1266 	}
1267 
1268 	return (DDI_SUCCESS);
1269 }
1270 
1271 int
ddi_get_iblock_cookie(dev_info_t * dip,uint_t inumber,ddi_iblock_cookie_t * iblock_cookiep)1272 ddi_get_iblock_cookie(dev_info_t *dip, uint_t inumber,
1273     ddi_iblock_cookie_t *iblock_cookiep)
1274 {
1275 	ddi_intr_handle_t	hdl;
1276 	ddi_intr_handle_t	*hdl_p;
1277 	size_t			hdl_sz = 0;
1278 	int			actual, ret;
1279 	uint_t			pri;
1280 
1281 	DDI_INTR_APIDBG((CE_CONT, "ddi_get_iblock_cookie: name=%s%d dip=0x%p "
1282 	    "inum=0x%x\n", ddi_driver_name(dip), ddi_get_instance(dip),
1283 	    (void *)dip, inumber));
1284 
1285 	ASSERT(iblock_cookiep != NULL);
1286 
1287 	/*
1288 	 * The device driver may have already registed with the
1289 	 * framework. If so, first try to get the existing interrupt handle
1290 	 * for that given inumber and use that handle.
1291 	 */
1292 	if ((hdl = i_ddi_get_intr_handle(dip, inumber)) == NULL) {
1293 		hdl_sz = sizeof (ddi_intr_handle_t) * (inumber + 1);
1294 		hdl_p = kmem_zalloc(hdl_sz, KM_SLEEP);
1295 		if ((ret = ddi_intr_alloc(dip, hdl_p,
1296 		    DDI_INTR_TYPE_FIXED, inumber, 1, &actual,
1297 		    DDI_INTR_ALLOC_NORMAL)) != DDI_SUCCESS) {
1298 			DDI_INTR_APIDBG((CE_CONT, "ddi_get_iblock_cookie: "
1299 			    "ddi_intr_alloc failed, ret 0x%x\n", ret));
1300 			kmem_free(hdl_p, hdl_sz);
1301 			return (DDI_INTR_NOTFOUND);
1302 		}
1303 		hdl = hdl_p[inumber];
1304 	}
1305 
1306 	if ((ret = ddi_intr_get_pri(hdl, &pri)) != DDI_SUCCESS) {
1307 		DDI_INTR_APIDBG((CE_CONT, "ddi_get_iblock_cookie: "
1308 		    "ddi_intr_get_pri failed, ret 0x%x\n", ret));
1309 		(void) ddi_intr_free(hdl);
1310 		if (hdl_sz)
1311 			kmem_free(hdl_p, hdl_sz);
1312 		return (DDI_FAILURE);
1313 	}
1314 
1315 	*iblock_cookiep = (ddi_iblock_cookie_t)(uintptr_t)pri;
1316 	/* Free the handle allocated here only if no existing handle exists */
1317 	if (hdl_sz) {
1318 		(void) ddi_intr_free(hdl);
1319 		kmem_free(hdl_p, hdl_sz);
1320 	}
1321 
1322 	return (DDI_SUCCESS);
1323 }
1324 
1325 int
ddi_add_intr(dev_info_t * dip,uint_t inumber,ddi_iblock_cookie_t * iblock_cookiep,ddi_idevice_cookie_t * idevice_cookiep,uint_t (* int_handler)(caddr_t int_handler_arg),caddr_t int_handler_arg)1326 ddi_add_intr(dev_info_t *dip, uint_t inumber,
1327     ddi_iblock_cookie_t *iblock_cookiep,
1328     ddi_idevice_cookie_t *idevice_cookiep,
1329     uint_t (*int_handler)(caddr_t int_handler_arg),
1330     caddr_t int_handler_arg)
1331 {
1332 	ddi_intr_handle_t	*hdl_p;
1333 	size_t			hdl_sz;
1334 	int			actual, ret;
1335 	uint_t			pri;
1336 	ddi_intr_handler_t	*handler;
1337 
1338 	handler = (ddi_intr_handler_t *)(uintptr_t)int_handler;
1339 	DDI_INTR_APIDBG((CE_CONT, "ddi_add_intr: name=%s%d dip=0x%p "
1340 	    "inum=0x%x\n", ddi_driver_name(dip), ddi_get_instance(dip),
1341 	    (void *)dip, inumber));
1342 
1343 	hdl_sz = sizeof (ddi_intr_handle_t) * (inumber + 1);
1344 	hdl_p = kmem_zalloc(hdl_sz, KM_SLEEP);
1345 
1346 	if ((ret = ddi_intr_alloc(dip, hdl_p, DDI_INTR_TYPE_FIXED,
1347 	    inumber, 1, &actual, DDI_INTR_ALLOC_NORMAL)) != DDI_SUCCESS) {
1348 		DDI_INTR_APIDBG((CE_CONT, "ddi_add_intr: "
1349 		    "ddi_intr_alloc failed, ret 0x%x\n", ret));
1350 		kmem_free(hdl_p, hdl_sz);
1351 		return (DDI_INTR_NOTFOUND);
1352 	}
1353 
1354 	if ((ret = ddi_intr_get_pri(hdl_p[inumber], &pri)) != DDI_SUCCESS)  {
1355 		DDI_INTR_APIDBG((CE_CONT, "ddi_add_intr: "
1356 		    "ddi_intr_get_pri failed, ret 0x%x\n", ret));
1357 		(void) ddi_intr_free(hdl_p[inumber]);
1358 		kmem_free(hdl_p, hdl_sz);
1359 		return (DDI_FAILURE);
1360 	}
1361 
1362 	if ((ret = ddi_intr_add_handler(hdl_p[inumber], handler,
1363 	    int_handler_arg, NULL)) != DDI_SUCCESS) {
1364 		DDI_INTR_APIDBG((CE_CONT, "ddi_add_intr: "
1365 		    "ddi_intr_add_handler failed, ret 0x%x\n", ret));
1366 		(void) ddi_intr_free(hdl_p[inumber]);
1367 		kmem_free(hdl_p, hdl_sz);
1368 		return (DDI_FAILURE);
1369 	}
1370 
1371 	if ((ret = ddi_intr_enable(hdl_p[inumber])) != DDI_SUCCESS) {
1372 		DDI_INTR_APIDBG((CE_CONT, "ddi_add_intr: "
1373 		    "ddi_intr_enable failed, ret 0x%x\n", ret));
1374 		(void) ddi_intr_remove_handler(hdl_p[inumber]);
1375 		(void) ddi_intr_free(hdl_p[inumber]);
1376 		kmem_free(hdl_p, hdl_sz);
1377 		return (DDI_FAILURE);
1378 	}
1379 
1380 	if (iblock_cookiep)
1381 		*iblock_cookiep = (ddi_iblock_cookie_t)(uintptr_t)pri;
1382 
1383 	if (idevice_cookiep) {
1384 		idevice_cookiep->idev_vector = 0;
1385 		idevice_cookiep->idev_priority = pri;
1386 	}
1387 
1388 	kmem_free(hdl_p, hdl_sz);
1389 
1390 	return (DDI_SUCCESS);
1391 }
1392 
1393 /* ARGSUSED */
1394 int
ddi_add_fastintr(dev_info_t * dip,uint_t inumber,ddi_iblock_cookie_t * iblock_cookiep,ddi_idevice_cookie_t * idevice_cookiep,uint_t (* hi_int_handler)(void))1395 ddi_add_fastintr(dev_info_t *dip, uint_t inumber,
1396     ddi_iblock_cookie_t *iblock_cookiep,
1397     ddi_idevice_cookie_t *idevice_cookiep,
1398     uint_t (*hi_int_handler)(void))
1399 {
1400 	DDI_INTR_APIDBG((CE_CONT, "ddi_add_fastintr: name=%s%d dip=0x%p "
1401 	    "inum=0x%x: Not supported, return failure\n", ddi_driver_name(dip),
1402 	    ddi_get_instance(dip), (void *)dip, inumber));
1403 
1404 	return (DDI_FAILURE);
1405 }
1406 
1407 /* ARGSUSED */
1408 void
ddi_remove_intr(dev_info_t * dip,uint_t inum,ddi_iblock_cookie_t iblock_cookie)1409 ddi_remove_intr(dev_info_t *dip, uint_t inum, ddi_iblock_cookie_t iblock_cookie)
1410 {
1411 	ddi_intr_handle_t	hdl;
1412 	int			ret;
1413 
1414 	DDI_INTR_APIDBG((CE_CONT, "ddi_remove_intr: name=%s%d dip=0x%p "
1415 	    "inum=0x%x\n", ddi_driver_name(dip), ddi_get_instance(dip),
1416 	    (void *)dip, inum));
1417 
1418 	if ((hdl = i_ddi_get_intr_handle(dip, inum)) == NULL) {
1419 		DDI_INTR_APIDBG((CE_CONT, "ddi_remove_intr: no handle "
1420 		    "found\n"));
1421 		return;
1422 	}
1423 
1424 	if ((ret = ddi_intr_disable(hdl)) != DDI_SUCCESS) {
1425 		DDI_INTR_APIDBG((CE_CONT, "ddi_remove_intr: "
1426 		    "ddi_intr_disable failed, ret 0x%x\n", ret));
1427 		return;
1428 	}
1429 
1430 	if ((ret = ddi_intr_remove_handler(hdl)) != DDI_SUCCESS) {
1431 		DDI_INTR_APIDBG((CE_CONT, "ddi_remove_intr: "
1432 		    "ddi_intr_remove_handler failed, ret 0x%x\n", ret));
1433 		return;
1434 	}
1435 
1436 	if ((ret = ddi_intr_free(hdl)) != DDI_SUCCESS) {
1437 		DDI_INTR_APIDBG((CE_CONT, "ddi_remove_intr: "
1438 		    "ddi_intr_free failed, ret 0x%x\n", ret));
1439 		return;
1440 	}
1441 }
1442 
1443 /* ARGSUSED */
1444 int
ddi_get_soft_iblock_cookie(dev_info_t * dip,int preference,ddi_iblock_cookie_t * iblock_cookiep)1445 ddi_get_soft_iblock_cookie(dev_info_t *dip, int preference,
1446     ddi_iblock_cookie_t *iblock_cookiep)
1447 {
1448 	DDI_INTR_APIDBG((CE_CONT, "ddi_get_soft_iblock_cookie: name=%s%d "
1449 	    "dip=0x%p pref=0x%x\n", ddi_driver_name(dip), ddi_get_instance(dip),
1450 	    (void *)dip, preference));
1451 
1452 	ASSERT(iblock_cookiep != NULL);
1453 
1454 	if (preference == DDI_SOFTINT_FIXED)
1455 		return (DDI_FAILURE);
1456 
1457 	*iblock_cookiep = (ddi_iblock_cookie_t)((uintptr_t)
1458 	    ((preference > DDI_SOFTINT_MED) ? DDI_SOFT_INTR_PRI_H :
1459 	    DDI_SOFT_INTR_PRI_M));
1460 
1461 	return (DDI_SUCCESS);
1462 }
1463 
1464 int
ddi_add_softintr(dev_info_t * dip,int preference,ddi_softintr_t * idp,ddi_iblock_cookie_t * iblock_cookiep,ddi_idevice_cookie_t * idevice_cookiep,uint_t (* int_handler)(caddr_t int_handler_arg),caddr_t int_handler_arg)1465 ddi_add_softintr(dev_info_t *dip, int preference, ddi_softintr_t *idp,
1466     ddi_iblock_cookie_t *iblock_cookiep,
1467     ddi_idevice_cookie_t *idevice_cookiep,
1468     uint_t (*int_handler)(caddr_t int_handler_arg),
1469     caddr_t int_handler_arg)
1470 {
1471 	ddi_softint_handle_t	*hdl_p;
1472 	uint64_t		softpri;
1473 	int			ret;
1474 	ddi_intr_handler_t	*handler;
1475 
1476 	handler = (ddi_intr_handler_t *)(uintptr_t)int_handler;
1477 	DDI_INTR_APIDBG((CE_CONT, "ddi_add_softintr: name=%s%d dip=0x%p "
1478 	    "pref=0x%x\n", ddi_driver_name(dip), ddi_get_instance(dip),
1479 	    (void *)dip, preference));
1480 
1481 	if ((idp == NULL) || ((preference == DDI_SOFTINT_FIXED) &&
1482 	    (iblock_cookiep == NULL)))
1483 		return (DDI_FAILURE);
1484 
1485 	/* Translate the priority preference */
1486 	if (preference == DDI_SOFTINT_FIXED) {
1487 		softpri = (uint64_t)(uintptr_t)*iblock_cookiep;
1488 		softpri = MIN(softpri, DDI_SOFT_INTR_PRI_H);
1489 	} else {
1490 		softpri = (uint64_t)((preference > DDI_SOFTINT_MED) ?
1491 		    DDI_SOFT_INTR_PRI_H : DDI_SOFT_INTR_PRI_M);
1492 	}
1493 
1494 	DDI_INTR_APIDBG((CE_CONT, "ddi_add_softintr: preference 0x%x "
1495 	    "softpri 0x%lx\n", preference, (long)softpri));
1496 
1497 	hdl_p = kmem_zalloc(sizeof (ddi_softint_handle_t), KM_SLEEP);
1498 	if ((ret = ddi_intr_add_softint(dip, hdl_p, softpri,
1499 	    handler, int_handler_arg)) != DDI_SUCCESS) {
1500 		DDI_INTR_APIDBG((CE_CONT, "ddi_add_softintr: "
1501 		    "ddi_intr_add_softint failed, ret 0x%x\n", ret));
1502 
1503 		kmem_free(hdl_p, sizeof (ddi_softint_handle_t));
1504 		return (DDI_FAILURE);
1505 	}
1506 
1507 	if (iblock_cookiep)
1508 		*iblock_cookiep =  (ddi_iblock_cookie_t)(uintptr_t)softpri;
1509 
1510 	if (idevice_cookiep) {
1511 		idevice_cookiep->idev_vector = 0;
1512 		idevice_cookiep->idev_priority = softpri;
1513 	}
1514 
1515 	*idp = (ddi_softintr_t)hdl_p;
1516 
1517 	DDI_INTR_APIDBG((CE_CONT, "ddi_add_softintr: dip = 0x%p, "
1518 	    "idp = 0x%p, ret = %x\n", (void *)dip, (void *)*idp, ret));
1519 
1520 	return (DDI_SUCCESS);
1521 }
1522 
1523 void
ddi_remove_softintr(ddi_softintr_t id)1524 ddi_remove_softintr(ddi_softintr_t id)
1525 {
1526 	ddi_softint_handle_t	*h_p = (ddi_softint_handle_t *)id;
1527 
1528 	DDI_INTR_APIDBG((CE_CONT, "ddi_remove_softintr: id=0x%p\n",
1529 	    (void *)id));
1530 
1531 	if (h_p == NULL)
1532 		return;
1533 
1534 	DDI_INTR_APIDBG((CE_CONT, "ddi_remove_softintr: handle 0x%p\n",
1535 	    (void *)h_p));
1536 
1537 	(void) ddi_intr_remove_softint(*h_p);
1538 	kmem_free(h_p, sizeof (ddi_softint_handle_t));
1539 }
1540 
1541 void
ddi_trigger_softintr(ddi_softintr_t id)1542 ddi_trigger_softintr(ddi_softintr_t id)
1543 {
1544 	ddi_softint_handle_t	*h_p = (ddi_softint_handle_t *)id;
1545 	int			ret;
1546 
1547 	if (h_p == NULL)
1548 		return;
1549 
1550 	if ((ret = ddi_intr_trigger_softint(*h_p, NULL)) != DDI_SUCCESS) {
1551 		DDI_INTR_APIDBG((CE_CONT, "ddi_trigger_softintr: "
1552 		    "ddi_intr_trigger_softint failed, hdlp 0x%p "
1553 		    "ret 0x%x\n", (void *)h_p, ret));
1554 	}
1555 }
1556