1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright 2013 Pluribus Networks, Inc.
24  * Copyright 2017 Joyent, Inc.
25  */
26 
27 /*
28  * apic_introp.c:
29  *	Has code for Advanced DDI interrupt framework support.
30  */
31 
32 #include <sys/cpuvar.h>
33 #include <sys/psm.h>
34 #include <sys/archsystm.h>
35 #include <sys/apic.h>
36 #include <sys/sunddi.h>
37 #include <sys/ddi_impldefs.h>
38 #include <sys/mach_intr.h>
39 #include <sys/sysmacros.h>
40 #include <sys/trap.h>
41 #include <sys/pci.h>
42 #include <sys/pci_intr_lib.h>
43 #include <sys/apic_common.h>
44 
45 #define	UCHAR_MAX	UINT8_MAX
46 
47 extern struct av_head autovect[];
48 
49 /*
50  *	Local Function Prototypes
51  */
52 apic_irq_t	*apic_find_irq(dev_info_t *, struct intrspec *, int);
53 
54 /*
55  * apic_pci_msi_enable_vector:
56  *	Set the address/data fields in the MSI/X capability structure
57  *	XXX: MSI-X support
58  */
59 /* ARGSUSED */
60 void
apic_pci_msi_enable_vector(apic_irq_t * irq_ptr,int type,int inum,int vector,int count,int target_apic_id)61 apic_pci_msi_enable_vector(apic_irq_t *irq_ptr, int type, int inum, int vector,
62     int count, int target_apic_id)
63 {
64 	uint64_t		msi_addr, msi_data;
65 	ushort_t		msi_ctrl;
66 	dev_info_t		*dip = irq_ptr->airq_dip;
67 	int			cap_ptr = i_ddi_get_msi_msix_cap_ptr(dip);
68 	ddi_acc_handle_t	handle = i_ddi_get_pci_config_handle(dip);
69 	msi_regs_t		msi_regs;
70 	int			irqno, i;
71 	void			*intrmap_tbl[PCI_MSI_MAX_INTRS];
72 
73 	DDI_INTR_IMPLDBG((CE_CONT, "apic_pci_msi_enable_vector: dip=0x%p\n"
74 	    "\tdriver = %s, inum=0x%x vector=0x%x apicid=0x%x\n", (void *)dip,
75 	    ddi_driver_name(dip), inum, vector, target_apic_id));
76 
77 	ASSERT((handle != NULL) && (cap_ptr != 0));
78 
79 	msi_regs.mr_data = vector;
80 	msi_regs.mr_addr = target_apic_id;
81 
82 	for (i = 0; i < count; i++) {
83 		irqno = apic_vector_to_irq[vector + i];
84 		intrmap_tbl[i] = apic_irq_table[irqno]->airq_intrmap_private;
85 	}
86 	apic_vt_ops->apic_intrmap_alloc_entry(intrmap_tbl, dip, type,
87 	    count, 0xff);
88 	for (i = 0; i < count; i++) {
89 		irqno = apic_vector_to_irq[vector + i];
90 		apic_irq_table[irqno]->airq_intrmap_private =
91 		    intrmap_tbl[i];
92 	}
93 
94 	apic_vt_ops->apic_intrmap_map_entry(irq_ptr->airq_intrmap_private,
95 	    (void *)&msi_regs, type, count);
96 	apic_vt_ops->apic_intrmap_record_msi(irq_ptr->airq_intrmap_private,
97 	    &msi_regs);
98 
99 	/* MSI Address */
100 	msi_addr = msi_regs.mr_addr;
101 
102 	/* MSI Data: MSI is edge triggered according to spec */
103 	msi_data = msi_regs.mr_data;
104 
105 	DDI_INTR_IMPLDBG((CE_CONT, "apic_pci_msi_enable_vector: addr=0x%lx "
106 	    "data=0x%lx\n", (long)msi_addr, (long)msi_data));
107 
108 	if (type == DDI_INTR_TYPE_MSI) {
109 		msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL);
110 
111 		/* Set the bits to inform how many MSIs are enabled */
112 		msi_ctrl |= ((highbit(count) -1) << PCI_MSI_MME_SHIFT);
113 		pci_config_put16(handle, cap_ptr + PCI_MSI_CTRL, msi_ctrl);
114 
115 		/*
116 		 * Only set vector if not on hypervisor
117 		 */
118 		pci_config_put32(handle,
119 		    cap_ptr + PCI_MSI_ADDR_OFFSET, msi_addr);
120 
121 		if (msi_ctrl &  PCI_MSI_64BIT_MASK) {
122 			pci_config_put32(handle,
123 			    cap_ptr + PCI_MSI_ADDR_OFFSET + 4, msi_addr >> 32);
124 			pci_config_put16(handle,
125 			    cap_ptr + PCI_MSI_64BIT_DATA, msi_data);
126 		} else {
127 			pci_config_put16(handle,
128 			    cap_ptr + PCI_MSI_32BIT_DATA, msi_data);
129 		}
130 
131 	} else if (type == DDI_INTR_TYPE_MSIX) {
132 		uintptr_t	off;
133 		ddi_intr_msix_t	*msix_p = i_ddi_get_msix(dip);
134 
135 		ASSERT(msix_p != NULL);
136 
137 		/* Offset into the "inum"th entry in the MSI-X table */
138 		off = (uintptr_t)msix_p->msix_tbl_addr +
139 		    (inum  * PCI_MSIX_VECTOR_SIZE);
140 
141 		ddi_put32(msix_p->msix_tbl_hdl,
142 		    (uint32_t *)(off + PCI_MSIX_DATA_OFFSET), msi_data);
143 		ddi_put32(msix_p->msix_tbl_hdl,
144 		    (uint32_t *)(off + PCI_MSIX_LOWER_ADDR_OFFSET), msi_addr);
145 		ddi_put32(msix_p->msix_tbl_hdl,
146 		    (uint32_t *)(off + PCI_MSIX_UPPER_ADDR_OFFSET),
147 		    msi_addr >> 32);
148 	}
149 }
150 
151 /*
152  * This function returns the no. of vectors available for the pri.
153  * dip is not used at this moment.  If we really don't need that,
154  * it will be removed.
155  */
156 /*ARGSUSED*/
157 int
apic_navail_vector(dev_info_t * dip,int pri)158 apic_navail_vector(dev_info_t *dip, int pri)
159 {
160 	int	lowest, highest, i, navail, count;
161 
162 	DDI_INTR_IMPLDBG((CE_CONT, "apic_navail_vector: dip: %p, pri: %x\n",
163 	    (void *)dip, pri));
164 
165 	highest = apic_ipltopri[pri] + APIC_VECTOR_MASK;
166 	lowest = apic_ipltopri[pri - 1] + APIC_VECTOR_PER_IPL;
167 	navail = count = 0;
168 
169 	if (highest < lowest) /* Both ipl and ipl - 1 map to same pri */
170 		lowest -= APIC_VECTOR_PER_IPL;
171 
172 	/* It has to be contiguous */
173 	for (i = lowest; i <= highest; i++) {
174 		count = 0;
175 		while ((apic_vector_to_irq[i] == APIC_RESV_IRQ) &&
176 		    (i <= highest)) {
177 			if (APIC_CHECK_RESERVE_VECTORS(i))
178 				break;
179 			count++;
180 			i++;
181 		}
182 		if (count > navail)
183 			navail = count;
184 	}
185 	return (navail);
186 }
187 
188 /*
189  * Finds "count" contiguous MSI vectors starting at the proper alignment
190  * at "pri".
191  * Caller needs to make sure that count has to be power of 2 and should not
192  * be < 1.
193  */
194 uchar_t
apic_find_multi_vectors(int pri,int count)195 apic_find_multi_vectors(int pri, int count)
196 {
197 	int	lowest, highest, i, navail, start, msibits;
198 
199 	DDI_INTR_IMPLDBG((CE_CONT, "apic_find_mult: pri: %x, count: %x\n",
200 	    pri, count));
201 
202 	highest = apic_ipltopri[pri] + APIC_VECTOR_MASK;
203 	lowest = apic_ipltopri[pri - 1] + APIC_VECTOR_PER_IPL;
204 	navail = 0;
205 
206 	if (highest < lowest) /* Both ipl and ipl - 1 map to same pri */
207 		lowest -= APIC_VECTOR_PER_IPL;
208 
209 	/*
210 	 * msibits is the no. of lower order message data bits for the
211 	 * allocated MSI vectors and is used to calculate the aligned
212 	 * starting vector
213 	 */
214 	msibits = count - 1;
215 
216 	/* It has to be contiguous */
217 	for (i = lowest; i <= highest; i++) {
218 		navail = 0;
219 
220 		/*
221 		 * starting vector has to be aligned accordingly for
222 		 * multiple MSIs
223 		 */
224 		if (msibits)
225 			i = (i + msibits) & ~msibits;
226 		start = i;
227 		while ((apic_vector_to_irq[i] == APIC_RESV_IRQ) &&
228 		    (i <= highest)) {
229 			if (APIC_CHECK_RESERVE_VECTORS(i))
230 				break;
231 			navail++;
232 			if (navail >= count) {
233 				ASSERT(start >= 0 && start <= UCHAR_MAX);
234 				return ((uchar_t)start);
235 			}
236 			i++;
237 		}
238 	}
239 	return (0);
240 }
241 
242 
243 /*
244  * It finds the apic_irq_t associates with the dip, ispec and type.
245  */
246 apic_irq_t *
apic_find_irq(dev_info_t * dip,struct intrspec * ispec,int type)247 apic_find_irq(dev_info_t *dip, struct intrspec *ispec, int type)
248 {
249 	apic_irq_t	*irqp;
250 	int i;
251 
252 	DDI_INTR_IMPLDBG((CE_CONT, "apic_find_irq: dip=0x%p vec=0x%x "
253 	    "ipl=0x%x type=0x%x\n", (void *)dip, ispec->intrspec_vec,
254 	    ispec->intrspec_pri, type));
255 
256 	for (i = apic_min_device_irq; i <= apic_max_device_irq; i++) {
257 		for (irqp = apic_irq_table[i]; irqp; irqp = irqp->airq_next) {
258 			if ((irqp->airq_dip == dip) &&
259 			    (irqp->airq_origirq == ispec->intrspec_vec) &&
260 			    (irqp->airq_ipl == ispec->intrspec_pri)) {
261 				if (type == DDI_INTR_TYPE_MSI) {
262 					if (irqp->airq_mps_intr_index ==
263 					    MSI_INDEX)
264 						return (irqp);
265 				} else if (type == DDI_INTR_TYPE_MSIX) {
266 					if (irqp->airq_mps_intr_index ==
267 					    MSIX_INDEX)
268 						return (irqp);
269 				} else
270 					return (irqp);
271 			}
272 		}
273 	}
274 	DDI_INTR_IMPLDBG((CE_CONT, "apic_find_irq: return NULL\n"));
275 	return (NULL);
276 }
277 
278 /*
279  * This function will return the pending bit of the irqp.
280  * It either comes from the IRR register of the APIC or the RDT
281  * entry of the I/O APIC.
282  * For the IRR to work, it needs to be to its binding CPU
283  */
284 static int
apic_get_pending(apic_irq_t * irqp,int type)285 apic_get_pending(apic_irq_t *irqp, int type)
286 {
287 	int			bit, index, irr, pending;
288 	int			intin_no;
289 	int			apic_ix;
290 
291 	DDI_INTR_IMPLDBG((CE_CONT, "apic_get_pending: irqp: %p, cpuid: %x "
292 	    "type: %x\n", (void *)irqp, irqp->airq_cpu & ~IRQ_USER_BOUND,
293 	    type));
294 
295 	/* need to get on the bound cpu */
296 	mutex_enter(&cpu_lock);
297 	affinity_set(irqp->airq_cpu & ~IRQ_USER_BOUND);
298 
299 	index = irqp->airq_vector / 32;
300 	bit = irqp->airq_vector % 32;
301 	irr = apic_reg_ops->apic_read(APIC_IRR_REG + index);
302 
303 	affinity_clear();
304 	mutex_exit(&cpu_lock);
305 
306 	pending = (irr & (1 << bit)) ? 1 : 0;
307 	if (!pending && (type == DDI_INTR_TYPE_FIXED)) {
308 		/* check I/O APIC for fixed interrupt */
309 		intin_no = irqp->airq_intin_no;
310 		apic_ix = irqp->airq_ioapicindex;
311 		pending = (READ_IOAPIC_RDT_ENTRY_LOW_DWORD(apic_ix, intin_no) &
312 		    AV_PENDING) ? 1 : 0;
313 	}
314 	return (pending);
315 }
316 
317 
318 /*
319  * This function will clear the mask for the interrupt on the I/O APIC
320  */
321 static void
apic_clear_mask(apic_irq_t * irqp)322 apic_clear_mask(apic_irq_t *irqp)
323 {
324 	int			intin_no;
325 	ulong_t			iflag;
326 	int32_t			rdt_entry;
327 	int 			apic_ix;
328 
329 	DDI_INTR_IMPLDBG((CE_CONT, "apic_clear_mask: irqp: %p\n",
330 	    (void *)irqp));
331 
332 	intin_no = irqp->airq_intin_no;
333 	apic_ix = irqp->airq_ioapicindex;
334 
335 	iflag = intr_clear();
336 	lock_set(&apic_ioapic_lock);
337 
338 	rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(apic_ix, intin_no);
339 
340 	/* clear mask */
341 	WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(apic_ix, intin_no,
342 	    ((~AV_MASK) & rdt_entry));
343 
344 	lock_clear(&apic_ioapic_lock);
345 	intr_restore(iflag);
346 }
347 
348 
349 /*
350  * This function will mask the interrupt on the I/O APIC
351  */
352 static void
apic_set_mask(apic_irq_t * irqp)353 apic_set_mask(apic_irq_t *irqp)
354 {
355 	int			intin_no;
356 	int 			apic_ix;
357 	ulong_t			iflag;
358 	int32_t			rdt_entry;
359 
360 	DDI_INTR_IMPLDBG((CE_CONT, "apic_set_mask: irqp: %p\n", (void *)irqp));
361 
362 	intin_no = irqp->airq_intin_no;
363 	apic_ix = irqp->airq_ioapicindex;
364 
365 	iflag = intr_clear();
366 
367 	lock_set(&apic_ioapic_lock);
368 
369 	rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(apic_ix, intin_no);
370 
371 	/* mask it */
372 	WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(apic_ix, intin_no,
373 	    (AV_MASK | rdt_entry));
374 
375 	lock_clear(&apic_ioapic_lock);
376 	intr_restore(iflag);
377 }
378 
379 
380 void
apic_free_vectors(dev_info_t * dip,int inum,int count,int pri,int type)381 apic_free_vectors(dev_info_t *dip, int inum, int count, int pri, int type)
382 {
383 	int i;
384 	apic_irq_t *irqptr;
385 	struct intrspec ispec;
386 
387 	DDI_INTR_IMPLDBG((CE_CONT, "apic_free_vectors: dip: %p inum: %x "
388 	    "count: %x pri: %x type: %x\n",
389 	    (void *)dip, inum, count, pri, type));
390 
391 	/* for MSI/X only */
392 	if (!DDI_INTR_IS_MSI_OR_MSIX(type))
393 		return;
394 
395 	for (i = 0; i < count; i++) {
396 		DDI_INTR_IMPLDBG((CE_CONT, "apic_free_vectors: inum=0x%x "
397 		    "pri=0x%x count=0x%x\n", inum, pri, count));
398 		ispec.intrspec_vec = inum + i;
399 		ispec.intrspec_pri = pri;
400 		if ((irqptr = apic_find_irq(dip, &ispec, type)) == NULL) {
401 			DDI_INTR_IMPLDBG((CE_CONT, "apic_free_vectors: "
402 			    "dip=0x%p inum=0x%x pri=0x%x apic_find_irq() "
403 			    "failed\n", (void *)dip, inum, pri));
404 			continue;
405 		}
406 		irqptr->airq_mps_intr_index = FREE_INDEX;
407 		apic_vector_to_irq[irqptr->airq_vector] = APIC_RESV_IRQ;
408 	}
409 }
410 
411 /*
412  * apic_pci_msi_enable_mode:
413  */
414 void
apic_pci_msi_enable_mode(dev_info_t * rdip,int type,int inum)415 apic_pci_msi_enable_mode(dev_info_t *rdip, int type, int inum)
416 {
417 	ushort_t		msi_ctrl;
418 	int			cap_ptr = i_ddi_get_msi_msix_cap_ptr(rdip);
419 	ddi_acc_handle_t	handle = i_ddi_get_pci_config_handle(rdip);
420 
421 	ASSERT((handle != NULL) && (cap_ptr != 0));
422 
423 	if (type == DDI_INTR_TYPE_MSI) {
424 		msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL);
425 		if ((msi_ctrl & PCI_MSI_ENABLE_BIT))
426 			return;
427 
428 		msi_ctrl |= PCI_MSI_ENABLE_BIT;
429 		pci_config_put16(handle, cap_ptr + PCI_MSI_CTRL, msi_ctrl);
430 
431 	} else if (type == DDI_INTR_TYPE_MSIX) {
432 		uintptr_t	off;
433 		uint32_t	mask;
434 		ddi_intr_msix_t	*msix_p;
435 
436 		msix_p = i_ddi_get_msix(rdip);
437 
438 		ASSERT(msix_p != NULL);
439 
440 		/* Offset into "inum"th entry in the MSI-X table & clear mask */
441 		off = (uintptr_t)msix_p->msix_tbl_addr + (inum *
442 		    PCI_MSIX_VECTOR_SIZE) + PCI_MSIX_VECTOR_CTRL_OFFSET;
443 
444 		mask = ddi_get32(msix_p->msix_tbl_hdl, (uint32_t *)off);
445 
446 		ddi_put32(msix_p->msix_tbl_hdl, (uint32_t *)off, (mask & ~1));
447 
448 		msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSIX_CTRL);
449 
450 		if (!(msi_ctrl & PCI_MSIX_ENABLE_BIT)) {
451 			msi_ctrl |= PCI_MSIX_ENABLE_BIT;
452 			pci_config_put16(handle, cap_ptr + PCI_MSIX_CTRL,
453 			    msi_ctrl);
454 		}
455 	}
456 }
457 
458 static int
apic_set_cpu(int irqno,int cpu,int * result)459 apic_set_cpu(int irqno, int cpu, int *result)
460 {
461 	apic_irq_t *irqp;
462 	ulong_t iflag;
463 	int ret;
464 
465 	DDI_INTR_IMPLDBG((CE_CONT, "APIC_SET_CPU\n"));
466 
467 	mutex_enter(&airq_mutex);
468 	irqp = apic_irq_table[irqno];
469 	mutex_exit(&airq_mutex);
470 
471 	if (irqp == NULL) {
472 		*result = ENXIO;
473 		return (PSM_FAILURE);
474 	}
475 
476 	/* Fail if this is an MSI intr and is part of a group. */
477 	if ((irqp->airq_mps_intr_index == MSI_INDEX) &&
478 	    (irqp->airq_intin_no > 1)) {
479 		*result = ENXIO;
480 		return (PSM_FAILURE);
481 	}
482 
483 	iflag = intr_clear();
484 	lock_set(&apic_ioapic_lock);
485 
486 	ret = apic_rebind_all(irqp, cpu);
487 
488 	lock_clear(&apic_ioapic_lock);
489 	intr_restore(iflag);
490 
491 	if (ret) {
492 		*result = EIO;
493 		return (PSM_FAILURE);
494 	}
495 	/*
496 	 * keep tracking the default interrupt cpu binding
497 	 */
498 	irqp->airq_cpu = cpu;
499 
500 	*result = 0;
501 	return (PSM_SUCCESS);
502 }
503 
504 static int
apic_grp_set_cpu(int irqno,int new_cpu,int * result)505 apic_grp_set_cpu(int irqno, int new_cpu, int *result)
506 {
507 	dev_info_t *orig_dip;
508 	uint32_t orig_cpu;
509 	ulong_t iflag;
510 	apic_irq_t *irqps[PCI_MSI_MAX_INTRS];
511 	int i;
512 	int cap_ptr;
513 	int msi_mask_off = 0;
514 	ushort_t msi_ctrl;
515 	uint32_t msi_pvm = 0;
516 	ddi_acc_handle_t handle;
517 	int num_vectors = 0;
518 	uint32_t vector;
519 
520 	DDI_INTR_IMPLDBG((CE_CONT, "APIC_GRP_SET_CPU\n"));
521 
522 	/*
523 	 * Take mutex to insure that table doesn't change out from underneath
524 	 * us while we're playing with it.
525 	 */
526 	mutex_enter(&airq_mutex);
527 	irqps[0] = apic_irq_table[irqno];
528 	orig_cpu = irqps[0]->airq_temp_cpu;
529 	orig_dip = irqps[0]->airq_dip;
530 	num_vectors = irqps[0]->airq_intin_no;
531 	vector = irqps[0]->airq_vector;
532 
533 	/* A "group" of 1 */
534 	if (num_vectors == 1) {
535 		mutex_exit(&airq_mutex);
536 		return (apic_set_cpu(irqno, new_cpu, result));
537 	}
538 
539 	*result = ENXIO;
540 
541 	if (irqps[0]->airq_mps_intr_index != MSI_INDEX) {
542 		mutex_exit(&airq_mutex);
543 		DDI_INTR_IMPLDBG((CE_CONT, "set_grp: intr not MSI\n"));
544 		goto set_grp_intr_done;
545 	}
546 	if ((num_vectors < 1) || ((num_vectors - 1) & vector)) {
547 		mutex_exit(&airq_mutex);
548 		DDI_INTR_IMPLDBG((CE_CONT,
549 		    "set_grp: base vec not part of a grp or not aligned: "
550 		    "vec:0x%x, num_vec:0x%x\n", vector, num_vectors));
551 		goto set_grp_intr_done;
552 	}
553 	DDI_INTR_IMPLDBG((CE_CONT, "set_grp: num intrs in grp: %d\n",
554 	    num_vectors));
555 
556 	ASSERT((num_vectors + vector) < APIC_MAX_VECTOR);
557 
558 	*result = EIO;
559 
560 	/*
561 	 * All IRQ entries in the table for the given device will be not
562 	 * shared.  Since they are not shared, the dip in the table will
563 	 * be true to the device of interest.
564 	 */
565 	for (i = 1; i < num_vectors; i++) {
566 		irqps[i] = apic_irq_table[apic_vector_to_irq[vector + i]];
567 		if (irqps[i] == NULL) {
568 			mutex_exit(&airq_mutex);
569 			goto set_grp_intr_done;
570 		}
571 #ifdef DEBUG
572 		/* Sanity check: CPU and dip is the same for all entries. */
573 		if ((irqps[i]->airq_dip != orig_dip) ||
574 		    (irqps[i]->airq_temp_cpu != orig_cpu)) {
575 			mutex_exit(&airq_mutex);
576 			DDI_INTR_IMPLDBG((CE_CONT,
577 			    "set_grp: cpu or dip for vec 0x%x difft than for "
578 			    "vec 0x%x\n", vector, vector + i));
579 			DDI_INTR_IMPLDBG((CE_CONT,
580 			    "  cpu: %d vs %d, dip: 0x%p vs 0x%p\n", orig_cpu,
581 			    irqps[i]->airq_temp_cpu, (void *)orig_dip,
582 			    (void *)irqps[i]->airq_dip));
583 			goto set_grp_intr_done;
584 		}
585 #endif /* DEBUG */
586 	}
587 	mutex_exit(&airq_mutex);
588 
589 	cap_ptr = i_ddi_get_msi_msix_cap_ptr(orig_dip);
590 	handle = i_ddi_get_pci_config_handle(orig_dip);
591 	msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL);
592 
593 	/* MSI Per vector masking is supported. */
594 	if (msi_ctrl & PCI_MSI_PVM_MASK) {
595 		if (msi_ctrl &  PCI_MSI_64BIT_MASK)
596 			msi_mask_off = cap_ptr + PCI_MSI_64BIT_MASKBITS;
597 		else
598 			msi_mask_off = cap_ptr + PCI_MSI_32BIT_MASK;
599 		msi_pvm = pci_config_get32(handle, msi_mask_off);
600 		pci_config_put32(handle, msi_mask_off, (uint32_t)-1);
601 		DDI_INTR_IMPLDBG((CE_CONT,
602 		    "set_grp: pvm supported.  Mask set to 0x%x\n",
603 		    pci_config_get32(handle, msi_mask_off)));
604 	}
605 
606 	iflag = intr_clear();
607 	lock_set(&apic_ioapic_lock);
608 
609 	/*
610 	 * Do the first rebind and check for errors.  Apic_rebind_all returns
611 	 * an error if the CPU is not accepting interrupts.  If the first one
612 	 * succeeds they all will.
613 	 */
614 	if (apic_rebind_all(irqps[0], new_cpu))
615 		(void) apic_rebind_all(irqps[0], orig_cpu);
616 	else {
617 		irqps[0]->airq_cpu = new_cpu;
618 
619 		for (i = 1; i < num_vectors; i++) {
620 			(void) apic_rebind_all(irqps[i], new_cpu);
621 			irqps[i]->airq_cpu = new_cpu;
622 		}
623 		*result = 0;	/* SUCCESS */
624 	}
625 
626 	lock_clear(&apic_ioapic_lock);
627 	intr_restore(iflag);
628 
629 	/* Reenable vectors if per vector masking is supported. */
630 	if (msi_ctrl & PCI_MSI_PVM_MASK) {
631 		pci_config_put32(handle, msi_mask_off, msi_pvm);
632 		DDI_INTR_IMPLDBG((CE_CONT,
633 		    "set_grp: pvm supported.  Mask restored to 0x%x\n",
634 		    pci_config_get32(handle, msi_mask_off)));
635 	}
636 
637 set_grp_intr_done:
638 	if (*result != 0)
639 		return (PSM_FAILURE);
640 
641 	return (PSM_SUCCESS);
642 }
643 
644 int
apic_get_vector_intr_info(int vecirq,apic_get_intr_t * intr_params_p)645 apic_get_vector_intr_info(int vecirq, apic_get_intr_t *intr_params_p)
646 {
647 	struct autovec *av_dev;
648 	uchar_t irqno;
649 	uint_t i;
650 	apic_irq_t *irq_p;
651 
652 	/* Sanity check the vector/irq argument. */
653 	ASSERT((vecirq >= 0) || (vecirq <= APIC_MAX_VECTOR));
654 
655 	mutex_enter(&airq_mutex);
656 
657 	/*
658 	 * Convert the vecirq arg to an irq using vector_to_irq table
659 	 * if the arg is a vector.  Pass thru if already an irq.
660 	 */
661 	if ((intr_params_p->avgi_req_flags & PSMGI_INTRBY_FLAGS) ==
662 	    PSMGI_INTRBY_VEC)
663 		irqno = apic_vector_to_irq[vecirq];
664 	else
665 		irqno = (uchar_t)vecirq;
666 
667 	irq_p = apic_irq_table[irqno];
668 
669 	if ((irq_p == NULL) ||
670 	    ((irq_p->airq_mps_intr_index != RESERVE_INDEX) &&
671 	    ((irq_p->airq_temp_cpu == IRQ_UNBOUND) ||
672 	    (irq_p->airq_temp_cpu == IRQ_UNINIT)))) {
673 		mutex_exit(&airq_mutex);
674 		return (PSM_FAILURE);
675 	}
676 
677 	if (intr_params_p->avgi_req_flags & PSMGI_REQ_CPUID) {
678 
679 		/* Get the (temp) cpu from apic_irq table, indexed by irq. */
680 		intr_params_p->avgi_cpu_id = irq_p->airq_temp_cpu;
681 
682 		/* Return user bound info for intrd. */
683 		if (intr_params_p->avgi_cpu_id & IRQ_USER_BOUND) {
684 			intr_params_p->avgi_cpu_id &= ~IRQ_USER_BOUND;
685 			intr_params_p->avgi_cpu_id |= PSMGI_CPU_USER_BOUND;
686 		}
687 	}
688 
689 	if (intr_params_p->avgi_req_flags & PSMGI_REQ_VECTOR)
690 		intr_params_p->avgi_vector = irq_p->airq_vector;
691 
692 	if (intr_params_p->avgi_req_flags &
693 	    (PSMGI_REQ_NUM_DEVS | PSMGI_REQ_GET_DEVS))
694 		/* Get number of devices from apic_irq table shared field. */
695 		intr_params_p->avgi_num_devs = irq_p->airq_share;
696 
697 	if (intr_params_p->avgi_req_flags &  PSMGI_REQ_GET_DEVS) {
698 
699 		intr_params_p->avgi_req_flags  |= PSMGI_REQ_NUM_DEVS;
700 
701 		/* Some devices have NULL dip.  Don't count these. */
702 		if (intr_params_p->avgi_num_devs > 0) {
703 			for (i = 0, av_dev = autovect[irqno].avh_link;
704 			    av_dev; av_dev = av_dev->av_link)
705 				if (av_dev->av_vector && av_dev->av_dip)
706 					i++;
707 			intr_params_p->avgi_num_devs =
708 			    (uchar_t)MIN(intr_params_p->avgi_num_devs, i);
709 		}
710 
711 		/* There are no viable dips to return. */
712 		if (intr_params_p->avgi_num_devs == 0)
713 			intr_params_p->avgi_dip_list = NULL;
714 
715 		else {	/* Return list of dips */
716 
717 			/* Allocate space in array for that number of devs. */
718 			intr_params_p->avgi_dip_list = kmem_zalloc(
719 			    intr_params_p->avgi_num_devs *
720 			    sizeof (dev_info_t *),
721 			    KM_SLEEP);
722 
723 			/*
724 			 * Loop through the device list of the autovec table
725 			 * filling in the dip array.
726 			 *
727 			 * Note that the autovect table may have some special
728 			 * entries which contain NULL dips.  These will be
729 			 * ignored.
730 			 */
731 			for (i = 0, av_dev = autovect[irqno].avh_link;
732 			    av_dev; av_dev = av_dev->av_link)
733 				if (av_dev->av_vector && av_dev->av_dip)
734 					intr_params_p->avgi_dip_list[i++] =
735 					    av_dev->av_dip;
736 		}
737 	}
738 
739 	mutex_exit(&airq_mutex);
740 
741 	return (PSM_SUCCESS);
742 }
743 
744 /*
745  * This function provides external interface to the nexus for all
746  * functionalities related to the new DDI interrupt framework.
747  *
748  * Input:
749  * dip     - pointer to the dev_info structure of the requested device
750  * hdlp    - pointer to the internal interrupt handle structure for the
751  *	     requested interrupt
752  * intr_op - opcode for this call
753  * result  - pointer to the integer that will hold the result to be
754  *	     passed back if return value is PSM_SUCCESS
755  *
756  * Output:
757  * return value is either PSM_SUCCESS or PSM_FAILURE
758  */
759 int
apic_intr_ops(dev_info_t * dip,ddi_intr_handle_impl_t * hdlp,psm_intr_op_t intr_op,int * result)760 apic_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp,
761     psm_intr_op_t intr_op, int *result)
762 {
763 	int		cap;
764 	int		count_vec;
765 	int		old_priority;
766 	int		new_priority;
767 	int		new_cpu;
768 	apic_irq_t	*irqp;
769 	struct intrspec *ispec, intr_spec;
770 
771 	DDI_INTR_IMPLDBG((CE_CONT, "apic_intr_ops: dip: %p hdlp: %p "
772 	    "intr_op: %x\n", (void *)dip, (void *)hdlp, intr_op));
773 
774 	ispec = &intr_spec;
775 	ispec->intrspec_pri = hdlp->ih_pri;
776 	ispec->intrspec_vec = hdlp->ih_inum;
777 	ispec->intrspec_func = hdlp->ih_cb_func;
778 
779 	switch (intr_op) {
780 	case PSM_INTR_OP_CHECK_MSI:
781 		/*
782 		 * Check MSI/X is supported or not at APIC level and
783 		 * masked off the MSI/X bits in hdlp->ih_type if not
784 		 * supported before return.  If MSI/X is supported,
785 		 * leave the ih_type unchanged and return.
786 		 *
787 		 * hdlp->ih_type passed in from the nexus has all the
788 		 * interrupt types supported by the device.
789 		 */
790 		if (apic_support_msi == 0) {
791 			/*
792 			 * if apic_support_msi is not set, call
793 			 * apic_check_msi_support() to check whether msi
794 			 * is supported first
795 			 */
796 			if (apic_check_msi_support() == PSM_SUCCESS)
797 				apic_support_msi = 1;
798 			else
799 				apic_support_msi = -1;
800 		}
801 		if (apic_support_msi == 1) {
802 			if (apic_msix_enable)
803 				*result = hdlp->ih_type;
804 			else
805 				*result = hdlp->ih_type & ~DDI_INTR_TYPE_MSIX;
806 		} else
807 			*result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI |
808 			    DDI_INTR_TYPE_MSIX);
809 		break;
810 	case PSM_INTR_OP_ALLOC_VECTORS:
811 		if (hdlp->ih_type == DDI_INTR_TYPE_MSI)
812 			*result = apic_alloc_msi_vectors(dip, hdlp->ih_inum,
813 			    hdlp->ih_scratch1, hdlp->ih_pri,
814 			    (int)(uintptr_t)hdlp->ih_scratch2);
815 		else
816 			*result = apic_alloc_msix_vectors(dip, hdlp->ih_inum,
817 			    hdlp->ih_scratch1, hdlp->ih_pri,
818 			    (int)(uintptr_t)hdlp->ih_scratch2);
819 		break;
820 	case PSM_INTR_OP_FREE_VECTORS:
821 		apic_free_vectors(dip, hdlp->ih_inum, hdlp->ih_scratch1,
822 		    hdlp->ih_pri, hdlp->ih_type);
823 		break;
824 	case PSM_INTR_OP_NAVAIL_VECTORS:
825 		*result = apic_navail_vector(dip, hdlp->ih_pri);
826 		break;
827 	case PSM_INTR_OP_XLATE_VECTOR:
828 		ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp;
829 		*result = apic_introp_xlate(dip, ispec, hdlp->ih_type);
830 		if (*result == -1)
831 			return (PSM_FAILURE);
832 		break;
833 	case PSM_INTR_OP_GET_PENDING:
834 		if ((irqp = apic_find_irq(dip, ispec, hdlp->ih_type)) == NULL)
835 			return (PSM_FAILURE);
836 		*result = apic_get_pending(irqp, hdlp->ih_type);
837 		break;
838 	case PSM_INTR_OP_CLEAR_MASK:
839 		if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
840 			return (PSM_FAILURE);
841 		irqp = apic_find_irq(dip, ispec, hdlp->ih_type);
842 		if (irqp == NULL)
843 			return (PSM_FAILURE);
844 		apic_clear_mask(irqp);
845 		break;
846 	case PSM_INTR_OP_SET_MASK:
847 		if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
848 			return (PSM_FAILURE);
849 		if ((irqp = apic_find_irq(dip, ispec, hdlp->ih_type)) == NULL)
850 			return (PSM_FAILURE);
851 		apic_set_mask(irqp);
852 		break;
853 	case PSM_INTR_OP_GET_CAP:
854 		cap = DDI_INTR_FLAG_PENDING;
855 		if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
856 			cap |= DDI_INTR_FLAG_MASKABLE;
857 		*result = cap;
858 		break;
859 	case PSM_INTR_OP_GET_SHARED:
860 		if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
861 			return (PSM_FAILURE);
862 		ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp;
863 		if ((irqp = apic_find_irq(dip, ispec, hdlp->ih_type)) == NULL)
864 			return (PSM_FAILURE);
865 		*result = (irqp->airq_share > 1) ? 1: 0;
866 		break;
867 	case PSM_INTR_OP_SET_PRI:
868 		old_priority = hdlp->ih_pri;	/* save old value */
869 		new_priority = *(int *)result;	/* try the new value */
870 
871 		if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) {
872 			return (PSM_SUCCESS);
873 		}
874 
875 		/* Now allocate the vectors */
876 		if (hdlp->ih_type == DDI_INTR_TYPE_MSI) {
877 			/* SET_PRI does not support the case of multiple MSI */
878 			if (i_ddi_intr_get_current_nintrs(hdlp->ih_dip) > 1)
879 				return (PSM_FAILURE);
880 
881 			count_vec = apic_alloc_msi_vectors(dip, hdlp->ih_inum,
882 			    1, new_priority,
883 			    DDI_INTR_ALLOC_STRICT);
884 		} else {
885 			count_vec = apic_alloc_msix_vectors(dip, hdlp->ih_inum,
886 			    1, new_priority,
887 			    DDI_INTR_ALLOC_STRICT);
888 		}
889 
890 		/* Did we get new vectors? */
891 		if (!count_vec)
892 			return (PSM_FAILURE);
893 
894 		/* Finally, free the previously allocated vectors */
895 		apic_free_vectors(dip, hdlp->ih_inum, count_vec,
896 		    old_priority, hdlp->ih_type);
897 		break;
898 	case PSM_INTR_OP_SET_CPU:
899 	case PSM_INTR_OP_GRP_SET_CPU:
900 		/*
901 		 * The interrupt handle given here has been allocated
902 		 * specifically for this command, and ih_private carries
903 		 * a CPU value.
904 		 */
905 		new_cpu = (int)(intptr_t)hdlp->ih_private;
906 		if (!apic_cpu_in_range(new_cpu)) {
907 			DDI_INTR_IMPLDBG((CE_CONT,
908 			    "[grp_]set_cpu: cpu out of range: %d\n", new_cpu));
909 			*result = EINVAL;
910 			return (PSM_FAILURE);
911 		}
912 		if (hdlp->ih_vector > APIC_MAX_VECTOR) {
913 			DDI_INTR_IMPLDBG((CE_CONT,
914 			    "[grp_]set_cpu: vector out of range: %d\n",
915 			    hdlp->ih_vector));
916 			*result = EINVAL;
917 			return (PSM_FAILURE);
918 		}
919 		if ((hdlp->ih_flags & PSMGI_INTRBY_FLAGS) == PSMGI_INTRBY_VEC)
920 			hdlp->ih_vector = apic_vector_to_irq[hdlp->ih_vector];
921 		if (intr_op == PSM_INTR_OP_SET_CPU) {
922 			if (apic_set_cpu(hdlp->ih_vector, new_cpu, result) !=
923 			    PSM_SUCCESS)
924 				return (PSM_FAILURE);
925 		} else {
926 			if (apic_grp_set_cpu(hdlp->ih_vector, new_cpu,
927 			    result) != PSM_SUCCESS)
928 				return (PSM_FAILURE);
929 		}
930 		break;
931 	case PSM_INTR_OP_GET_INTR:
932 		/*
933 		 * The interrupt handle given here has been allocated
934 		 * specifically for this command, and ih_private carries
935 		 * a pointer to a apic_get_intr_t.
936 		 */
937 		if (apic_get_vector_intr_info(
938 		    hdlp->ih_vector, hdlp->ih_private) != PSM_SUCCESS)
939 			return (PSM_FAILURE);
940 		break;
941 	case PSM_INTR_OP_APIC_TYPE:
942 		((apic_get_type_t *)(hdlp->ih_private))->avgi_type =
943 		    apic_get_apic_type();
944 		((apic_get_type_t *)(hdlp->ih_private))->avgi_num_intr =
945 		    APIC_MAX_VECTOR;
946 		((apic_get_type_t *)(hdlp->ih_private))->avgi_num_cpu =
947 		    boot_ncpus;
948 		hdlp->ih_ver = apic_get_apic_version();
949 		break;
950 	case PSM_INTR_OP_SET_CAP:
951 	default:
952 		return (PSM_FAILURE);
953 	}
954 	return (PSM_SUCCESS);
955 }
956