1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 /*
26 * Copyright 2023 Oxide Computer Company
27 */
28
29 #include <sys/sysmacros.h>
30 #include <sys/types.h>
31 #include <sys/mkdev.h>
32 #include <sys/stat.h>
33 #include <sys/sunddi.h>
34 #include <vm/seg_kmem.h>
35 #include <sys/machparam.h>
36 #include <sys/sunndi.h>
37 #include <sys/ontrap.h>
38 #include <sys/psm.h>
39 #include <sys/pcie.h>
40 #include <sys/pci_cfgspace.h>
41 #include <sys/pci_tools.h>
42 #include <io/pci/pci_tools_ext.h>
43 #include <sys/apic.h>
44 #include <sys/apix.h>
45 #include <io/pci/pci_var.h>
46 #include <sys/pci_impl.h>
47 #include <sys/promif.h>
48 #include <sys/x86_archext.h>
49 #include <sys/cpuvar.h>
50 #include <sys/pci_cfgacc.h>
51
52 #ifdef __xpv
53 #include <sys/hypervisor.h>
54 #endif
55
56 #define PCIEX_BDF_OFFSET_DELTA 4
57 #define PCIEX_REG_FUNC_SHIFT (PCI_REG_FUNC_SHIFT + PCIEX_BDF_OFFSET_DELTA)
58 #define PCIEX_REG_DEV_SHIFT (PCI_REG_DEV_SHIFT + PCIEX_BDF_OFFSET_DELTA)
59 #define PCIEX_REG_BUS_SHIFT (PCI_REG_BUS_SHIFT + PCIEX_BDF_OFFSET_DELTA)
60
61 #define SUCCESS 0
62
63 extern uint64_t mcfg_mem_base;
64 extern uint_t pci_iocfg_max_offset;
65 int pcitool_debug = 0;
66
67 /*
68 * Offsets of BARS in config space. First entry of 0 means config space.
69 * Entries here correlate to pcitool_bars_t enumerated type.
70 */
71 static uint8_t pci_bars[] = {
72 0x0,
73 PCI_CONF_BASE0,
74 PCI_CONF_BASE1,
75 PCI_CONF_BASE2,
76 PCI_CONF_BASE3,
77 PCI_CONF_BASE4,
78 PCI_CONF_BASE5,
79 PCI_CONF_ROM
80 };
81
82 /* Max offset allowed into config space for a particular device. */
83 static uint64_t max_cfg_size = PCI_CONF_HDR_SIZE;
84
85 static uint64_t pcitool_swap_endian(uint64_t data, int size);
86 static int pcitool_cfg_access(pcitool_reg_t *prg, boolean_t write_flag,
87 boolean_t io_access);
88 static int pcitool_io_access(pcitool_reg_t *prg, boolean_t write_flag);
89 static int pcitool_mem_access(pcitool_reg_t *prg, uint64_t virt_addr,
90 boolean_t write_flag);
91 static uint64_t pcitool_map(uint64_t phys_addr, size_t size, size_t *num_pages);
92 static void pcitool_unmap(uint64_t virt_addr, size_t num_pages);
93
94 /* Extern declarations */
95 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
96 psm_intr_op_t, int *);
97
98 int
pcitool_init(dev_info_t * dip,boolean_t is_pciex)99 pcitool_init(dev_info_t *dip, boolean_t is_pciex)
100 {
101 int instance = ddi_get_instance(dip);
102
103 /* Create pcitool nodes for register access and interrupt routing. */
104
105 if (ddi_create_minor_node(dip, PCI_MINOR_REG, S_IFCHR,
106 PCI_MINOR_NUM(instance, PCI_TOOL_REG_MINOR_NUM),
107 DDI_NT_REGACC, 0) != DDI_SUCCESS) {
108 return (DDI_FAILURE);
109 }
110
111 if (ddi_create_minor_node(dip, PCI_MINOR_INTR, S_IFCHR,
112 PCI_MINOR_NUM(instance, PCI_TOOL_INTR_MINOR_NUM),
113 DDI_NT_INTRCTL, 0) != DDI_SUCCESS) {
114 ddi_remove_minor_node(dip, PCI_MINOR_REG);
115 return (DDI_FAILURE);
116 }
117
118 if (is_pciex)
119 max_cfg_size = PCIE_CONF_HDR_SIZE;
120
121 return (DDI_SUCCESS);
122 }
123
124 void
pcitool_uninit(dev_info_t * dip)125 pcitool_uninit(dev_info_t *dip)
126 {
127 ddi_remove_minor_node(dip, PCI_MINOR_INTR);
128 ddi_remove_minor_node(dip, PCI_MINOR_REG);
129 }
130
131 /*ARGSUSED*/
132 static int
pcitool_set_intr(dev_info_t * dip,void * arg,int mode)133 pcitool_set_intr(dev_info_t *dip, void *arg, int mode)
134 {
135 ddi_intr_handle_impl_t info_hdl;
136 pcitool_intr_set_t iset;
137 uint32_t old_cpu;
138 int ret, result;
139 size_t copyinout_size;
140 int rval = SUCCESS;
141 apic_get_type_t type_info;
142
143 /* Version 1 of pcitool_intr_set_t doesn't have flags. */
144 copyinout_size = (size_t)&iset.flags - (size_t)&iset;
145
146 if (ddi_copyin(arg, &iset, copyinout_size, mode) != DDI_SUCCESS)
147 return (EFAULT);
148
149 switch (iset.user_version) {
150 case PCITOOL_V1:
151 break;
152
153 case PCITOOL_V2:
154 copyinout_size = sizeof (pcitool_intr_set_t);
155 if (ddi_copyin(arg, &iset, copyinout_size, mode) != DDI_SUCCESS)
156 return (EFAULT);
157 break;
158
159 default:
160 iset.status = PCITOOL_OUT_OF_RANGE;
161 rval = ENOTSUP;
162 goto done_set_intr;
163 }
164
165 if (iset.flags & PCITOOL_INTR_FLAG_SET_MSI) {
166 rval = ENOTSUP;
167 iset.status = PCITOOL_IO_ERROR;
168 goto done_set_intr;
169 }
170
171 info_hdl.ih_private = &type_info;
172
173 if ((*psm_intr_ops)(NULL, &info_hdl,
174 PSM_INTR_OP_APIC_TYPE, NULL) != PSM_SUCCESS) {
175 rval = ENOTSUP;
176 iset.status = PCITOOL_IO_ERROR;
177 goto done_set_intr;
178 }
179
180 if (strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
181 if (iset.old_cpu > type_info.avgi_num_cpu) {
182 rval = EINVAL;
183 iset.status = PCITOOL_INVALID_CPUID;
184 goto done_set_intr;
185 }
186 old_cpu = iset.old_cpu;
187 } else {
188 if ((old_cpu =
189 pci_get_cpu_from_vecirq(iset.ino, IS_VEC)) == -1) {
190 iset.status = PCITOOL_IO_ERROR;
191 rval = EINVAL;
192 goto done_set_intr;
193 }
194 }
195
196 if (iset.ino > type_info.avgi_num_intr) {
197 rval = EINVAL;
198 iset.status = PCITOOL_INVALID_INO;
199 goto done_set_intr;
200 }
201
202 iset.status = PCITOOL_SUCCESS;
203
204 old_cpu &= ~PSMGI_CPU_USER_BOUND;
205
206 /*
207 * For this locally-declared and used handle, ih_private will contain a
208 * CPU value, not an ihdl_plat_t as used for global interrupt handling.
209 */
210 if (strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
211 info_hdl.ih_vector = APIX_VIRTVECTOR(old_cpu, iset.ino);
212 } else {
213 info_hdl.ih_vector = iset.ino;
214 }
215 info_hdl.ih_private = (void *)(uintptr_t)iset.cpu_id;
216 info_hdl.ih_flags = PSMGI_INTRBY_VEC;
217 if (pcitool_debug)
218 prom_printf("user version:%d, flags:0x%x\n",
219 iset.user_version, iset.flags);
220
221 result = ENOTSUP;
222 if ((iset.user_version >= PCITOOL_V2) &&
223 (iset.flags & PCITOOL_INTR_FLAG_SET_GROUP)) {
224 ret = (*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_GRP_SET_CPU,
225 &result);
226 } else {
227 ret = (*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_SET_CPU,
228 &result);
229 }
230
231 if (ret != PSM_SUCCESS) {
232 switch (result) {
233 case EIO: /* Error making the change */
234 rval = EIO;
235 iset.status = PCITOOL_IO_ERROR;
236 break;
237 case ENXIO: /* Couldn't convert vector to irq */
238 rval = EINVAL;
239 iset.status = PCITOOL_INVALID_INO;
240 break;
241 case EINVAL: /* CPU out of range */
242 rval = EINVAL;
243 iset.status = PCITOOL_INVALID_CPUID;
244 break;
245 case ENOTSUP: /* Requested PSM intr ops missing */
246 rval = ENOTSUP;
247 iset.status = PCITOOL_IO_ERROR;
248 break;
249 }
250 }
251
252 /* Return original CPU. */
253 iset.cpu_id = old_cpu;
254
255 /* Return new vector */
256 if (strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
257 iset.ino = APIX_VIRTVEC_VECTOR(info_hdl.ih_vector);
258 }
259
260 done_set_intr:
261 iset.drvr_version = PCITOOL_VERSION;
262 if (ddi_copyout(&iset, arg, copyinout_size, mode) != DDI_SUCCESS)
263 rval = EFAULT;
264 return (rval);
265 }
266
267
268 /* It is assumed that dip != NULL */
269 static void
pcitool_get_intr_dev_info(dev_info_t * dip,pcitool_intr_dev_t * devs)270 pcitool_get_intr_dev_info(dev_info_t *dip, pcitool_intr_dev_t *devs)
271 {
272 (void) strncpy(devs->driver_name,
273 ddi_driver_name(dip), MAXMODCONFNAME-2);
274 devs->driver_name[MAXMODCONFNAME-1] = '\0';
275 (void) ddi_pathname(dip, devs->path);
276 devs->dev_inst = ddi_get_instance(dip);
277 }
278
279 static int
pcitool_get_intr(dev_info_t * dip,void * arg,int mode)280 pcitool_get_intr(dev_info_t *dip, void *arg, int mode)
281 {
282 /* Array part isn't used here, but oh well... */
283 pcitool_intr_get_t partial_iget;
284 pcitool_intr_get_t *iget = &partial_iget;
285 size_t iget_kmem_alloc_size = 0;
286 uint8_t num_devs_ret = 0;
287 int copyout_rval;
288 int rval = SUCCESS;
289 int i;
290 ddi_intr_handle_impl_t info_hdl;
291 apic_get_intr_t intr_info;
292 apic_get_type_t type_info;
293
294 /* Read in just the header part, no array section. */
295 if (ddi_copyin(arg, &partial_iget, PCITOOL_IGET_SIZE(0), mode) !=
296 DDI_SUCCESS)
297 return (EFAULT);
298
299 if (partial_iget.flags & PCITOOL_INTR_FLAG_GET_MSI) {
300 partial_iget.status = PCITOOL_IO_ERROR;
301 partial_iget.num_devs_ret = 0;
302 rval = ENOTSUP;
303 goto done_get_intr;
304 }
305
306 info_hdl.ih_private = &type_info;
307
308 if ((*psm_intr_ops)(NULL, &info_hdl,
309 PSM_INTR_OP_APIC_TYPE, NULL) != PSM_SUCCESS) {
310 iget->status = PCITOOL_IO_ERROR;
311 iget->num_devs_ret = 0;
312 rval = EINVAL;
313 goto done_get_intr;
314 }
315
316 if (strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
317 if (partial_iget.cpu_id > type_info.avgi_num_cpu) {
318 partial_iget.status = PCITOOL_INVALID_CPUID;
319 partial_iget.num_devs_ret = 0;
320 rval = EINVAL;
321 goto done_get_intr;
322 }
323 }
324
325 /* Validate argument. */
326 if ((partial_iget.ino & APIX_VIRTVEC_VECMASK) >
327 type_info.avgi_num_intr) {
328 partial_iget.status = PCITOOL_INVALID_INO;
329 partial_iget.num_devs_ret = 0;
330 rval = EINVAL;
331 goto done_get_intr;
332 }
333
334 num_devs_ret = partial_iget.num_devs_ret;
335 intr_info.avgi_dip_list = NULL;
336 intr_info.avgi_req_flags =
337 PSMGI_REQ_CPUID | PSMGI_REQ_NUM_DEVS | PSMGI_INTRBY_VEC;
338 /*
339 * For this locally-declared and used handle, ih_private will contain a
340 * pointer to apic_get_intr_t, not an ihdl_plat_t as used for
341 * global interrupt handling.
342 */
343 info_hdl.ih_private = &intr_info;
344
345 if (strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
346 info_hdl.ih_vector =
347 APIX_VIRTVECTOR(partial_iget.cpu_id, partial_iget.ino);
348 } else {
349 info_hdl.ih_vector = partial_iget.ino;
350 }
351
352 /* Caller wants device information returned. */
353 if (num_devs_ret > 0) {
354
355 intr_info.avgi_req_flags |= PSMGI_REQ_GET_DEVS;
356
357 /*
358 * Allocate room.
359 * If num_devs_ret == 0 iget remains pointing to partial_iget.
360 */
361 iget_kmem_alloc_size = PCITOOL_IGET_SIZE(num_devs_ret);
362 iget = kmem_alloc(iget_kmem_alloc_size, KM_SLEEP);
363
364 /* Read in whole structure to verify there's room. */
365 if (ddi_copyin(arg, iget, iget_kmem_alloc_size, mode) !=
366 SUCCESS) {
367
368 /* Be consistent and just return EFAULT here. */
369 kmem_free(iget, iget_kmem_alloc_size);
370
371 return (EFAULT);
372 }
373 }
374
375 bzero(iget, PCITOOL_IGET_SIZE(num_devs_ret));
376 iget->ino = info_hdl.ih_vector;
377
378 /*
379 * Lock device tree branch from the pci root nexus on down if info will
380 * be extracted from dips returned from the tree.
381 */
382 if (intr_info.avgi_req_flags & PSMGI_REQ_GET_DEVS) {
383 ndi_devi_enter(dip);
384 }
385
386 /* Call psm_intr_ops(PSM_INTR_OP_GET_INTR) to get information. */
387 if ((rval = (*psm_intr_ops)(NULL, &info_hdl,
388 PSM_INTR_OP_GET_INTR, NULL)) != PSM_SUCCESS) {
389 iget->status = PCITOOL_IO_ERROR;
390 iget->num_devs_ret = 0;
391 rval = EINVAL;
392 goto done_get_intr;
393 }
394
395 /*
396 * Fill in the pcitool_intr_get_t to be returned,
397 * with the CPU, num_devs_ret and num_devs.
398 */
399 if (intr_info.avgi_cpu_id == IRQ_UNBOUND ||
400 intr_info.avgi_cpu_id == IRQ_UNINIT)
401 iget->cpu_id = 0;
402 else
403 iget->cpu_id = intr_info.avgi_cpu_id & ~PSMGI_CPU_USER_BOUND;
404
405 /* Number of devices returned by apic. */
406 iget->num_devs = intr_info.avgi_num_devs;
407
408 /* Device info was returned. */
409 if (intr_info.avgi_req_flags & PSMGI_REQ_GET_DEVS) {
410
411 /*
412 * num devs returned is num devs ret by apic,
413 * space permitting.
414 */
415 iget->num_devs_ret = min(num_devs_ret, intr_info.avgi_num_devs);
416
417 /*
418 * Loop thru list of dips and extract driver, name and instance.
419 * Fill in the pcitool_intr_dev_t's with this info.
420 */
421 for (i = 0; i < iget->num_devs_ret; i++)
422 pcitool_get_intr_dev_info(intr_info.avgi_dip_list[i],
423 &iget->dev[i]);
424
425 /* Free kmem_alloc'ed memory of the apic_get_intr_t */
426 kmem_free(intr_info.avgi_dip_list,
427 intr_info.avgi_num_devs * sizeof (dev_info_t *));
428 }
429
430 done_get_intr:
431
432 if (intr_info.avgi_req_flags & PSMGI_REQ_GET_DEVS) {
433 ndi_devi_exit(dip);
434 }
435
436 iget->drvr_version = PCITOOL_VERSION;
437 copyout_rval = ddi_copyout(iget, arg,
438 PCITOOL_IGET_SIZE(num_devs_ret), mode);
439
440 if (iget_kmem_alloc_size > 0)
441 kmem_free(iget, iget_kmem_alloc_size);
442
443 if (copyout_rval != DDI_SUCCESS)
444 rval = EFAULT;
445
446 return (rval);
447 }
448
449 /*ARGSUSED*/
450 static int
pcitool_intr_info(dev_info_t * dip,void * arg,int mode)451 pcitool_intr_info(dev_info_t *dip, void *arg, int mode)
452 {
453 pcitool_intr_info_t intr_info;
454 ddi_intr_handle_impl_t info_hdl;
455 int rval = SUCCESS;
456 apic_get_type_t type_info;
457
458 /* If we need user_version, and to ret same user version as passed in */
459 if (ddi_copyin(arg, &intr_info, sizeof (pcitool_intr_info_t), mode) !=
460 DDI_SUCCESS) {
461 if (pcitool_debug)
462 prom_printf("Error reading arguments\n");
463 return (EFAULT);
464 }
465
466 if (intr_info.flags & PCITOOL_INTR_FLAG_GET_MSI)
467 return (ENOTSUP);
468
469 info_hdl.ih_private = &type_info;
470
471 /* For UPPC systems, psm_intr_ops has no entry for APIC_TYPE. */
472 if ((rval = (*psm_intr_ops)(NULL, &info_hdl,
473 PSM_INTR_OP_APIC_TYPE, NULL)) != PSM_SUCCESS) {
474 intr_info.ctlr_type = PCITOOL_CTLR_TYPE_UPPC;
475 intr_info.ctlr_version = 0;
476 intr_info.num_intr = APIC_MAX_VECTOR;
477 } else {
478 intr_info.ctlr_version = (uint32_t)info_hdl.ih_ver;
479 intr_info.num_cpu = type_info.avgi_num_cpu;
480 if (strcmp(type_info.avgi_type,
481 APIC_PCPLUSMP_NAME) == 0) {
482 intr_info.ctlr_type = PCITOOL_CTLR_TYPE_PCPLUSMP;
483 intr_info.num_intr = type_info.avgi_num_intr;
484 } else if (strcmp(type_info.avgi_type,
485 APIC_APIX_NAME) == 0) {
486 intr_info.ctlr_type = PCITOOL_CTLR_TYPE_APIX;
487 intr_info.num_intr = type_info.avgi_num_intr;
488 } else {
489 intr_info.ctlr_type = PCITOOL_CTLR_TYPE_UNKNOWN;
490 intr_info.num_intr = APIC_MAX_VECTOR;
491 }
492 }
493
494 intr_info.drvr_version = PCITOOL_VERSION;
495 if (ddi_copyout(&intr_info, arg, sizeof (pcitool_intr_info_t), mode) !=
496 DDI_SUCCESS) {
497 if (pcitool_debug)
498 prom_printf("Error returning arguments.\n");
499 rval = EFAULT;
500 }
501
502 return (rval);
503 }
504
505
506
507 /*
508 * Main function for handling interrupt CPU binding requests and queries.
509 * Need to implement later
510 */
511 int
pcitool_intr_admn(dev_info_t * dip,void * arg,int cmd,int mode)512 pcitool_intr_admn(dev_info_t *dip, void *arg, int cmd, int mode)
513 {
514 int rval;
515
516 switch (cmd) {
517
518 /* Associate a new CPU with a given vector */
519 case PCITOOL_DEVICE_SET_INTR:
520 rval = pcitool_set_intr(dip, arg, mode);
521 break;
522
523 case PCITOOL_DEVICE_GET_INTR:
524 rval = pcitool_get_intr(dip, arg, mode);
525 break;
526
527 case PCITOOL_SYSTEM_INTR_INFO:
528 rval = pcitool_intr_info(dip, arg, mode);
529 break;
530
531 default:
532 rval = ENOTSUP;
533 }
534
535 return (rval);
536 }
537
538 /*
539 * Perform register accesses on the nexus device itself.
540 * No explicit PCI nexus device for X86, so not applicable.
541 */
542
543 /*ARGSUSED*/
544 int
pcitool_bus_reg_ops(dev_info_t * dip,void * arg,int cmd,int mode)545 pcitool_bus_reg_ops(dev_info_t *dip, void *arg, int cmd, int mode)
546 {
547 return (ENOTSUP);
548 }
549
550 /* Swap endianness. */
551 static uint64_t
pcitool_swap_endian(uint64_t data,int size)552 pcitool_swap_endian(uint64_t data, int size)
553 {
554 typedef union {
555 uint64_t data64;
556 uint8_t data8[8];
557 } data_split_t;
558
559 data_split_t orig_data;
560 data_split_t returned_data;
561 int i;
562
563 orig_data.data64 = data;
564 returned_data.data64 = 0;
565
566 for (i = 0; i < size; i++) {
567 returned_data.data8[i] = orig_data.data8[size - 1 - i];
568 }
569
570 return (returned_data.data64);
571 }
572
573 /*
574 * A note about ontrap handling:
575 *
576 * X86 systems on which this module was tested return FFs instead of bus errors
577 * when accessing devices with invalid addresses. Ontrap handling, which
578 * gracefully handles kernel bus errors, is installed anyway for I/O and mem
579 * space accessing (not for pci config space), in case future X86 platforms
580 * require it.
581 */
582
583 /* Access device. prg is modified. */
584 static int
pcitool_cfg_access(pcitool_reg_t * prg,boolean_t write_flag,boolean_t io_access)585 pcitool_cfg_access(pcitool_reg_t *prg, boolean_t write_flag,
586 boolean_t io_access)
587 {
588 int size = PCITOOL_ACC_ATTR_SIZE(prg->acc_attr);
589 boolean_t big_endian = PCITOOL_ACC_IS_BIG_ENDIAN(prg->acc_attr);
590 int rval = SUCCESS;
591 uint64_t local_data;
592 pci_cfgacc_req_t req;
593 uint32_t max_offset;
594
595 if ((size <= 0) || (size > 8) || !ISP2(size)) {
596 prg->status = PCITOOL_INVALID_SIZE;
597 return (ENOTSUP);
598 }
599
600 /*
601 * NOTE: there is no way to verify whether or not the address is
602 * valid other than that it is within the maximum offset. The
603 * put functions return void and the get functions return -1 on error.
604 */
605
606 if (io_access)
607 max_offset = pci_iocfg_max_offset;
608 else
609 max_offset = 0xFFF;
610 if (prg->offset + size - 1 > max_offset) {
611 prg->status = PCITOOL_INVALID_ADDRESS;
612 return (ENOTSUP);
613 }
614
615 prg->status = PCITOOL_SUCCESS;
616
617 req.rcdip = NULL;
618 req.bdf = PCI_GETBDF(prg->bus_no, prg->dev_no, prg->func_no);
619 req.offset = prg->offset;
620 req.size = size;
621 req.write = write_flag;
622 req.ioacc = io_access;
623 if (write_flag) {
624 if (big_endian) {
625 local_data = pcitool_swap_endian(prg->data, size);
626 } else {
627 local_data = prg->data;
628 }
629 VAL64(&req) = local_data;
630 pci_cfgacc_acc(&req);
631 } else {
632 pci_cfgacc_acc(&req);
633 switch (size) {
634 case 1:
635 local_data = VAL8(&req);
636 break;
637 case 2:
638 local_data = VAL16(&req);
639 break;
640 case 4:
641 local_data = VAL32(&req);
642 break;
643 case 8:
644 local_data = VAL64(&req);
645 break;
646 default:
647 prg->status = PCITOOL_INVALID_ADDRESS;
648 return (ENOTSUP);
649 }
650 if (big_endian) {
651 prg->data =
652 pcitool_swap_endian(local_data, size);
653 } else {
654 prg->data = local_data;
655 }
656 }
657 /*
658 * Check if legacy I/O config access is used, in which case the valid
659 * range varies with the I/O space mechanism used.
660 */
661 if (req.ioacc && (prg->offset + size - 1 > pci_iocfg_max_offset)) {
662 prg->status = PCITOOL_INVALID_ADDRESS;
663 return (ENOTSUP);
664 }
665
666 /* Set phys_addr only if MMIO is used */
667 prg->phys_addr = 0;
668 if (!req.ioacc && mcfg_mem_base != 0) {
669 prg->phys_addr = mcfg_mem_base + prg->offset +
670 ((prg->bus_no << PCIEX_REG_BUS_SHIFT) |
671 (prg->dev_no << PCIEX_REG_DEV_SHIFT) |
672 (prg->func_no << PCIEX_REG_FUNC_SHIFT));
673 }
674
675 return (rval);
676 }
677
678 static int
pcitool_io_access(pcitool_reg_t * prg,boolean_t write_flag)679 pcitool_io_access(pcitool_reg_t *prg, boolean_t write_flag)
680 {
681 int port = (int)prg->phys_addr;
682 size_t size = PCITOOL_ACC_ATTR_SIZE(prg->acc_attr);
683 boolean_t big_endian = PCITOOL_ACC_IS_BIG_ENDIAN(prg->acc_attr);
684 volatile int rval = SUCCESS;
685 on_trap_data_t otd;
686 volatile uint64_t local_data;
687
688
689 /*
690 * on_trap works like setjmp.
691 *
692 * A non-zero return here means on_trap has returned from an error.
693 *
694 * A zero return here means that on_trap has just returned from setup.
695 */
696 if (on_trap(&otd, OT_DATA_ACCESS)) {
697 no_trap();
698 if (pcitool_debug)
699 prom_printf(
700 "pcitool_io_access: on_trap caught an error...\n");
701 prg->status = PCITOOL_INVALID_ADDRESS;
702 return (EFAULT);
703 }
704
705 if (write_flag) {
706
707 if (big_endian) {
708 local_data = pcitool_swap_endian(prg->data, size);
709 } else {
710 local_data = prg->data;
711 }
712
713 if (pcitool_debug)
714 prom_printf("Writing %ld byte(s) to port 0x%x\n",
715 size, port);
716
717 switch (size) {
718 case 1:
719 outb(port, (uint8_t)local_data);
720 break;
721 case 2:
722 outw(port, (uint16_t)local_data);
723 break;
724 case 4:
725 outl(port, (uint32_t)local_data);
726 break;
727 default:
728 rval = ENOTSUP;
729 prg->status = PCITOOL_INVALID_SIZE;
730 break;
731 }
732 } else {
733 if (pcitool_debug)
734 prom_printf("Reading %ld byte(s) from port 0x%x\n",
735 size, port);
736
737 switch (size) {
738 case 1:
739 local_data = inb(port);
740 break;
741 case 2:
742 local_data = inw(port);
743 break;
744 case 4:
745 local_data = inl(port);
746 break;
747 default:
748 rval = ENOTSUP;
749 prg->status = PCITOOL_INVALID_SIZE;
750 break;
751 }
752
753 if (rval == SUCCESS) {
754 if (big_endian) {
755 prg->data =
756 pcitool_swap_endian(local_data, size);
757 } else {
758 prg->data = local_data;
759 }
760 }
761 }
762
763 no_trap();
764 return (rval);
765 }
766
767 static int
pcitool_mem_access(pcitool_reg_t * prg,uint64_t virt_addr,boolean_t write_flag)768 pcitool_mem_access(pcitool_reg_t *prg, uint64_t virt_addr, boolean_t write_flag)
769 {
770 size_t size = PCITOOL_ACC_ATTR_SIZE(prg->acc_attr);
771 boolean_t big_endian = PCITOOL_ACC_IS_BIG_ENDIAN(prg->acc_attr);
772 volatile int rval = DDI_SUCCESS;
773 on_trap_data_t otd;
774 volatile uint64_t local_data;
775
776 /*
777 * on_trap works like setjmp.
778 *
779 * A non-zero return here means on_trap has returned from an error.
780 *
781 * A zero return here means that on_trap has just returned from setup.
782 */
783 if (on_trap(&otd, OT_DATA_ACCESS)) {
784 no_trap();
785 if (pcitool_debug)
786 prom_printf(
787 "pcitool_mem_access: on_trap caught an error...\n");
788 prg->status = PCITOOL_INVALID_ADDRESS;
789 return (EFAULT);
790 }
791
792 if (write_flag) {
793
794 if (big_endian) {
795 local_data = pcitool_swap_endian(prg->data, size);
796 } else {
797 local_data = prg->data;
798 }
799
800 switch (size) {
801 case 1:
802 *((uint8_t *)(uintptr_t)virt_addr) = local_data;
803 break;
804 case 2:
805 *((uint16_t *)(uintptr_t)virt_addr) = local_data;
806 break;
807 case 4:
808 *((uint32_t *)(uintptr_t)virt_addr) = local_data;
809 break;
810 case 8:
811 *((uint64_t *)(uintptr_t)virt_addr) = local_data;
812 break;
813 default:
814 rval = ENOTSUP;
815 prg->status = PCITOOL_INVALID_SIZE;
816 break;
817 }
818 } else {
819 switch (size) {
820 case 1:
821 local_data = *((uint8_t *)(uintptr_t)virt_addr);
822 break;
823 case 2:
824 local_data = *((uint16_t *)(uintptr_t)virt_addr);
825 break;
826 case 4:
827 local_data = *((uint32_t *)(uintptr_t)virt_addr);
828 break;
829 case 8:
830 local_data = *((uint64_t *)(uintptr_t)virt_addr);
831 break;
832 default:
833 rval = ENOTSUP;
834 prg->status = PCITOOL_INVALID_SIZE;
835 break;
836 }
837
838 if (rval == SUCCESS) {
839 if (big_endian) {
840 prg->data =
841 pcitool_swap_endian(local_data, size);
842 } else {
843 prg->data = local_data;
844 }
845 }
846 }
847
848 no_trap();
849 return (rval);
850 }
851
852 /*
853 * Map up to 2 pages which contain the address we want to access.
854 *
855 * Mapping should span no more than 8 bytes. With X86 it is possible for an
856 * 8 byte value to start on a 4 byte boundary, so it can cross a page boundary.
857 * We'll never have to map more than two pages.
858 */
859
860 static uint64_t
pcitool_map(uint64_t phys_addr,size_t size,size_t * num_pages)861 pcitool_map(uint64_t phys_addr, size_t size, size_t *num_pages)
862 {
863
864 uint64_t page_base = phys_addr & ~MMU_PAGEOFFSET;
865 uint64_t offset = phys_addr & MMU_PAGEOFFSET;
866 void *virt_base;
867 uint64_t returned_addr;
868 pfn_t pfn;
869
870 if (pcitool_debug)
871 prom_printf("pcitool_map: Called with PA:0x%p\n",
872 (void *)(uintptr_t)phys_addr);
873
874 *num_pages = 1;
875
876 /* Desired mapping would span more than two pages. */
877 if ((offset + size) > (MMU_PAGESIZE * 2)) {
878 if (pcitool_debug)
879 prom_printf("boundary violation: "
880 "offset:0x%" PRIx64 ", size:%ld, pagesize:0x%lx\n",
881 offset, (uintptr_t)size, (uintptr_t)MMU_PAGESIZE);
882 return (0);
883
884 } else if ((offset + size) > MMU_PAGESIZE) {
885 (*num_pages)++;
886 }
887
888 /* Get page(s) of virtual space. */
889 virt_base = vmem_alloc(heap_arena, ptob(*num_pages), VM_NOSLEEP);
890 if (virt_base == NULL) {
891 if (pcitool_debug)
892 prom_printf("Couldn't get virtual base address.\n");
893 return (0);
894 }
895
896 if (pcitool_debug)
897 prom_printf("Got base virtual address:0x%p\n", virt_base);
898
899 #ifdef __xpv
900 /*
901 * We should only get here if we are dom0.
902 * We're using a real device so we need to translate the MA to a PFN.
903 */
904 ASSERT(DOMAIN_IS_INITDOMAIN(xen_info));
905 pfn = xen_assign_pfn(mmu_btop(page_base));
906 #else
907 pfn = btop(page_base);
908 #endif
909
910 /* Now map the allocated virtual space to the physical address. */
911 hat_devload(kas.a_hat, virt_base, mmu_ptob(*num_pages), pfn,
912 PROT_READ | PROT_WRITE | HAT_STRICTORDER,
913 HAT_LOAD_LOCK);
914
915 returned_addr = ((uintptr_t)(virt_base)) + offset;
916
917 if (pcitool_debug)
918 prom_printf("pcitool_map: returning VA:0x%p\n",
919 (void *)(uintptr_t)returned_addr);
920
921 return (returned_addr);
922 }
923
924 /* Unmap the mapped page(s). */
925 static void
pcitool_unmap(uint64_t virt_addr,size_t num_pages)926 pcitool_unmap(uint64_t virt_addr, size_t num_pages)
927 {
928 void *base_virt_addr = (void *)(uintptr_t)(virt_addr & ~MMU_PAGEOFFSET);
929
930 hat_unload(kas.a_hat, base_virt_addr, ptob(num_pages),
931 HAT_UNLOAD_UNLOCK);
932 vmem_free(heap_arena, base_virt_addr, ptob(num_pages));
933 }
934
935
936 /* Perform register accesses on PCI leaf devices. */
937 /*ARGSUSED*/
938 int
pcitool_dev_reg_ops(dev_info_t * dip,void * arg,int cmd,int mode)939 pcitool_dev_reg_ops(dev_info_t *dip, void *arg, int cmd, int mode)
940 {
941 boolean_t write_flag = B_FALSE;
942 boolean_t io_access = B_TRUE;
943 int rval = 0;
944 pcitool_reg_t prg;
945 uint8_t size;
946
947 uint64_t base_addr;
948 uint64_t virt_addr;
949 size_t num_virt_pages;
950
951 switch (cmd) {
952 case (PCITOOL_DEVICE_SET_REG):
953 write_flag = B_TRUE;
954
955 /*FALLTHRU*/
956 case (PCITOOL_DEVICE_GET_REG):
957 if (pcitool_debug)
958 prom_printf("pci_dev_reg_ops set/get reg\n");
959 if (ddi_copyin(arg, &prg, sizeof (pcitool_reg_t), mode) !=
960 DDI_SUCCESS) {
961 if (pcitool_debug)
962 prom_printf("Error reading arguments\n");
963 return (EFAULT);
964 }
965
966 if (prg.barnum >= (sizeof (pci_bars) / sizeof (pci_bars[0]))) {
967 prg.status = PCITOOL_OUT_OF_RANGE;
968 rval = EINVAL;
969 goto done_reg;
970 }
971
972 if (pcitool_debug)
973 prom_printf("raw bus:0x%x, dev:0x%x, func:0x%x\n",
974 prg.bus_no, prg.dev_no, prg.func_no);
975 /* Validate address arguments of bus / dev / func */
976 if (((prg.bus_no &
977 (PCI_REG_BUS_M >> PCI_REG_BUS_SHIFT)) !=
978 prg.bus_no) ||
979 ((prg.dev_no &
980 (PCI_REG_DEV_M >> PCI_REG_DEV_SHIFT)) !=
981 prg.dev_no) ||
982 ((prg.func_no &
983 (PCI_REG_FUNC_M >> PCI_REG_FUNC_SHIFT)) !=
984 prg.func_no)) {
985 prg.status = PCITOOL_INVALID_ADDRESS;
986 rval = EINVAL;
987 goto done_reg;
988 }
989
990 size = PCITOOL_ACC_ATTR_SIZE(prg.acc_attr);
991
992 /* Proper config space desired. */
993 if (prg.barnum == 0) {
994
995 if (pcitool_debug)
996 prom_printf(
997 "config access: offset:0x%" PRIx64 ", "
998 "phys_addr:0x%" PRIx64 "\n",
999 prg.offset, prg.phys_addr);
1000
1001 if (prg.offset >= max_cfg_size) {
1002 prg.status = PCITOOL_OUT_OF_RANGE;
1003 rval = EINVAL;
1004 goto done_reg;
1005 }
1006 if (max_cfg_size == PCIE_CONF_HDR_SIZE)
1007 io_access = B_FALSE;
1008
1009 rval = pcitool_cfg_access(&prg, write_flag, io_access);
1010 if (pcitool_debug)
1011 prom_printf(
1012 "config access: data:0x%" PRIx64 "\n",
1013 prg.data);
1014
1015 /* IO/ MEM/ MEM64 space. */
1016 } else {
1017
1018 pcitool_reg_t prg2;
1019 bcopy(&prg, &prg2, sizeof (pcitool_reg_t));
1020
1021 /*
1022 * Translate BAR number into offset of the BAR in
1023 * the device's config space.
1024 */
1025 prg2.offset = pci_bars[prg2.barnum];
1026 prg2.acc_attr =
1027 PCITOOL_ACC_ATTR_SIZE_4 | PCITOOL_ACC_ATTR_ENDN_LTL;
1028
1029 if (pcitool_debug)
1030 prom_printf(
1031 "barnum:%d, bar_offset:0x%" PRIx64 "\n",
1032 prg2.barnum, prg2.offset);
1033 /*
1034 * Get Bus Address Register (BAR) from config space.
1035 * prg2.offset is the offset into config space of the
1036 * BAR desired. prg.status is modified on error.
1037 */
1038 rval = pcitool_cfg_access(&prg2, B_FALSE, B_TRUE);
1039 if (rval != SUCCESS) {
1040 if (pcitool_debug)
1041 prom_printf("BAR access failed\n");
1042 prg.status = prg2.status;
1043 goto done_reg;
1044 }
1045 /*
1046 * Reference proper PCI space based on the BAR.
1047 * If 64 bit MEM space, need to load other half of the
1048 * BAR first.
1049 */
1050
1051 if (pcitool_debug)
1052 prom_printf("bar returned is 0x%" PRIx64 "\n",
1053 prg2.data);
1054 if (!prg2.data) {
1055 if (pcitool_debug)
1056 prom_printf("BAR data == 0\n");
1057 rval = EINVAL;
1058 prg.status = PCITOOL_INVALID_ADDRESS;
1059 goto done_reg;
1060 }
1061 if (prg2.data == 0xffffffff) {
1062 if (pcitool_debug)
1063 prom_printf("BAR data == -1\n");
1064 rval = EINVAL;
1065 prg.status = PCITOOL_INVALID_ADDRESS;
1066 goto done_reg;
1067 }
1068
1069 /*
1070 * BAR has bits saying this space is IO space, unless
1071 * this is the ROM address register.
1072 */
1073 if (((PCI_BASE_SPACE_M & prg2.data) ==
1074 PCI_BASE_SPACE_IO) &&
1075 (prg2.offset != PCI_CONF_ROM)) {
1076 if (pcitool_debug)
1077 prom_printf("IO space\n");
1078
1079 prg2.data &= PCI_BASE_IO_ADDR_M;
1080 prg.phys_addr = prg2.data + prg.offset;
1081
1082 rval = pcitool_io_access(&prg, write_flag);
1083 if ((rval != SUCCESS) && (pcitool_debug))
1084 prom_printf("IO access failed\n");
1085
1086 goto done_reg;
1087
1088
1089 /*
1090 * BAR has bits saying this space is 64 bit memory
1091 * space, unless this is the ROM address register.
1092 *
1093 * The 64 bit address stored in two BAR cells is not
1094 * necessarily aligned on an 8-byte boundary.
1095 * Need to keep the first 4 bytes read,
1096 * and do a separate read of the high 4 bytes.
1097 */
1098
1099 } else if ((PCI_BASE_TYPE_ALL & prg2.data) &&
1100 (prg2.offset != PCI_CONF_ROM)) {
1101
1102 uint32_t low_bytes =
1103 (uint32_t)(prg2.data & ~PCI_BASE_TYPE_ALL);
1104
1105 /*
1106 * Don't try to read the next 4 bytes
1107 * past the end of BARs.
1108 */
1109 if (prg2.offset >= PCI_CONF_BASE5) {
1110 prg.status = PCITOOL_OUT_OF_RANGE;
1111 rval = EIO;
1112 goto done_reg;
1113 }
1114
1115 /*
1116 * Access device.
1117 * prg2.status is modified on error.
1118 */
1119 prg2.offset += 4;
1120 rval = pcitool_cfg_access(&prg2,
1121 B_FALSE, B_TRUE);
1122 if (rval != SUCCESS) {
1123 prg.status = prg2.status;
1124 goto done_reg;
1125 }
1126
1127 if (prg2.data == 0xffffffff) {
1128 prg.status = PCITOOL_INVALID_ADDRESS;
1129 prg.status = EFAULT;
1130 goto done_reg;
1131 }
1132
1133 prg2.data = (prg2.data << 32) + low_bytes;
1134 if (pcitool_debug)
1135 prom_printf(
1136 "64 bit mem space. "
1137 "64-bit bar is 0x%" PRIx64 "\n",
1138 prg2.data);
1139
1140 /* Mem32 space, including ROM */
1141 } else {
1142
1143 if (prg2.offset == PCI_CONF_ROM) {
1144 if (pcitool_debug)
1145 prom_printf(
1146 "Additional ROM "
1147 "checking\n");
1148 /* Can't write to ROM */
1149 if (write_flag) {
1150 prg.status = PCITOOL_ROM_WRITE;
1151 rval = EIO;
1152 goto done_reg;
1153
1154 /* ROM disabled for reading */
1155 } else if (!(prg2.data & 0x00000001)) {
1156 prg.status =
1157 PCITOOL_ROM_DISABLED;
1158 rval = EIO;
1159 goto done_reg;
1160 }
1161 }
1162
1163 if (pcitool_debug)
1164 prom_printf("32 bit mem space\n");
1165 }
1166
1167 /* Common code for all IO/MEM range spaces. */
1168
1169 base_addr = prg2.data;
1170 if (pcitool_debug)
1171 prom_printf(
1172 "addr portion of bar is 0x%" PRIx64 ", "
1173 "base=0x%" PRIx64 ", "
1174 "offset:0x%" PRIx64 "\n",
1175 prg2.data, base_addr, prg.offset);
1176 /*
1177 * Use offset provided by caller to index into
1178 * desired space, then access.
1179 * Note that prg.status is modified on error.
1180 */
1181 prg.phys_addr = base_addr + prg.offset;
1182
1183 virt_addr = pcitool_map(prg.phys_addr, size,
1184 &num_virt_pages);
1185 if (virt_addr == 0) {
1186 prg.status = PCITOOL_IO_ERROR;
1187 rval = EIO;
1188 goto done_reg;
1189 }
1190
1191 rval = pcitool_mem_access(&prg, virt_addr, write_flag);
1192 pcitool_unmap(virt_addr, num_virt_pages);
1193 }
1194 done_reg:
1195 prg.drvr_version = PCITOOL_VERSION;
1196 if (ddi_copyout(&prg, arg, sizeof (pcitool_reg_t), mode) !=
1197 DDI_SUCCESS) {
1198 if (pcitool_debug)
1199 prom_printf("Error returning arguments.\n");
1200 rval = EFAULT;
1201 }
1202 break;
1203 default:
1204 rval = ENOTTY;
1205 break;
1206 }
1207 return (rval);
1208 }
1209