13a634bfcSVikram Hegde /*
23a634bfcSVikram Hegde * CDDL HEADER START
33a634bfcSVikram Hegde *
43a634bfcSVikram Hegde * The contents of this file are subject to the terms of the
53a634bfcSVikram Hegde * Common Development and Distribution License (the "License").
63a634bfcSVikram Hegde * You may not use this file except in compliance with the License.
73a634bfcSVikram Hegde *
83a634bfcSVikram Hegde * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
93a634bfcSVikram Hegde * or http://www.opensolaris.org/os/licensing.
103a634bfcSVikram Hegde * See the License for the specific language governing permissions
113a634bfcSVikram Hegde * and limitations under the License.
123a634bfcSVikram Hegde *
133a634bfcSVikram Hegde * When distributing Covered Code, include this CDDL HEADER in each
143a634bfcSVikram Hegde * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
153a634bfcSVikram Hegde * If applicable, add the following below this CDDL HEADER, with the
163a634bfcSVikram Hegde * fields enclosed by brackets "[]" replaced with your own identifying
173a634bfcSVikram Hegde * information: Portions Copyright [yyyy] [name of copyright owner]
183a634bfcSVikram Hegde *
193a634bfcSVikram Hegde * CDDL HEADER END
203a634bfcSVikram Hegde */
213a634bfcSVikram Hegde /*
22c94adbf9SFrank Van Der Linden * Portions Copyright (c) 2010, Oracle and/or its affiliates.
23c94adbf9SFrank Van Der Linden * All rights reserved.
243a634bfcSVikram Hegde */
253a634bfcSVikram Hegde
263a634bfcSVikram Hegde /*
273a634bfcSVikram Hegde * immu_regs.c - File that operates on a IMMU unit's regsiters
283a634bfcSVikram Hegde */
293a634bfcSVikram Hegde #include <sys/dditypes.h>
303a634bfcSVikram Hegde #include <sys/ddi.h>
313a634bfcSVikram Hegde #include <sys/archsystm.h>
323a634bfcSVikram Hegde #include <sys/x86_archext.h>
333a634bfcSVikram Hegde #include <sys/spl.h>
34e03dceedSVikram Hegde #include <sys/sysmacros.h>
353a634bfcSVikram Hegde #include <sys/immu.h>
3650200e77SFrank Van Der Linden #include <sys/cpu.h>
373a634bfcSVikram Hegde
383a634bfcSVikram Hegde #define get_reg32(immu, offset) ddi_get32((immu)->immu_regs_handle, \
393a634bfcSVikram Hegde (uint32_t *)(immu->immu_regs_addr + (offset)))
403a634bfcSVikram Hegde #define get_reg64(immu, offset) ddi_get64((immu)->immu_regs_handle, \
413a634bfcSVikram Hegde (uint64_t *)(immu->immu_regs_addr + (offset)))
423a634bfcSVikram Hegde #define put_reg32(immu, offset, val) ddi_put32\
433a634bfcSVikram Hegde ((immu)->immu_regs_handle, \
443a634bfcSVikram Hegde (uint32_t *)(immu->immu_regs_addr + (offset)), val)
453a634bfcSVikram Hegde #define put_reg64(immu, offset, val) ddi_put64\
463a634bfcSVikram Hegde ((immu)->immu_regs_handle, \
473a634bfcSVikram Hegde (uint64_t *)(immu->immu_regs_addr + (offset)), val)
483a634bfcSVikram Hegde
4950200e77SFrank Van Der Linden static void immu_regs_inv_wait(immu_inv_wait_t *iwp);
5050200e77SFrank Van Der Linden
51c94adbf9SFrank Van Der Linden struct immu_flushops immu_regs_flushops = {
52c94adbf9SFrank Van Der Linden immu_regs_context_fsi,
53c94adbf9SFrank Van Der Linden immu_regs_context_dsi,
54c94adbf9SFrank Van Der Linden immu_regs_context_gbl,
55c94adbf9SFrank Van Der Linden immu_regs_iotlb_psi,
56c94adbf9SFrank Van Der Linden immu_regs_iotlb_dsi,
5750200e77SFrank Van Der Linden immu_regs_iotlb_gbl,
5850200e77SFrank Van Der Linden immu_regs_inv_wait
59c94adbf9SFrank Van Der Linden };
60c94adbf9SFrank Van Der Linden
613a634bfcSVikram Hegde /*
623a634bfcSVikram Hegde * wait max 60s for the hardware completion
633a634bfcSVikram Hegde */
643a634bfcSVikram Hegde #define IMMU_MAX_WAIT_TIME 60000000
653a634bfcSVikram Hegde #define wait_completion(immu, offset, getf, completion, status) \
663a634bfcSVikram Hegde { \
673a634bfcSVikram Hegde clock_t stick = ddi_get_lbolt(); \
683a634bfcSVikram Hegde clock_t ntick; \
693a634bfcSVikram Hegde _NOTE(CONSTCOND) \
703a634bfcSVikram Hegde while (1) { \
713a634bfcSVikram Hegde status = getf(immu, offset); \
723a634bfcSVikram Hegde ntick = ddi_get_lbolt(); \
733a634bfcSVikram Hegde if (completion) { \
743a634bfcSVikram Hegde break; \
753a634bfcSVikram Hegde } \
763a634bfcSVikram Hegde if (ntick - stick >= drv_usectohz(IMMU_MAX_WAIT_TIME)) { \
773a634bfcSVikram Hegde ddi_err(DER_PANIC, NULL, \
783a634bfcSVikram Hegde "immu wait completion time out"); \
793a634bfcSVikram Hegde /*NOTREACHED*/ \
803a634bfcSVikram Hegde } else { \
8150200e77SFrank Van Der Linden ht_pause();\
823a634bfcSVikram Hegde }\
833a634bfcSVikram Hegde }\
843a634bfcSVikram Hegde }
853a634bfcSVikram Hegde
863a634bfcSVikram Hegde static ddi_device_acc_attr_t immu_regs_attr = {
873a634bfcSVikram Hegde DDI_DEVICE_ATTR_V0,
883a634bfcSVikram Hegde DDI_NEVERSWAP_ACC,
893a634bfcSVikram Hegde DDI_STRICTORDER_ACC,
903a634bfcSVikram Hegde };
913a634bfcSVikram Hegde
923a634bfcSVikram Hegde /*
933a634bfcSVikram Hegde * iotlb_flush()
943a634bfcSVikram Hegde * flush the iotlb cache
953a634bfcSVikram Hegde */
963a634bfcSVikram Hegde static void
iotlb_flush(immu_t * immu,uint_t domain_id,uint64_t addr,uint_t am,uint_t hint,immu_iotlb_inv_t type)973a634bfcSVikram Hegde iotlb_flush(immu_t *immu, uint_t domain_id,
983a634bfcSVikram Hegde uint64_t addr, uint_t am, uint_t hint, immu_iotlb_inv_t type)
993a634bfcSVikram Hegde {
1003a634bfcSVikram Hegde uint64_t command = 0, iva = 0;
1013a634bfcSVikram Hegde uint_t iva_offset, iotlb_offset;
1023a634bfcSVikram Hegde uint64_t status = 0;
1033a634bfcSVikram Hegde
1043a634bfcSVikram Hegde /* no lock needed since cap and excap fields are RDONLY */
1053a634bfcSVikram Hegde iva_offset = IMMU_ECAP_GET_IRO(immu->immu_regs_excap);
1063a634bfcSVikram Hegde iotlb_offset = iva_offset + 8;
1073a634bfcSVikram Hegde
1083a634bfcSVikram Hegde /*
1093a634bfcSVikram Hegde * prepare drain read/write command
1103a634bfcSVikram Hegde */
1113a634bfcSVikram Hegde if (IMMU_CAP_GET_DWD(immu->immu_regs_cap)) {
1123a634bfcSVikram Hegde command |= TLB_INV_DRAIN_WRITE;
1133a634bfcSVikram Hegde }
1143a634bfcSVikram Hegde
1153a634bfcSVikram Hegde if (IMMU_CAP_GET_DRD(immu->immu_regs_cap)) {
1163a634bfcSVikram Hegde command |= TLB_INV_DRAIN_READ;
1173a634bfcSVikram Hegde }
1183a634bfcSVikram Hegde
1193a634bfcSVikram Hegde /*
1203a634bfcSVikram Hegde * if the hardward doesn't support page selective invalidation, we
1213a634bfcSVikram Hegde * will use domain type. Otherwise, use global type
1223a634bfcSVikram Hegde */
1233a634bfcSVikram Hegde switch (type) {
1243a634bfcSVikram Hegde case IOTLB_PSI:
1253a634bfcSVikram Hegde command |= TLB_INV_PAGE | TLB_INV_IVT |
1263a634bfcSVikram Hegde TLB_INV_DID(domain_id);
1273a634bfcSVikram Hegde iva = addr | am | TLB_IVA_HINT(hint);
1283a634bfcSVikram Hegde break;
1293a634bfcSVikram Hegde case IOTLB_DSI:
1303a634bfcSVikram Hegde command |= TLB_INV_DOMAIN | TLB_INV_IVT |
1313a634bfcSVikram Hegde TLB_INV_DID(domain_id);
1323a634bfcSVikram Hegde break;
1333a634bfcSVikram Hegde case IOTLB_GLOBAL:
1343a634bfcSVikram Hegde command |= TLB_INV_GLOBAL | TLB_INV_IVT;
1353a634bfcSVikram Hegde break;
1363a634bfcSVikram Hegde default:
1373a634bfcSVikram Hegde ddi_err(DER_MODE, NULL, "%s: incorrect iotlb flush type",
1383a634bfcSVikram Hegde immu->immu_name);
1393a634bfcSVikram Hegde return;
1403a634bfcSVikram Hegde }
1413a634bfcSVikram Hegde
1423a634bfcSVikram Hegde if (iva)
1433a634bfcSVikram Hegde put_reg64(immu, iva_offset, iva);
1443a634bfcSVikram Hegde put_reg64(immu, iotlb_offset, command);
1453a634bfcSVikram Hegde wait_completion(immu, iotlb_offset, get_reg64,
1463a634bfcSVikram Hegde (!(status & TLB_INV_IVT)), status);
1473a634bfcSVikram Hegde }
1483a634bfcSVikram Hegde
1493a634bfcSVikram Hegde /*
150c94adbf9SFrank Van Der Linden * immu_regs_iotlb_psi()
1513a634bfcSVikram Hegde * iotlb page specific invalidation
1523a634bfcSVikram Hegde */
15350200e77SFrank Van Der Linden /*ARGSUSED*/
154c94adbf9SFrank Van Der Linden void
immu_regs_iotlb_psi(immu_t * immu,uint_t did,uint64_t dvma,uint_t snpages,uint_t hint,immu_inv_wait_t * iwp)155c94adbf9SFrank Van Der Linden immu_regs_iotlb_psi(immu_t *immu, uint_t did, uint64_t dvma, uint_t snpages,
15650200e77SFrank Van Der Linden uint_t hint, immu_inv_wait_t *iwp)
1573a634bfcSVikram Hegde {
158e03dceedSVikram Hegde int dvma_am;
159e03dceedSVikram Hegde int npg_am;
160e03dceedSVikram Hegde int max_am;
161e03dceedSVikram Hegde int am;
162e03dceedSVikram Hegde uint64_t align;
163e03dceedSVikram Hegde int npages_left;
164e03dceedSVikram Hegde int npages;
165e03dceedSVikram Hegde int i;
166e03dceedSVikram Hegde
167c94adbf9SFrank Van Der Linden if (!IMMU_CAP_GET_PSI(immu->immu_regs_cap)) {
16850200e77SFrank Van Der Linden immu_regs_iotlb_dsi(immu, did, iwp);
169c94adbf9SFrank Van Der Linden return;
170c94adbf9SFrank Van Der Linden }
171c94adbf9SFrank Van Der Linden
172e03dceedSVikram Hegde max_am = IMMU_CAP_GET_MAMV(immu->immu_regs_cap);
1733a634bfcSVikram Hegde
1743a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock));
1753a634bfcSVikram Hegde
176e03dceedSVikram Hegde npages_left = snpages;
177e03dceedSVikram Hegde for (i = 0; i < immu_flush_gran && npages_left > 0; i++) {
178e03dceedSVikram Hegde /* First calculate alignment of DVMA */
179e03dceedSVikram Hegde
180e03dceedSVikram Hegde if (dvma == 0) {
181e03dceedSVikram Hegde dvma_am = max_am;
182e03dceedSVikram Hegde } else {
183e03dceedSVikram Hegde for (align = (1 << 12), dvma_am = 1;
184e03dceedSVikram Hegde (dvma & align) == 0; align <<= 1, dvma_am++)
1853a634bfcSVikram Hegde ;
186e03dceedSVikram Hegde dvma_am--;
1873a634bfcSVikram Hegde }
188e03dceedSVikram Hegde
189e03dceedSVikram Hegde /* Calculate the npg_am */
190e03dceedSVikram Hegde npages = npages_left;
191e03dceedSVikram Hegde for (npg_am = 0, npages >>= 1; npages; npages >>= 1, npg_am++)
192e03dceedSVikram Hegde ;
193e03dceedSVikram Hegde
194e03dceedSVikram Hegde am = MIN(max_am, MIN(dvma_am, npg_am));
195e03dceedSVikram Hegde
196e03dceedSVikram Hegde iotlb_flush(immu, did, dvma, am, hint, IOTLB_PSI);
197e03dceedSVikram Hegde
198e03dceedSVikram Hegde npages = (1 << am);
199e03dceedSVikram Hegde npages_left -= npages;
200e03dceedSVikram Hegde dvma += (npages * IMMU_PAGESIZE);
2013a634bfcSVikram Hegde }
2023a634bfcSVikram Hegde
203e03dceedSVikram Hegde if (npages_left) {
204e03dceedSVikram Hegde iotlb_flush(immu, did, 0, 0, 0, IOTLB_DSI);
205e03dceedSVikram Hegde }
2063a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock));
2073a634bfcSVikram Hegde }
2083a634bfcSVikram Hegde
2093a634bfcSVikram Hegde /*
210c94adbf9SFrank Van Der Linden * immu_regs_iotlb_dsi()
2113a634bfcSVikram Hegde * domain specific invalidation
2123a634bfcSVikram Hegde */
21350200e77SFrank Van Der Linden /*ARGSUSED*/
214c94adbf9SFrank Van Der Linden void
immu_regs_iotlb_dsi(immu_t * immu,uint_t domain_id,immu_inv_wait_t * iwp)21550200e77SFrank Van Der Linden immu_regs_iotlb_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp)
2163a634bfcSVikram Hegde {
2173a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock));
2183a634bfcSVikram Hegde iotlb_flush(immu, domain_id, 0, 0, 0, IOTLB_DSI);
2193a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock));
2203a634bfcSVikram Hegde }
2213a634bfcSVikram Hegde
2223a634bfcSVikram Hegde /*
223c94adbf9SFrank Van Der Linden * immu_regs_iotlb_gbl()
2243a634bfcSVikram Hegde * global iotlb invalidation
2253a634bfcSVikram Hegde */
22650200e77SFrank Van Der Linden /*ARGSUSED*/
227c94adbf9SFrank Van Der Linden void
immu_regs_iotlb_gbl(immu_t * immu,immu_inv_wait_t * iwp)22850200e77SFrank Van Der Linden immu_regs_iotlb_gbl(immu_t *immu, immu_inv_wait_t *iwp)
2293a634bfcSVikram Hegde {
2303a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock));
2313a634bfcSVikram Hegde iotlb_flush(immu, 0, 0, 0, 0, IOTLB_GLOBAL);
2323a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock));
2333a634bfcSVikram Hegde }
2343a634bfcSVikram Hegde
2353a634bfcSVikram Hegde
2363a634bfcSVikram Hegde static int
gaw2agaw(int gaw)2373a634bfcSVikram Hegde gaw2agaw(int gaw)
2383a634bfcSVikram Hegde {
2393a634bfcSVikram Hegde int r, agaw;
2403a634bfcSVikram Hegde
2413a634bfcSVikram Hegde r = (gaw - 12) % 9;
2423a634bfcSVikram Hegde
2433a634bfcSVikram Hegde if (r == 0)
2443a634bfcSVikram Hegde agaw = gaw;
2453a634bfcSVikram Hegde else
2463a634bfcSVikram Hegde agaw = gaw + 9 - r;
2473a634bfcSVikram Hegde
2483a634bfcSVikram Hegde if (agaw > 64)
2493a634bfcSVikram Hegde agaw = 64;
2503a634bfcSVikram Hegde
2513a634bfcSVikram Hegde return (agaw);
2523a634bfcSVikram Hegde }
2533a634bfcSVikram Hegde
2543a634bfcSVikram Hegde /*
2553a634bfcSVikram Hegde * set_immu_agaw()
256*7fd1b424SToomas Soome * calculate agaw for a IOMMU unit
2573a634bfcSVikram Hegde */
2583a634bfcSVikram Hegde static int
set_agaw(immu_t * immu)2593a634bfcSVikram Hegde set_agaw(immu_t *immu)
2603a634bfcSVikram Hegde {
2613a634bfcSVikram Hegde int mgaw, magaw, agaw;
2623a634bfcSVikram Hegde uint_t bitpos;
2633a634bfcSVikram Hegde int max_sagaw_mask, sagaw_mask, mask;
2643a634bfcSVikram Hegde int nlevels;
2653a634bfcSVikram Hegde
2663a634bfcSVikram Hegde /*
2673a634bfcSVikram Hegde * mgaw is the maximum guest address width.
2683a634bfcSVikram Hegde * Addresses above this value will be
2693a634bfcSVikram Hegde * blocked by the IOMMU unit.
2703a634bfcSVikram Hegde * sagaw is a bitmask that lists all the
2713a634bfcSVikram Hegde * AGAWs supported by this IOMMU unit.
2723a634bfcSVikram Hegde */
2733a634bfcSVikram Hegde mgaw = IMMU_CAP_MGAW(immu->immu_regs_cap);
2743a634bfcSVikram Hegde sagaw_mask = IMMU_CAP_SAGAW(immu->immu_regs_cap);
2753a634bfcSVikram Hegde
2763a634bfcSVikram Hegde magaw = gaw2agaw(mgaw);
2773a634bfcSVikram Hegde
2783a634bfcSVikram Hegde /*
2793a634bfcSVikram Hegde * Get bitpos corresponding to
2803a634bfcSVikram Hegde * magaw
2813a634bfcSVikram Hegde */
2823a634bfcSVikram Hegde
2833a634bfcSVikram Hegde /*
2843a634bfcSVikram Hegde * Maximum SAGAW is specified by
2853a634bfcSVikram Hegde * Vt-d spec.
2863a634bfcSVikram Hegde */
2873a634bfcSVikram Hegde max_sagaw_mask = ((1 << 5) - 1);
2883a634bfcSVikram Hegde
2893a634bfcSVikram Hegde if (sagaw_mask > max_sagaw_mask) {
2903a634bfcSVikram Hegde ddi_err(DER_WARN, NULL, "%s: SAGAW bitmask (%x) "
2913a634bfcSVikram Hegde "is larger than maximu SAGAW bitmask "
2923a634bfcSVikram Hegde "(%x) specified by Intel Vt-d spec",
2933a634bfcSVikram Hegde immu->immu_name, sagaw_mask, max_sagaw_mask);
2943a634bfcSVikram Hegde return (DDI_FAILURE);
2953a634bfcSVikram Hegde }
2963a634bfcSVikram Hegde
2973a634bfcSVikram Hegde /*
2983a634bfcSVikram Hegde * Find a supported AGAW <= magaw
2993a634bfcSVikram Hegde *
3003a634bfcSVikram Hegde * sagaw_mask bitpos AGAW (bits) nlevels
3013a634bfcSVikram Hegde * ==============================================
3023a634bfcSVikram Hegde * 0 0 0 0 1 0 30 2
3033a634bfcSVikram Hegde * 0 0 0 1 0 1 39 3
3043a634bfcSVikram Hegde * 0 0 1 0 0 2 48 4
3053a634bfcSVikram Hegde * 0 1 0 0 0 3 57 5
3063a634bfcSVikram Hegde * 1 0 0 0 0 4 64(66) 6
3073a634bfcSVikram Hegde */
3083a634bfcSVikram Hegde mask = 1;
3093a634bfcSVikram Hegde nlevels = 0;
3103a634bfcSVikram Hegde agaw = 0;
3113a634bfcSVikram Hegde for (mask = 1, bitpos = 0; bitpos < 5;
3123a634bfcSVikram Hegde bitpos++, mask <<= 1) {
3133a634bfcSVikram Hegde if (mask & sagaw_mask) {
3143a634bfcSVikram Hegde nlevels = bitpos + 2;
3153a634bfcSVikram Hegde agaw = 30 + (bitpos * 9);
3163a634bfcSVikram Hegde }
3173a634bfcSVikram Hegde }
3183a634bfcSVikram Hegde
3193a634bfcSVikram Hegde /* calculated agaw can be > 64 */
3203a634bfcSVikram Hegde agaw = (agaw > 64) ? 64 : agaw;
3213a634bfcSVikram Hegde
3223a634bfcSVikram Hegde if (agaw < 30 || agaw > magaw) {
3233a634bfcSVikram Hegde ddi_err(DER_WARN, NULL, "%s: Calculated AGAW (%d) "
3243a634bfcSVikram Hegde "is outside valid limits [30,%d] specified by Vt-d spec "
3253a634bfcSVikram Hegde "and magaw", immu->immu_name, agaw, magaw);
3263a634bfcSVikram Hegde return (DDI_FAILURE);
3273a634bfcSVikram Hegde }
3283a634bfcSVikram Hegde
3293a634bfcSVikram Hegde if (nlevels < 2 || nlevels > 6) {
3303a634bfcSVikram Hegde ddi_err(DER_WARN, NULL, "%s: Calculated pagetable "
3313a634bfcSVikram Hegde "level (%d) is outside valid limits [2,6]",
3323a634bfcSVikram Hegde immu->immu_name, nlevels);
3333a634bfcSVikram Hegde return (DDI_FAILURE);
3343a634bfcSVikram Hegde }
3353a634bfcSVikram Hegde
3363a634bfcSVikram Hegde ddi_err(DER_LOG, NULL, "Calculated pagetable "
3373a634bfcSVikram Hegde "level (%d), agaw = %d", nlevels, agaw);
3383a634bfcSVikram Hegde
3393a634bfcSVikram Hegde immu->immu_dvma_nlevels = nlevels;
3403a634bfcSVikram Hegde immu->immu_dvma_agaw = agaw;
3413a634bfcSVikram Hegde
3423a634bfcSVikram Hegde return (DDI_SUCCESS);
3433a634bfcSVikram Hegde }
3443a634bfcSVikram Hegde
3453a634bfcSVikram Hegde static int
setup_regs(immu_t * immu)3463a634bfcSVikram Hegde setup_regs(immu_t *immu)
3473a634bfcSVikram Hegde {
3483a634bfcSVikram Hegde int error;
3493a634bfcSVikram Hegde
3503a634bfcSVikram Hegde /*
3513a634bfcSVikram Hegde * This lock may be acquired by the IOMMU interrupt handler
3523a634bfcSVikram Hegde */
3533a634bfcSVikram Hegde mutex_init(&(immu->immu_regs_lock), NULL, MUTEX_DRIVER,
3543a634bfcSVikram Hegde (void *)ipltospl(IMMU_INTR_IPL));
3553a634bfcSVikram Hegde
3563a634bfcSVikram Hegde /*
3573a634bfcSVikram Hegde * map the register address space
3583a634bfcSVikram Hegde */
3593a634bfcSVikram Hegde error = ddi_regs_map_setup(immu->immu_dip, 0,
3603a634bfcSVikram Hegde (caddr_t *)&(immu->immu_regs_addr), (offset_t)0,
3613a634bfcSVikram Hegde (offset_t)IMMU_REGSZ, &immu_regs_attr,
3623a634bfcSVikram Hegde &(immu->immu_regs_handle));
3633a634bfcSVikram Hegde
3643a634bfcSVikram Hegde if (error == DDI_FAILURE) {
3653a634bfcSVikram Hegde ddi_err(DER_WARN, NULL, "%s: Intel IOMMU register map failed",
3663a634bfcSVikram Hegde immu->immu_name);
3673a634bfcSVikram Hegde mutex_destroy(&(immu->immu_regs_lock));
3683a634bfcSVikram Hegde return (DDI_FAILURE);
3693a634bfcSVikram Hegde }
3703a634bfcSVikram Hegde
3713a634bfcSVikram Hegde /*
3723a634bfcSVikram Hegde * get the register value
3733a634bfcSVikram Hegde */
3743a634bfcSVikram Hegde immu->immu_regs_cap = get_reg64(immu, IMMU_REG_CAP);
3753a634bfcSVikram Hegde immu->immu_regs_excap = get_reg64(immu, IMMU_REG_EXCAP);
3763a634bfcSVikram Hegde
3773a634bfcSVikram Hegde /*
3783a634bfcSVikram Hegde * if the hardware access is non-coherent, we need clflush
3793a634bfcSVikram Hegde */
3803a634bfcSVikram Hegde if (IMMU_ECAP_GET_C(immu->immu_regs_excap)) {
3813a634bfcSVikram Hegde immu->immu_dvma_coherent = B_TRUE;
3823a634bfcSVikram Hegde } else {
3833a634bfcSVikram Hegde immu->immu_dvma_coherent = B_FALSE;
3847417cfdeSKuriakose Kuruvilla if (!is_x86_feature(x86_featureset, X86FSET_CLFSH)) {
3853a634bfcSVikram Hegde ddi_err(DER_WARN, NULL,
3863a634bfcSVikram Hegde "immu unit %s can't be enabled due to "
3873a634bfcSVikram Hegde "missing clflush functionality", immu->immu_name);
3883a634bfcSVikram Hegde ddi_regs_map_free(&(immu->immu_regs_handle));
3893a634bfcSVikram Hegde mutex_destroy(&(immu->immu_regs_lock));
3903a634bfcSVikram Hegde return (DDI_FAILURE);
3913a634bfcSVikram Hegde }
3923a634bfcSVikram Hegde }
3933a634bfcSVikram Hegde
394e03dceedSVikram Hegde /* Setup SNP and TM reserved fields */
395e03dceedSVikram Hegde immu->immu_SNP_reserved = immu_regs_is_SNP_reserved(immu);
396e03dceedSVikram Hegde immu->immu_TM_reserved = immu_regs_is_TM_reserved(immu);
397e03dceedSVikram Hegde
39850200e77SFrank Van Der Linden if (IMMU_ECAP_GET_CH(immu->immu_regs_excap) && immu_use_tm)
39950200e77SFrank Van Der Linden immu->immu_ptemask = PDTE_MASK_TM;
40050200e77SFrank Van Der Linden else
40150200e77SFrank Van Der Linden immu->immu_ptemask = 0;
40250200e77SFrank Van Der Linden
4033a634bfcSVikram Hegde /*
4043a634bfcSVikram Hegde * Check for Mobile 4 series chipset
4053a634bfcSVikram Hegde */
4063a634bfcSVikram Hegde if (immu_quirk_mobile4 == B_TRUE &&
4073a634bfcSVikram Hegde !IMMU_CAP_GET_RWBF(immu->immu_regs_cap)) {
4083a634bfcSVikram Hegde ddi_err(DER_LOG, NULL,
4093a634bfcSVikram Hegde "IMMU: Mobile 4 chipset quirk detected. "
4103a634bfcSVikram Hegde "Force-setting RWBF");
4113a634bfcSVikram Hegde IMMU_CAP_SET_RWBF(immu->immu_regs_cap);
4123a634bfcSVikram Hegde }
4133a634bfcSVikram Hegde
4143a634bfcSVikram Hegde /*
4153a634bfcSVikram Hegde * retrieve the maximum number of domains
4163a634bfcSVikram Hegde */
4173a634bfcSVikram Hegde immu->immu_max_domains = IMMU_CAP_ND(immu->immu_regs_cap);
4183a634bfcSVikram Hegde
4193a634bfcSVikram Hegde /*
4203a634bfcSVikram Hegde * calculate the agaw
4213a634bfcSVikram Hegde */
4223a634bfcSVikram Hegde if (set_agaw(immu) != DDI_SUCCESS) {
4233a634bfcSVikram Hegde ddi_regs_map_free(&(immu->immu_regs_handle));
4243a634bfcSVikram Hegde mutex_destroy(&(immu->immu_regs_lock));
4253a634bfcSVikram Hegde return (DDI_FAILURE);
4263a634bfcSVikram Hegde }
4273a634bfcSVikram Hegde immu->immu_regs_cmdval = 0;
4283a634bfcSVikram Hegde
429c94adbf9SFrank Van Der Linden immu->immu_flushops = &immu_regs_flushops;
430c94adbf9SFrank Van Der Linden
4313a634bfcSVikram Hegde return (DDI_SUCCESS);
4323a634bfcSVikram Hegde }
4333a634bfcSVikram Hegde
4343a634bfcSVikram Hegde /* ############### Functions exported ################## */
4353a634bfcSVikram Hegde
4363a634bfcSVikram Hegde /*
4373a634bfcSVikram Hegde * immu_regs_setup()
4383a634bfcSVikram Hegde * Setup mappings to a IMMU unit's registers
4393a634bfcSVikram Hegde * so that they can be read/written
4403a634bfcSVikram Hegde */
4413a634bfcSVikram Hegde void
immu_regs_setup(list_t * listp)4423a634bfcSVikram Hegde immu_regs_setup(list_t *listp)
4433a634bfcSVikram Hegde {
4443a634bfcSVikram Hegde int i;
4453a634bfcSVikram Hegde immu_t *immu;
4463a634bfcSVikram Hegde
4473a634bfcSVikram Hegde for (i = 0; i < IMMU_MAXSEG; i++) {
4483a634bfcSVikram Hegde immu = list_head(listp);
4493a634bfcSVikram Hegde for (; immu; immu = list_next(listp, immu)) {
4503a634bfcSVikram Hegde /* do your best, continue on error */
4513a634bfcSVikram Hegde if (setup_regs(immu) != DDI_SUCCESS) {
4523a634bfcSVikram Hegde immu->immu_regs_setup = B_FALSE;
4533a634bfcSVikram Hegde } else {
4543a634bfcSVikram Hegde immu->immu_regs_setup = B_TRUE;
4553a634bfcSVikram Hegde }
4563a634bfcSVikram Hegde }
4573a634bfcSVikram Hegde }
4583a634bfcSVikram Hegde }
4593a634bfcSVikram Hegde
4603a634bfcSVikram Hegde /*
4613a634bfcSVikram Hegde * immu_regs_map()
4623a634bfcSVikram Hegde */
4633a634bfcSVikram Hegde int
immu_regs_resume(immu_t * immu)4643a634bfcSVikram Hegde immu_regs_resume(immu_t *immu)
4653a634bfcSVikram Hegde {
4663a634bfcSVikram Hegde int error;
4673a634bfcSVikram Hegde
4683a634bfcSVikram Hegde /*
4693a634bfcSVikram Hegde * remap the register address space
4703a634bfcSVikram Hegde */
4713a634bfcSVikram Hegde error = ddi_regs_map_setup(immu->immu_dip, 0,
4723a634bfcSVikram Hegde (caddr_t *)&(immu->immu_regs_addr), (offset_t)0,
4733a634bfcSVikram Hegde (offset_t)IMMU_REGSZ, &immu_regs_attr,
4743a634bfcSVikram Hegde &(immu->immu_regs_handle));
4753a634bfcSVikram Hegde if (error != DDI_SUCCESS) {
4763a634bfcSVikram Hegde return (DDI_FAILURE);
4773a634bfcSVikram Hegde }
4783a634bfcSVikram Hegde
4793a634bfcSVikram Hegde immu_regs_set_root_table(immu);
4803a634bfcSVikram Hegde
4813a634bfcSVikram Hegde immu_regs_intr_enable(immu, immu->immu_regs_intr_msi_addr,
4823a634bfcSVikram Hegde immu->immu_regs_intr_msi_data, immu->immu_regs_intr_uaddr);
4833a634bfcSVikram Hegde
484*7fd1b424SToomas Soome (void) immu_intr_handler((caddr_t)immu, NULL);
4853a634bfcSVikram Hegde
4863a634bfcSVikram Hegde immu_regs_intrmap_enable(immu, immu->immu_intrmap_irta_reg);
4873a634bfcSVikram Hegde
4883a634bfcSVikram Hegde immu_regs_qinv_enable(immu, immu->immu_qinv_reg_value);
4893a634bfcSVikram Hegde
4903a634bfcSVikram Hegde
4913a634bfcSVikram Hegde return (error);
4923a634bfcSVikram Hegde }
4933a634bfcSVikram Hegde
4943a634bfcSVikram Hegde /*
4953a634bfcSVikram Hegde * immu_regs_suspend()
4963a634bfcSVikram Hegde */
4973a634bfcSVikram Hegde void
immu_regs_suspend(immu_t * immu)4983a634bfcSVikram Hegde immu_regs_suspend(immu_t *immu)
4993a634bfcSVikram Hegde {
5003a634bfcSVikram Hegde
5013a634bfcSVikram Hegde immu->immu_intrmap_running = B_FALSE;
5023a634bfcSVikram Hegde
5033a634bfcSVikram Hegde /* Finally, unmap the regs */
5043a634bfcSVikram Hegde ddi_regs_map_free(&(immu->immu_regs_handle));
5053a634bfcSVikram Hegde }
5063a634bfcSVikram Hegde
5073a634bfcSVikram Hegde /*
5083a634bfcSVikram Hegde * immu_regs_startup()
5093a634bfcSVikram Hegde * set a IMMU unit's registers to startup the unit
5103a634bfcSVikram Hegde */
5113a634bfcSVikram Hegde void
immu_regs_startup(immu_t * immu)5123a634bfcSVikram Hegde immu_regs_startup(immu_t *immu)
5133a634bfcSVikram Hegde {
5143a634bfcSVikram Hegde uint32_t status;
5153a634bfcSVikram Hegde
5163a634bfcSVikram Hegde if (immu->immu_regs_setup == B_FALSE) {
5173a634bfcSVikram Hegde return;
5183a634bfcSVikram Hegde }
5193a634bfcSVikram Hegde
5203a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock));
5213a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD,
5223a634bfcSVikram Hegde immu->immu_regs_cmdval | IMMU_GCMD_TE);
5233a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS,
5243a634bfcSVikram Hegde get_reg32, (status & IMMU_GSTS_TES), status);
5253a634bfcSVikram Hegde immu->immu_regs_cmdval |= IMMU_GCMD_TE;
5263a634bfcSVikram Hegde immu->immu_regs_running = B_TRUE;
5273a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock));
5283a634bfcSVikram Hegde
52950200e77SFrank Van Der Linden ddi_err(DER_NOTE, NULL, "%s running", immu->immu_name);
5303a634bfcSVikram Hegde }
5313a634bfcSVikram Hegde
5323a634bfcSVikram Hegde /*
5333a634bfcSVikram Hegde * immu_regs_shutdown()
5343a634bfcSVikram Hegde * shutdown a unit
5353a634bfcSVikram Hegde */
5363a634bfcSVikram Hegde void
immu_regs_shutdown(immu_t * immu)5373a634bfcSVikram Hegde immu_regs_shutdown(immu_t *immu)
5383a634bfcSVikram Hegde {
5393a634bfcSVikram Hegde uint32_t status;
5403a634bfcSVikram Hegde
5413a634bfcSVikram Hegde if (immu->immu_regs_running == B_FALSE) {
5423a634bfcSVikram Hegde return;
5433a634bfcSVikram Hegde }
5443a634bfcSVikram Hegde
5453a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock));
5463a634bfcSVikram Hegde immu->immu_regs_cmdval &= ~IMMU_GCMD_TE;
5473a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD,
5483a634bfcSVikram Hegde immu->immu_regs_cmdval);
5493a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS,
5503a634bfcSVikram Hegde get_reg32, !(status & IMMU_GSTS_TES), status);
5513a634bfcSVikram Hegde immu->immu_regs_running = B_FALSE;
5523a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock));
5533a634bfcSVikram Hegde
5543a634bfcSVikram Hegde ddi_err(DER_NOTE, NULL, "IOMMU %s stopped", immu->immu_name);
5553a634bfcSVikram Hegde }
5563a634bfcSVikram Hegde
5573a634bfcSVikram Hegde /*
5583a634bfcSVikram Hegde * immu_regs_intr()
5593a634bfcSVikram Hegde * Set a IMMU unit regs to setup a IMMU unit's
5603a634bfcSVikram Hegde * interrupt handler
5613a634bfcSVikram Hegde */
5623a634bfcSVikram Hegde void
immu_regs_intr_enable(immu_t * immu,uint32_t msi_addr,uint32_t msi_data,uint32_t uaddr)5633a634bfcSVikram Hegde immu_regs_intr_enable(immu_t *immu, uint32_t msi_addr, uint32_t msi_data,
5643a634bfcSVikram Hegde uint32_t uaddr)
5653a634bfcSVikram Hegde {
5663a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock));
5673a634bfcSVikram Hegde immu->immu_regs_intr_msi_addr = msi_addr;
5683a634bfcSVikram Hegde immu->immu_regs_intr_uaddr = uaddr;
5693a634bfcSVikram Hegde immu->immu_regs_intr_msi_data = msi_data;
5703a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_FEVNT_ADDR, msi_addr);
5713a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_FEVNT_UADDR, uaddr);
5723a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_FEVNT_DATA, msi_data);
5733a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_FEVNT_CON, 0);
5743a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock));
5753a634bfcSVikram Hegde }
5763a634bfcSVikram Hegde
5773a634bfcSVikram Hegde /*
5783a634bfcSVikram Hegde * immu_regs_passthru_supported()
5793a634bfcSVikram Hegde * Returns B_TRUE ifi passthru is supported
5803a634bfcSVikram Hegde */
5813a634bfcSVikram Hegde boolean_t
immu_regs_passthru_supported(immu_t * immu)5823a634bfcSVikram Hegde immu_regs_passthru_supported(immu_t *immu)
5833a634bfcSVikram Hegde {
5843a634bfcSVikram Hegde if (IMMU_ECAP_GET_PT(immu->immu_regs_excap)) {
5853a634bfcSVikram Hegde return (B_TRUE);
5863a634bfcSVikram Hegde }
5873a634bfcSVikram Hegde
5883a634bfcSVikram Hegde ddi_err(DER_WARN, NULL, "Passthru not supported");
5893a634bfcSVikram Hegde return (B_FALSE);
5903a634bfcSVikram Hegde }
5913a634bfcSVikram Hegde
5923a634bfcSVikram Hegde /*
5933a634bfcSVikram Hegde * immu_regs_is_TM_reserved()
5943a634bfcSVikram Hegde * Returns B_TRUE if TM field is reserved
5953a634bfcSVikram Hegde */
5963a634bfcSVikram Hegde boolean_t
immu_regs_is_TM_reserved(immu_t * immu)5973a634bfcSVikram Hegde immu_regs_is_TM_reserved(immu_t *immu)
5983a634bfcSVikram Hegde {
5993a634bfcSVikram Hegde if (IMMU_ECAP_GET_DI(immu->immu_regs_excap) ||
6003a634bfcSVikram Hegde IMMU_ECAP_GET_CH(immu->immu_regs_excap)) {
6013a634bfcSVikram Hegde return (B_FALSE);
6023a634bfcSVikram Hegde }
6033a634bfcSVikram Hegde return (B_TRUE);
6043a634bfcSVikram Hegde }
6053a634bfcSVikram Hegde
6063a634bfcSVikram Hegde /*
6073a634bfcSVikram Hegde * immu_regs_is_SNP_reserved()
6083a634bfcSVikram Hegde * Returns B_TRUE if SNP field is reserved
6093a634bfcSVikram Hegde */
6103a634bfcSVikram Hegde boolean_t
immu_regs_is_SNP_reserved(immu_t * immu)6113a634bfcSVikram Hegde immu_regs_is_SNP_reserved(immu_t *immu)
6123a634bfcSVikram Hegde {
6133a634bfcSVikram Hegde
6143a634bfcSVikram Hegde return (IMMU_ECAP_GET_SC(immu->immu_regs_excap) ? B_FALSE : B_TRUE);
6153a634bfcSVikram Hegde }
6163a634bfcSVikram Hegde
6173a634bfcSVikram Hegde /*
6183a634bfcSVikram Hegde * immu_regs_wbf_flush()
6193a634bfcSVikram Hegde * If required and supported, write to IMMU
6203a634bfcSVikram Hegde * unit's regs to flush DMA write buffer(s)
6213a634bfcSVikram Hegde */
6223a634bfcSVikram Hegde void
immu_regs_wbf_flush(immu_t * immu)6233a634bfcSVikram Hegde immu_regs_wbf_flush(immu_t *immu)
6243a634bfcSVikram Hegde {
6253a634bfcSVikram Hegde uint32_t status;
6263a634bfcSVikram Hegde
6273a634bfcSVikram Hegde if (!IMMU_CAP_GET_RWBF(immu->immu_regs_cap)) {
6283a634bfcSVikram Hegde return;
6293a634bfcSVikram Hegde }
6303a634bfcSVikram Hegde
6313a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock));
6323a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD,
6333a634bfcSVikram Hegde immu->immu_regs_cmdval | IMMU_GCMD_WBF);
6343a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS,
6353a634bfcSVikram Hegde get_reg32, (!(status & IMMU_GSTS_WBFS)), status);
6363a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock));
6373a634bfcSVikram Hegde }
6383a634bfcSVikram Hegde
6393a634bfcSVikram Hegde /*
6403a634bfcSVikram Hegde * immu_regs_cpu_flush()
641*7fd1b424SToomas Soome * flush the cpu cache line after CPU memory writes, so
6423a634bfcSVikram Hegde * IOMMU can see the writes
6433a634bfcSVikram Hegde */
6443a634bfcSVikram Hegde void
immu_regs_cpu_flush(immu_t * immu,caddr_t addr,uint_t size)6453a634bfcSVikram Hegde immu_regs_cpu_flush(immu_t *immu, caddr_t addr, uint_t size)
6463a634bfcSVikram Hegde {
64750200e77SFrank Van Der Linden uintptr_t startline, endline;
6483a634bfcSVikram Hegde
6493a634bfcSVikram Hegde if (immu->immu_dvma_coherent == B_TRUE)
6503a634bfcSVikram Hegde return;
6513a634bfcSVikram Hegde
65250200e77SFrank Van Der Linden startline = (uintptr_t)addr & ~(uintptr_t)(x86_clflush_size - 1);
65350200e77SFrank Van Der Linden endline = ((uintptr_t)addr + size - 1) &
65450200e77SFrank Van Der Linden ~(uintptr_t)(x86_clflush_size - 1);
65550200e77SFrank Van Der Linden while (startline <= endline) {
65650200e77SFrank Van Der Linden clflush_insn((caddr_t)startline);
65750200e77SFrank Van Der Linden startline += x86_clflush_size;
6583a634bfcSVikram Hegde }
6593a634bfcSVikram Hegde
6603a634bfcSVikram Hegde mfence_insn();
6613a634bfcSVikram Hegde }
6623a634bfcSVikram Hegde
6633a634bfcSVikram Hegde /*
6643a634bfcSVikram Hegde * immu_regs_context_flush()
6653a634bfcSVikram Hegde * flush the context cache
6663a634bfcSVikram Hegde */
667c94adbf9SFrank Van Der Linden static void
context_flush(immu_t * immu,uint8_t function_mask,uint16_t sid,uint_t did,immu_context_inv_t type)668c94adbf9SFrank Van Der Linden context_flush(immu_t *immu, uint8_t function_mask,
6693a634bfcSVikram Hegde uint16_t sid, uint_t did, immu_context_inv_t type)
6703a634bfcSVikram Hegde {
6713a634bfcSVikram Hegde uint64_t command = 0;
6723a634bfcSVikram Hegde uint64_t status;
6733a634bfcSVikram Hegde
6743a634bfcSVikram Hegde /*
6753a634bfcSVikram Hegde * define the command
6763a634bfcSVikram Hegde */
6773a634bfcSVikram Hegde switch (type) {
6783a634bfcSVikram Hegde case CONTEXT_FSI:
6793a634bfcSVikram Hegde command |= CCMD_INV_ICC | CCMD_INV_DEVICE
6803a634bfcSVikram Hegde | CCMD_INV_DID(did)
6813a634bfcSVikram Hegde | CCMD_INV_SID(sid) | CCMD_INV_FM(function_mask);
6823a634bfcSVikram Hegde break;
6833a634bfcSVikram Hegde case CONTEXT_DSI:
6843a634bfcSVikram Hegde command |= CCMD_INV_ICC | CCMD_INV_DOMAIN
6853a634bfcSVikram Hegde | CCMD_INV_DID(did);
6863a634bfcSVikram Hegde break;
6873a634bfcSVikram Hegde case CONTEXT_GLOBAL:
6883a634bfcSVikram Hegde command |= CCMD_INV_ICC | CCMD_INV_GLOBAL;
6893a634bfcSVikram Hegde break;
6903a634bfcSVikram Hegde default:
6913a634bfcSVikram Hegde ddi_err(DER_PANIC, NULL,
6923a634bfcSVikram Hegde "%s: incorrect context cache flush type",
6933a634bfcSVikram Hegde immu->immu_name);
6943a634bfcSVikram Hegde /*NOTREACHED*/
6953a634bfcSVikram Hegde }
6963a634bfcSVikram Hegde
6973a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock));
6983a634bfcSVikram Hegde put_reg64(immu, IMMU_REG_CONTEXT_CMD, command);
6993a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_CONTEXT_CMD, get_reg64,
7003a634bfcSVikram Hegde (!(status & CCMD_INV_ICC)), status);
7013a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock));
7023a634bfcSVikram Hegde }
7033a634bfcSVikram Hegde
70450200e77SFrank Van Der Linden /*ARGSUSED*/
705c94adbf9SFrank Van Der Linden void
immu_regs_context_fsi(immu_t * immu,uint8_t function_mask,uint16_t source_id,uint_t domain_id,immu_inv_wait_t * iwp)706c94adbf9SFrank Van Der Linden immu_regs_context_fsi(immu_t *immu, uint8_t function_mask,
70750200e77SFrank Van Der Linden uint16_t source_id, uint_t domain_id, immu_inv_wait_t *iwp)
708c94adbf9SFrank Van Der Linden {
709c94adbf9SFrank Van Der Linden context_flush(immu, function_mask, source_id, domain_id, CONTEXT_FSI);
710c94adbf9SFrank Van Der Linden }
711c94adbf9SFrank Van Der Linden
71250200e77SFrank Van Der Linden /*ARGSUSED*/
713c94adbf9SFrank Van Der Linden void
immu_regs_context_dsi(immu_t * immu,uint_t domain_id,immu_inv_wait_t * iwp)71450200e77SFrank Van Der Linden immu_regs_context_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp)
715c94adbf9SFrank Van Der Linden {
716c94adbf9SFrank Van Der Linden context_flush(immu, 0, 0, domain_id, CONTEXT_DSI);
717c94adbf9SFrank Van Der Linden }
718c94adbf9SFrank Van Der Linden
71950200e77SFrank Van Der Linden /*ARGSUSED*/
720c94adbf9SFrank Van Der Linden void
immu_regs_context_gbl(immu_t * immu,immu_inv_wait_t * iwp)72150200e77SFrank Van Der Linden immu_regs_context_gbl(immu_t *immu, immu_inv_wait_t *iwp)
722c94adbf9SFrank Van Der Linden {
723c94adbf9SFrank Van Der Linden context_flush(immu, 0, 0, 0, CONTEXT_GLOBAL);
724c94adbf9SFrank Van Der Linden }
725c94adbf9SFrank Van Der Linden
72650200e77SFrank Van Der Linden /*
72750200e77SFrank Van Der Linden * Nothing to do, all register operations are synchronous.
72850200e77SFrank Van Der Linden */
72950200e77SFrank Van Der Linden /*ARGSUSED*/
73050200e77SFrank Van Der Linden static void
immu_regs_inv_wait(immu_inv_wait_t * iwp)73150200e77SFrank Van Der Linden immu_regs_inv_wait(immu_inv_wait_t *iwp)
73250200e77SFrank Van Der Linden {
73350200e77SFrank Van Der Linden }
73450200e77SFrank Van Der Linden
7353a634bfcSVikram Hegde void
immu_regs_set_root_table(immu_t * immu)7363a634bfcSVikram Hegde immu_regs_set_root_table(immu_t *immu)
7373a634bfcSVikram Hegde {
7383a634bfcSVikram Hegde uint32_t status;
7393a634bfcSVikram Hegde
7403a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock));
7413a634bfcSVikram Hegde put_reg64(immu, IMMU_REG_ROOTENTRY,
7423a634bfcSVikram Hegde immu->immu_ctx_root->hwpg_paddr);
7433a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD,
7443a634bfcSVikram Hegde immu->immu_regs_cmdval | IMMU_GCMD_SRTP);
7453a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS,
7463a634bfcSVikram Hegde get_reg32, (status & IMMU_GSTS_RTPS), status);
7473a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock));
7483a634bfcSVikram Hegde }
7493a634bfcSVikram Hegde
7503a634bfcSVikram Hegde
7513a634bfcSVikram Hegde /* enable queued invalidation interface */
7523a634bfcSVikram Hegde void
immu_regs_qinv_enable(immu_t * immu,uint64_t qinv_reg_value)7533a634bfcSVikram Hegde immu_regs_qinv_enable(immu_t *immu, uint64_t qinv_reg_value)
7543a634bfcSVikram Hegde {
7553a634bfcSVikram Hegde uint32_t status;
7563a634bfcSVikram Hegde
7573a634bfcSVikram Hegde if (immu_qinv_enable == B_FALSE)
7583a634bfcSVikram Hegde return;
7593a634bfcSVikram Hegde
7603a634bfcSVikram Hegde mutex_enter(&immu->immu_regs_lock);
7613a634bfcSVikram Hegde immu->immu_qinv_reg_value = qinv_reg_value;
7623a634bfcSVikram Hegde /* Initialize the Invalidation Queue Tail register to zero */
7633a634bfcSVikram Hegde put_reg64(immu, IMMU_REG_INVAL_QT, 0);
7643a634bfcSVikram Hegde
7653a634bfcSVikram Hegde /* set invalidation queue base address register */
7663a634bfcSVikram Hegde put_reg64(immu, IMMU_REG_INVAL_QAR, qinv_reg_value);
7673a634bfcSVikram Hegde
7683a634bfcSVikram Hegde /* enable queued invalidation interface */
7693a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD,
7703a634bfcSVikram Hegde immu->immu_regs_cmdval | IMMU_GCMD_QIE);
7713a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS,
7723a634bfcSVikram Hegde get_reg32, (status & IMMU_GSTS_QIES), status);
7733a634bfcSVikram Hegde mutex_exit(&immu->immu_regs_lock);
7743a634bfcSVikram Hegde
7753a634bfcSVikram Hegde immu->immu_regs_cmdval |= IMMU_GCMD_QIE;
7763a634bfcSVikram Hegde immu->immu_qinv_running = B_TRUE;
7773a634bfcSVikram Hegde
7783a634bfcSVikram Hegde }
7793a634bfcSVikram Hegde
7803a634bfcSVikram Hegde /* enable interrupt remapping hardware unit */
7813a634bfcSVikram Hegde void
immu_regs_intrmap_enable(immu_t * immu,uint64_t irta_reg)7823a634bfcSVikram Hegde immu_regs_intrmap_enable(immu_t *immu, uint64_t irta_reg)
7833a634bfcSVikram Hegde {
7843a634bfcSVikram Hegde uint32_t status;
7853a634bfcSVikram Hegde
7863a634bfcSVikram Hegde if (immu_intrmap_enable == B_FALSE)
7873a634bfcSVikram Hegde return;
7883a634bfcSVikram Hegde
7893a634bfcSVikram Hegde /* set interrupt remap table pointer */
7903a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock));
7913a634bfcSVikram Hegde immu->immu_intrmap_irta_reg = irta_reg;
7923a634bfcSVikram Hegde put_reg64(immu, IMMU_REG_IRTAR, irta_reg);
7933a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD,
7943a634bfcSVikram Hegde immu->immu_regs_cmdval | IMMU_GCMD_SIRTP);
7953a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS,
7963a634bfcSVikram Hegde get_reg32, (status & IMMU_GSTS_IRTPS), status);
7973a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock));
7983a634bfcSVikram Hegde
7993a634bfcSVikram Hegde /* global flush intr entry cache */
80050200e77SFrank Van Der Linden immu_qinv_intr_global(immu, &immu->immu_intrmap_inv_wait);
8013a634bfcSVikram Hegde
8023a634bfcSVikram Hegde /* enable interrupt remapping */
8033a634bfcSVikram Hegde mutex_enter(&(immu->immu_regs_lock));
8043a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD,
8053a634bfcSVikram Hegde immu->immu_regs_cmdval | IMMU_GCMD_IRE);
8063a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS,
8073a634bfcSVikram Hegde get_reg32, (status & IMMU_GSTS_IRES),
8083a634bfcSVikram Hegde status);
8093a634bfcSVikram Hegde immu->immu_regs_cmdval |= IMMU_GCMD_IRE;
8103a634bfcSVikram Hegde
8113a634bfcSVikram Hegde /* set compatible mode */
8123a634bfcSVikram Hegde put_reg32(immu, IMMU_REG_GLOBAL_CMD,
8133a634bfcSVikram Hegde immu->immu_regs_cmdval | IMMU_GCMD_CFI);
8143a634bfcSVikram Hegde wait_completion(immu, IMMU_REG_GLOBAL_STS,
8153a634bfcSVikram Hegde get_reg32, (status & IMMU_GSTS_CFIS),
8163a634bfcSVikram Hegde status);
8173a634bfcSVikram Hegde immu->immu_regs_cmdval |= IMMU_GCMD_CFI;
8183a634bfcSVikram Hegde mutex_exit(&(immu->immu_regs_lock));
8193a634bfcSVikram Hegde
8203a634bfcSVikram Hegde immu->immu_intrmap_running = B_TRUE;
8213a634bfcSVikram Hegde }
8223a634bfcSVikram Hegde
8233a634bfcSVikram Hegde uint64_t
immu_regs_get64(immu_t * immu,uint_t reg)8243a634bfcSVikram Hegde immu_regs_get64(immu_t *immu, uint_t reg)
8253a634bfcSVikram Hegde {
8263a634bfcSVikram Hegde return (get_reg64(immu, reg));
8273a634bfcSVikram Hegde }
8283a634bfcSVikram Hegde
8293a634bfcSVikram Hegde uint32_t
immu_regs_get32(immu_t * immu,uint_t reg)8303a634bfcSVikram Hegde immu_regs_get32(immu_t *immu, uint_t reg)
8313a634bfcSVikram Hegde {
8323a634bfcSVikram Hegde return (get_reg32(immu, reg));
8333a634bfcSVikram Hegde }
8343a634bfcSVikram Hegde
8353a634bfcSVikram Hegde void
immu_regs_put64(immu_t * immu,uint_t reg,uint64_t val)8363a634bfcSVikram Hegde immu_regs_put64(immu_t *immu, uint_t reg, uint64_t val)
8373a634bfcSVikram Hegde {
8383a634bfcSVikram Hegde put_reg64(immu, reg, val);
8393a634bfcSVikram Hegde }
8403a634bfcSVikram Hegde
8413a634bfcSVikram Hegde void
immu_regs_put32(immu_t * immu,uint_t reg,uint32_t val)8423a634bfcSVikram Hegde immu_regs_put32(immu_t *immu, uint_t reg, uint32_t val)
8433a634bfcSVikram Hegde {
8443a634bfcSVikram Hegde put_reg32(immu, reg, val);
8453a634bfcSVikram Hegde }
846