17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
500d0963fSdilpreet * Common Development and Distribution License (the "License").
600d0963fSdilpreet * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate *
87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate * and limitations under the License.
127c478bd9Sstevel@tonic-gate *
137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate *
197c478bd9Sstevel@tonic-gate * CDDL HEADER END
207c478bd9Sstevel@tonic-gate */
217c478bd9Sstevel@tonic-gate /*
22b57cd2d3SMark Johnson * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
237c478bd9Sstevel@tonic-gate */
24bd155b87SGarrett D'Amore /*
25bd155b87SGarrett D'Amore * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
2689b43686SBayard Bell * Copyright (c) 2011 Bayard G. Bell. All rights reserved.
27cd21e7c5SGarrett D'Amore * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved.
286667d029SHans Rosenfeld * Copyright 2017 Joyent, Inc.
2923a869ddSRyan Zezeski * Copyright 2020 Ryan Zezeski
30bd155b87SGarrett D'Amore */
317c478bd9Sstevel@tonic-gate
327c478bd9Sstevel@tonic-gate /*
3312f080e7Smrj * x86 root nexus driver
347c478bd9Sstevel@tonic-gate */
357c478bd9Sstevel@tonic-gate
367c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
377c478bd9Sstevel@tonic-gate #include <sys/conf.h>
387c478bd9Sstevel@tonic-gate #include <sys/autoconf.h>
397c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
407c478bd9Sstevel@tonic-gate #include <sys/debug.h>
417c478bd9Sstevel@tonic-gate #include <sys/psw.h>
427c478bd9Sstevel@tonic-gate #include <sys/ddidmareq.h>
437c478bd9Sstevel@tonic-gate #include <sys/promif.h>
447c478bd9Sstevel@tonic-gate #include <sys/devops.h>
457c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
467c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
477c478bd9Sstevel@tonic-gate #include <vm/seg.h>
487c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
497c478bd9Sstevel@tonic-gate #include <vm/seg_dev.h>
507c478bd9Sstevel@tonic-gate #include <sys/vmem.h>
517c478bd9Sstevel@tonic-gate #include <sys/mman.h>
527c478bd9Sstevel@tonic-gate #include <vm/hat.h>
537c478bd9Sstevel@tonic-gate #include <vm/as.h>
547c478bd9Sstevel@tonic-gate #include <vm/page.h>
557c478bd9Sstevel@tonic-gate #include <sys/avintr.h>
567c478bd9Sstevel@tonic-gate #include <sys/errno.h>
577c478bd9Sstevel@tonic-gate #include <sys/modctl.h>
587c478bd9Sstevel@tonic-gate #include <sys/ddi_impldefs.h>
597c478bd9Sstevel@tonic-gate #include <sys/sunddi.h>
607c478bd9Sstevel@tonic-gate #include <sys/sunndi.h>
617a364d25Sschwartz #include <sys/mach_intr.h>
627c478bd9Sstevel@tonic-gate #include <sys/psm.h>
637c478bd9Sstevel@tonic-gate #include <sys/ontrap.h>
6412f080e7Smrj #include <sys/atomic.h>
6512f080e7Smrj #include <sys/sdt.h>
6612f080e7Smrj #include <sys/rootnex.h>
6712f080e7Smrj #include <vm/hat_i86.h>
6800d0963fSdilpreet #include <sys/ddifm.h>
6936945f79Smrj #include <sys/ddi_isa.h>
707ff178cdSJimmy Vetayases #include <sys/apic.h>
717c478bd9Sstevel@tonic-gate
72843e1988Sjohnlev #ifdef __xpv
73843e1988Sjohnlev #include <sys/bootinfo.h>
74843e1988Sjohnlev #include <sys/hypervisor.h>
75843e1988Sjohnlev #include <sys/bootconf.h>
76843e1988Sjohnlev #include <vm/kboot_mmu.h>
773a634bfcSVikram Hegde #endif
783a634bfcSVikram Hegde
7986ef0a63SRichard Lowe #if !defined(__xpv)
803a634bfcSVikram Hegde #include <sys/immu.h>
8120906b23SVikram Hegde #endif
8286c1f4dcSVikram Hegde
8386c1f4dcSVikram Hegde
8412f080e7Smrj /*
8512f080e7Smrj * enable/disable extra checking of function parameters. Useful for debugging
8612f080e7Smrj * drivers.
8712f080e7Smrj */
8812f080e7Smrj #ifdef DEBUG
8912f080e7Smrj int rootnex_alloc_check_parms = 1;
9012f080e7Smrj int rootnex_bind_check_inuse = 1;
9112f080e7Smrj int rootnex_unbind_verify_buffer = 0;
9212f080e7Smrj int rootnex_sync_check_parms = 1;
9312f080e7Smrj #else
9412f080e7Smrj int rootnex_alloc_check_parms = 0;
9512f080e7Smrj int rootnex_bind_check_inuse = 0;
9612f080e7Smrj int rootnex_unbind_verify_buffer = 0;
9712f080e7Smrj int rootnex_sync_check_parms = 0;
9812f080e7Smrj #endif
997c478bd9Sstevel@tonic-gate
1003a634bfcSVikram Hegde boolean_t rootnex_dmar_not_setup;
1013a634bfcSVikram Hegde
1027aec1d6eScindi /* Master Abort and Target Abort panic flag */
1037aec1d6eScindi int rootnex_fm_ma_ta_panic_flag = 0;
1047aec1d6eScindi
10512f080e7Smrj /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */
1067c478bd9Sstevel@tonic-gate int rootnex_bind_fail = 1;
1077c478bd9Sstevel@tonic-gate int rootnex_bind_warn = 1;
1087c478bd9Sstevel@tonic-gate uint8_t *rootnex_warn_list;
1097c478bd9Sstevel@tonic-gate /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */
1107c478bd9Sstevel@tonic-gate #define ROOTNEX_BIND_WARNING (0x1 << 0)
1117c478bd9Sstevel@tonic-gate
1127c478bd9Sstevel@tonic-gate /*
11312f080e7Smrj * revert back to old broken behavior of always sync'ing entire copy buffer.
11412f080e7Smrj * This is useful if be have a buggy driver which doesn't correctly pass in
11512f080e7Smrj * the offset and size into ddi_dma_sync().
1167c478bd9Sstevel@tonic-gate */
11712f080e7Smrj int rootnex_sync_ignore_params = 0;
1187c478bd9Sstevel@tonic-gate
1197c478bd9Sstevel@tonic-gate /*
12012f080e7Smrj * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1
12186ef0a63SRichard Lowe * page for alignment. Allocate enough windows to handle a 256K buffer w/ at
12286ef0a63SRichard Lowe * least 65 sgllen DMA engine, and enough copybuf buffer state pages to handle
12386ef0a63SRichard Lowe * 2 pages (< 8K). We will still need to allocate the copy buffer during bind
12486ef0a63SRichard Lowe * though (if we need one). These can only be modified in /etc/system before
12586ef0a63SRichard Lowe * rootnex attach.
1267c478bd9Sstevel@tonic-gate */
12712f080e7Smrj int rootnex_prealloc_cookies = 65;
12812f080e7Smrj int rootnex_prealloc_windows = 4;
12912f080e7Smrj int rootnex_prealloc_copybuf = 2;
1307c478bd9Sstevel@tonic-gate
13112f080e7Smrj /* driver global state */
13212f080e7Smrj static rootnex_state_t *rootnex_state;
1337c478bd9Sstevel@tonic-gate
134ef4ab52fSFrank Van Der Linden #ifdef DEBUG
13512f080e7Smrj /* shortcut to rootnex counters */
13612f080e7Smrj static uint64_t *rootnex_cnt;
137ef4ab52fSFrank Van Der Linden #endif
1387c478bd9Sstevel@tonic-gate
1397c478bd9Sstevel@tonic-gate /*
14012f080e7Smrj * XXX - does x86 even need these or are they left over from the SPARC days?
1417c478bd9Sstevel@tonic-gate */
14212f080e7Smrj /* statically defined integer/boolean properties for the root node */
14312f080e7Smrj static rootnex_intprop_t rootnex_intprp[] = {
14412f080e7Smrj { "PAGESIZE", PAGESIZE },
14512f080e7Smrj { "MMU_PAGESIZE", MMU_PAGESIZE },
14612f080e7Smrj { "MMU_PAGEOFFSET", MMU_PAGEOFFSET },
14712f080e7Smrj { DDI_RELATIVE_ADDRESSING, 1 },
14812f080e7Smrj };
14912f080e7Smrj #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t))
15012f080e7Smrj
15150200e77SFrank Van Der Linden /*
15250200e77SFrank Van Der Linden * If we're dom0, we're using a real device so we need to load
15350200e77SFrank Van Der Linden * the cookies with MFNs instead of PFNs.
15450200e77SFrank Van Der Linden */
155843e1988Sjohnlev #ifdef __xpv
156843e1988Sjohnlev typedef maddr_t rootnex_addr_t;
15750200e77SFrank Van Der Linden #define ROOTNEX_PADDR_TO_RBASE(pa) \
15850200e77SFrank Van Der Linden (DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa))
159843e1988Sjohnlev #else
160843e1988Sjohnlev typedef paddr_t rootnex_addr_t;
16150200e77SFrank Van Der Linden #define ROOTNEX_PADDR_TO_RBASE(pa) (pa)
162843e1988Sjohnlev #endif
163843e1988Sjohnlev
16412f080e7Smrj static struct cb_ops rootnex_cb_ops = {
16512f080e7Smrj nodev, /* open */
16612f080e7Smrj nodev, /* close */
16712f080e7Smrj nodev, /* strategy */
16812f080e7Smrj nodev, /* print */
16912f080e7Smrj nodev, /* dump */
17012f080e7Smrj nodev, /* read */
17112f080e7Smrj nodev, /* write */
17212f080e7Smrj nodev, /* ioctl */
17312f080e7Smrj nodev, /* devmap */
17412f080e7Smrj nodev, /* mmap */
17512f080e7Smrj nodev, /* segmap */
17612f080e7Smrj nochpoll, /* chpoll */
17712f080e7Smrj ddi_prop_op, /* cb_prop_op */
17812f080e7Smrj NULL, /* struct streamtab */
17912f080e7Smrj D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */
18012f080e7Smrj CB_REV, /* Rev */
18112f080e7Smrj nodev, /* cb_aread */
18212f080e7Smrj nodev /* cb_awrite */
18312f080e7Smrj };
1847c478bd9Sstevel@tonic-gate
18512f080e7Smrj static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
1867c478bd9Sstevel@tonic-gate off_t offset, off_t len, caddr_t *vaddrp);
18712f080e7Smrj static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip,
1887c478bd9Sstevel@tonic-gate struct hat *hat, struct seg *seg, caddr_t addr,
1897c478bd9Sstevel@tonic-gate struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock);
19012f080e7Smrj static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
19112f080e7Smrj ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
19212f080e7Smrj ddi_dma_handle_t *handlep);
19312f080e7Smrj static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip,
19412f080e7Smrj ddi_dma_handle_t handle);
19512f080e7Smrj static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
19612f080e7Smrj ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
19712f080e7Smrj ddi_dma_cookie_t *cookiep, uint_t *ccountp);
19812f080e7Smrj static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
19912f080e7Smrj ddi_dma_handle_t handle);
20012f080e7Smrj static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip,
20112f080e7Smrj ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
20212f080e7Smrj static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip,
20312f080e7Smrj ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
20412f080e7Smrj ddi_dma_cookie_t *cookiep, uint_t *ccountp);
20512f080e7Smrj static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip,
2067c478bd9Sstevel@tonic-gate ddi_dma_handle_t handle, enum ddi_dma_ctlops request,
2077c478bd9Sstevel@tonic-gate off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags);
20812f080e7Smrj static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip,
20912f080e7Smrj ddi_ctl_enum_t ctlop, void *arg, void *result);
21000d0963fSdilpreet static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
21100d0963fSdilpreet ddi_iblock_cookie_t *ibc);
21212f080e7Smrj static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip,
21312f080e7Smrj ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result);
2147ff178cdSJimmy Vetayases static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *,
2157ff178cdSJimmy Vetayases void *);
2167ff178cdSJimmy Vetayases static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *);
2177c478bd9Sstevel@tonic-gate
21820906b23SVikram Hegde static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
21920906b23SVikram Hegde ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
22020906b23SVikram Hegde ddi_dma_handle_t *handlep);
22120906b23SVikram Hegde static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
22220906b23SVikram Hegde ddi_dma_handle_t handle);
22320906b23SVikram Hegde static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
22420906b23SVikram Hegde ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
22520906b23SVikram Hegde ddi_dma_cookie_t *cookiep, uint_t *ccountp);
22620906b23SVikram Hegde static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
22720906b23SVikram Hegde ddi_dma_handle_t handle);
22886ef0a63SRichard Lowe #if !defined(__xpv)
22920906b23SVikram Hegde static void rootnex_coredma_reset_cookies(dev_info_t *dip,
23020906b23SVikram Hegde ddi_dma_handle_t handle);
23120906b23SVikram Hegde static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
23294f1124eSVikram Hegde ddi_dma_cookie_t **cookiepp, uint_t *ccountp);
23394f1124eSVikram Hegde static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
23494f1124eSVikram Hegde ddi_dma_cookie_t *cookiep, uint_t ccount);
23594f1124eSVikram Hegde static int rootnex_coredma_clear_cookies(dev_info_t *dip,
23694f1124eSVikram Hegde ddi_dma_handle_t handle);
23794f1124eSVikram Hegde static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle);
2385dfdb46bSVikram Hegde #endif
23920906b23SVikram Hegde static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip,
24020906b23SVikram Hegde ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
24120906b23SVikram Hegde static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip,
24220906b23SVikram Hegde ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
24320906b23SVikram Hegde ddi_dma_cookie_t *cookiep, uint_t *ccountp);
2447c478bd9Sstevel@tonic-gate
24586ef0a63SRichard Lowe #if !defined(__xpv)
24650200e77SFrank Van Der Linden static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
24750200e77SFrank Van Der Linden ddi_dma_handle_t handle, void *v);
24850200e77SFrank Van Der Linden static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
24950200e77SFrank Van Der Linden ddi_dma_handle_t handle);
250ee7536e8SFrank Van Der Linden #endif
25150200e77SFrank Van Der Linden
25250200e77SFrank Van Der Linden
2537c478bd9Sstevel@tonic-gate static struct bus_ops rootnex_bus_ops = {
2547c478bd9Sstevel@tonic-gate BUSO_REV,
2557c478bd9Sstevel@tonic-gate rootnex_map,
2567c478bd9Sstevel@tonic-gate NULL,
2577c478bd9Sstevel@tonic-gate NULL,
2587c478bd9Sstevel@tonic-gate NULL,
2597c478bd9Sstevel@tonic-gate rootnex_map_fault,
260cd21e7c5SGarrett D'Amore 0,
2617c478bd9Sstevel@tonic-gate rootnex_dma_allochdl,
2627c478bd9Sstevel@tonic-gate rootnex_dma_freehdl,
2637c478bd9Sstevel@tonic-gate rootnex_dma_bindhdl,
2647c478bd9Sstevel@tonic-gate rootnex_dma_unbindhdl,
26512f080e7Smrj rootnex_dma_sync,
2667c478bd9Sstevel@tonic-gate rootnex_dma_win,
2677c478bd9Sstevel@tonic-gate rootnex_dma_mctl,
2687c478bd9Sstevel@tonic-gate rootnex_ctlops,
2697c478bd9Sstevel@tonic-gate ddi_bus_prop_op,
2707c478bd9Sstevel@tonic-gate i_ddi_rootnex_get_eventcookie,
2717c478bd9Sstevel@tonic-gate i_ddi_rootnex_add_eventcall,
2727c478bd9Sstevel@tonic-gate i_ddi_rootnex_remove_eventcall,
2737c478bd9Sstevel@tonic-gate i_ddi_rootnex_post_event,
2747c478bd9Sstevel@tonic-gate 0, /* bus_intr_ctl */
2757c478bd9Sstevel@tonic-gate 0, /* bus_config */
2767c478bd9Sstevel@tonic-gate 0, /* bus_unconfig */
27700d0963fSdilpreet rootnex_fm_init, /* bus_fm_init */
2787c478bd9Sstevel@tonic-gate NULL, /* bus_fm_fini */
2797c478bd9Sstevel@tonic-gate NULL, /* bus_fm_access_enter */
2807c478bd9Sstevel@tonic-gate NULL, /* bus_fm_access_exit */
2817c478bd9Sstevel@tonic-gate NULL, /* bus_powr */
2827c478bd9Sstevel@tonic-gate rootnex_intr_ops /* bus_intr_op */
2837c478bd9Sstevel@tonic-gate };
2847c478bd9Sstevel@tonic-gate
28512f080e7Smrj static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
28612f080e7Smrj static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
2873a634bfcSVikram Hegde static int rootnex_quiesce(dev_info_t *dip);
2887c478bd9Sstevel@tonic-gate
2897c478bd9Sstevel@tonic-gate static struct dev_ops rootnex_ops = {
2907c478bd9Sstevel@tonic-gate DEVO_REV,
29112f080e7Smrj 0,
29212f080e7Smrj ddi_no_info,
29312f080e7Smrj nulldev,
2947c478bd9Sstevel@tonic-gate nulldev,
2957c478bd9Sstevel@tonic-gate rootnex_attach,
29612f080e7Smrj rootnex_detach,
29712f080e7Smrj nulldev,
29812f080e7Smrj &rootnex_cb_ops,
29919397407SSherry Moore &rootnex_bus_ops,
30019397407SSherry Moore NULL,
3013a634bfcSVikram Hegde rootnex_quiesce, /* quiesce */
3027c478bd9Sstevel@tonic-gate };
3037c478bd9Sstevel@tonic-gate
30412f080e7Smrj static struct modldrv rootnex_modldrv = {
30512f080e7Smrj &mod_driverops,
306613b2871SRichard Bean "i86pc root nexus",
30712f080e7Smrj &rootnex_ops
3087c478bd9Sstevel@tonic-gate };
3097c478bd9Sstevel@tonic-gate
31012f080e7Smrj static struct modlinkage rootnex_modlinkage = {
31112f080e7Smrj MODREV_1,
31212f080e7Smrj (void *)&rootnex_modldrv,
31312f080e7Smrj NULL
3147c478bd9Sstevel@tonic-gate };
3157c478bd9Sstevel@tonic-gate
31686ef0a63SRichard Lowe #if !defined(__xpv)
31720906b23SVikram Hegde static iommulib_nexops_t iommulib_nexops = {
31820906b23SVikram Hegde IOMMU_NEXOPS_VERSION,
31920906b23SVikram Hegde "Rootnex IOMMU ops Vers 1.1",
32020906b23SVikram Hegde NULL,
32120906b23SVikram Hegde rootnex_coredma_allochdl,
32220906b23SVikram Hegde rootnex_coredma_freehdl,
32320906b23SVikram Hegde rootnex_coredma_bindhdl,
32420906b23SVikram Hegde rootnex_coredma_unbindhdl,
32520906b23SVikram Hegde rootnex_coredma_reset_cookies,
32620906b23SVikram Hegde rootnex_coredma_get_cookies,
32794f1124eSVikram Hegde rootnex_coredma_set_cookies,
32894f1124eSVikram Hegde rootnex_coredma_clear_cookies,
32994f1124eSVikram Hegde rootnex_coredma_get_sleep_flags,
33020906b23SVikram Hegde rootnex_coredma_sync,
33120906b23SVikram Hegde rootnex_coredma_win,
33250200e77SFrank Van Der Linden rootnex_coredma_hdl_setprivate,
33350200e77SFrank Van Der Linden rootnex_coredma_hdl_getprivate
33420906b23SVikram Hegde };
3355dfdb46bSVikram Hegde #endif
3367c478bd9Sstevel@tonic-gate
33712f080e7Smrj /*
33812f080e7Smrj * extern hacks
33912f080e7Smrj */
34012f080e7Smrj extern struct seg_ops segdev_ops;
34112f080e7Smrj extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */
34212f080e7Smrj #ifdef DDI_MAP_DEBUG
34312f080e7Smrj extern int ddi_map_debug_flag;
34412f080e7Smrj #define ddi_map_debug if (ddi_map_debug_flag) prom_printf
34512f080e7Smrj #endif
34612f080e7Smrj extern void i86_pp_map(page_t *pp, caddr_t kaddr);
34712f080e7Smrj extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr);
34812f080e7Smrj extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
34912f080e7Smrj psm_intr_op_t, int *);
35012f080e7Smrj extern int impl_ddi_sunbus_initchild(dev_info_t *dip);
35112f080e7Smrj extern void impl_ddi_sunbus_removechild(dev_info_t *dip);
35236945f79Smrj
35312f080e7Smrj /*
35412f080e7Smrj * Use device arena to use for device control register mappings.
35512f080e7Smrj * Various kernel memory walkers (debugger, dtrace) need to know
35612f080e7Smrj * to avoid this address range to prevent undesired device activity.
35712f080e7Smrj */
35812f080e7Smrj extern void *device_arena_alloc(size_t size, int vm_flag);
35912f080e7Smrj extern void device_arena_free(void * vaddr, size_t size);
36012f080e7Smrj
36112f080e7Smrj
36212f080e7Smrj /*
36312f080e7Smrj * Internal functions
36412f080e7Smrj */
36512f080e7Smrj static int rootnex_dma_init();
36612f080e7Smrj static void rootnex_add_props(dev_info_t *);
36712f080e7Smrj static int rootnex_ctl_reportdev(dev_info_t *dip);
36812f080e7Smrj static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum);
36912f080e7Smrj static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
37012f080e7Smrj static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
37112f080e7Smrj static int rootnex_map_handle(ddi_map_req_t *mp);
37212f080e7Smrj static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp);
37312f080e7Smrj static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize);
37412f080e7Smrj static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
37512f080e7Smrj rootnex_sglinfo_t *sglinfo);
37650200e77SFrank Van Der Linden static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object,
37750200e77SFrank Van Der Linden ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo);
37812f080e7Smrj static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
37950200e77SFrank Van Der Linden rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
38012f080e7Smrj static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
38112f080e7Smrj rootnex_dma_t *dma, ddi_dma_attr_t *attr);
38212f080e7Smrj static void rootnex_teardown_copybuf(rootnex_dma_t *dma);
38312f080e7Smrj static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
38450200e77SFrank Van Der Linden ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
38512f080e7Smrj static void rootnex_teardown_windows(rootnex_dma_t *dma);
38612f080e7Smrj static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
38712f080e7Smrj rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset);
38812f080e7Smrj static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object,
38912f080e7Smrj rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset,
39012f080e7Smrj size_t *copybuf_used, page_t **cur_pp);
39112f080e7Smrj static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp,
39212f080e7Smrj rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie,
39312f080e7Smrj ddi_dma_attr_t *attr, off_t cur_offset);
39412f080e7Smrj static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp,
39512f080e7Smrj rootnex_dma_t *dma, rootnex_window_t **windowp,
39612f080e7Smrj ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used);
39712f080e7Smrj static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp,
39812f080e7Smrj rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie);
39912f080e7Smrj static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
40012f080e7Smrj off_t offset, size_t size, uint_t cache_flags);
40112f080e7Smrj static int rootnex_verify_buffer(rootnex_dma_t *dma);
40200d0963fSdilpreet static int rootnex_dma_check(dev_info_t *dip, const void *handle,
40300d0963fSdilpreet const void *comp_addr, const void *not_used);
40407c6692fSMark Johnson static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object,
40507c6692fSMark Johnson rootnex_sglinfo_t *sglinfo);
40650200e77SFrank Van Der Linden static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object);
40712f080e7Smrj
40812f080e7Smrj /*
40912f080e7Smrj * _init()
41012f080e7Smrj *
41112f080e7Smrj */
4127c478bd9Sstevel@tonic-gate int
_init(void)4137c478bd9Sstevel@tonic-gate _init(void)
4147c478bd9Sstevel@tonic-gate {
41512f080e7Smrj
41612f080e7Smrj rootnex_state = NULL;
41712f080e7Smrj return (mod_install(&rootnex_modlinkage));
41812f080e7Smrj }
41912f080e7Smrj
42012f080e7Smrj
42112f080e7Smrj /*
42212f080e7Smrj * _info()
42312f080e7Smrj *
42412f080e7Smrj */
42512f080e7Smrj int
_info(struct modinfo * modinfop)42612f080e7Smrj _info(struct modinfo *modinfop)
42712f080e7Smrj {
42812f080e7Smrj return (mod_info(&rootnex_modlinkage, modinfop));
4297c478bd9Sstevel@tonic-gate }
4307c478bd9Sstevel@tonic-gate
43112f080e7Smrj
43212f080e7Smrj /*
43312f080e7Smrj * _fini()
43412f080e7Smrj *
43512f080e7Smrj */
4367c478bd9Sstevel@tonic-gate int
_fini(void)4377c478bd9Sstevel@tonic-gate _fini(void)
4387c478bd9Sstevel@tonic-gate {
4397c478bd9Sstevel@tonic-gate return (EBUSY);
4407c478bd9Sstevel@tonic-gate }
4417c478bd9Sstevel@tonic-gate
44212f080e7Smrj
44312f080e7Smrj /*
44412f080e7Smrj * rootnex_attach()
44512f080e7Smrj *
44612f080e7Smrj */
44712f080e7Smrj static int
rootnex_attach(dev_info_t * dip,ddi_attach_cmd_t cmd)44812f080e7Smrj rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
4497c478bd9Sstevel@tonic-gate {
4507aec1d6eScindi int fmcap;
45112f080e7Smrj int e;
45212f080e7Smrj
45312f080e7Smrj switch (cmd) {
45412f080e7Smrj case DDI_ATTACH:
45512f080e7Smrj break;
45612f080e7Smrj case DDI_RESUME:
45786ef0a63SRichard Lowe #if !defined(__xpv)
4583a634bfcSVikram Hegde return (immu_unquiesce());
4593a634bfcSVikram Hegde #else
46012f080e7Smrj return (DDI_SUCCESS);
4613a634bfcSVikram Hegde #endif
46212f080e7Smrj default:
46312f080e7Smrj return (DDI_FAILURE);
46412f080e7Smrj }
46512f080e7Smrj
46612f080e7Smrj /*
46712f080e7Smrj * We should only have one instance of rootnex. Save it away since we
46812f080e7Smrj * don't have an easy way to get it back later.
46912f080e7Smrj */
47012f080e7Smrj ASSERT(rootnex_state == NULL);
47112f080e7Smrj rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP);
47212f080e7Smrj
47312f080e7Smrj rootnex_state->r_dip = dip;
4747aec1d6eScindi rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15);
47512f080e7Smrj rootnex_state->r_reserved_msg_printed = B_FALSE;
476ef4ab52fSFrank Van Der Linden #ifdef DEBUG
47712f080e7Smrj rootnex_cnt = &rootnex_state->r_counters[0];
478ef4ab52fSFrank Van Der Linden #endif
47912f080e7Smrj
4807aec1d6eScindi /*
4817aec1d6eScindi * Set minimum fm capability level for i86pc platforms and then
4827aec1d6eScindi * initialize error handling. Since we're the rootnex, we don't
4837aec1d6eScindi * care what's returned in the fmcap field.
4847aec1d6eScindi */
48500d0963fSdilpreet ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
48600d0963fSdilpreet DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
4877aec1d6eScindi fmcap = ddi_system_fmcap;
4887aec1d6eScindi ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc);
4897aec1d6eScindi
49012f080e7Smrj /* initialize DMA related state */
49112f080e7Smrj e = rootnex_dma_init();
49212f080e7Smrj if (e != DDI_SUCCESS) {
49312f080e7Smrj kmem_free(rootnex_state, sizeof (rootnex_state_t));
49412f080e7Smrj return (DDI_FAILURE);
49512f080e7Smrj }
49612f080e7Smrj
49712f080e7Smrj /* Add static root node properties */
49812f080e7Smrj rootnex_add_props(dip);
49912f080e7Smrj
50012f080e7Smrj /* since we can't call ddi_report_dev() */
50112f080e7Smrj cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip));
50212f080e7Smrj
50312f080e7Smrj /* Initialize rootnex event handle */
50412f080e7Smrj i_ddi_rootnex_init_events(dip);
50512f080e7Smrj
50686ef0a63SRichard Lowe #if !defined(__xpv)
50720906b23SVikram Hegde e = iommulib_nexus_register(dip, &iommulib_nexops,
50820906b23SVikram Hegde &rootnex_state->r_iommulib_handle);
50920906b23SVikram Hegde
51020906b23SVikram Hegde ASSERT(e == DDI_SUCCESS);
51120906b23SVikram Hegde #endif
51220906b23SVikram Hegde
51312f080e7Smrj return (DDI_SUCCESS);
5147c478bd9Sstevel@tonic-gate }
5157c478bd9Sstevel@tonic-gate
51612f080e7Smrj
5177c478bd9Sstevel@tonic-gate /*
51812f080e7Smrj * rootnex_detach()
5197c478bd9Sstevel@tonic-gate *
5207c478bd9Sstevel@tonic-gate */
52112f080e7Smrj /*ARGSUSED*/
52212f080e7Smrj static int
rootnex_detach(dev_info_t * dip,ddi_detach_cmd_t cmd)52312f080e7Smrj rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
52412f080e7Smrj {
52512f080e7Smrj switch (cmd) {
52612f080e7Smrj case DDI_SUSPEND:
52786ef0a63SRichard Lowe #if !defined(__xpv)
5283a634bfcSVikram Hegde return (immu_quiesce());
5293a634bfcSVikram Hegde #else
5303a634bfcSVikram Hegde return (DDI_SUCCESS);
5313a634bfcSVikram Hegde #endif
53212f080e7Smrj default:
53312f080e7Smrj return (DDI_FAILURE);
53412f080e7Smrj }
5353a634bfcSVikram Hegde /*NOTREACHED*/
53612f080e7Smrj
53712f080e7Smrj }
5387c478bd9Sstevel@tonic-gate
5397c478bd9Sstevel@tonic-gate
54012f080e7Smrj /*
54112f080e7Smrj * rootnex_dma_init()
54212f080e7Smrj *
54312f080e7Smrj */
5447c478bd9Sstevel@tonic-gate /*ARGSUSED*/
5457c478bd9Sstevel@tonic-gate static int
rootnex_dma_init()54612f080e7Smrj rootnex_dma_init()
5477c478bd9Sstevel@tonic-gate {
54812f080e7Smrj size_t bufsize;
5497c478bd9Sstevel@tonic-gate
5507c478bd9Sstevel@tonic-gate
55112f080e7Smrj /*
55212f080e7Smrj * size of our cookie/window/copybuf state needed in dma bind that we
55312f080e7Smrj * pre-alloc in dma_alloc_handle
55412f080e7Smrj */
55512f080e7Smrj rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies;
55612f080e7Smrj rootnex_state->r_prealloc_size =
55712f080e7Smrj (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) +
55812f080e7Smrj (rootnex_prealloc_windows * sizeof (rootnex_window_t)) +
55912f080e7Smrj (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t));
5607c478bd9Sstevel@tonic-gate
56112f080e7Smrj /*
56212f080e7Smrj * setup DDI DMA handle kmem cache, align each handle on 64 bytes,
56312f080e7Smrj * allocate 16 extra bytes for struct pointer alignment
56412f080e7Smrj * (p->dmai_private & dma->dp_prealloc_buffer)
56512f080e7Smrj */
56612f080e7Smrj bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) +
56712f080e7Smrj rootnex_state->r_prealloc_size + 0x10;
56812f080e7Smrj rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl",
56912f080e7Smrj bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0);
57012f080e7Smrj if (rootnex_state->r_dmahdl_cache == NULL) {
57112f080e7Smrj return (DDI_FAILURE);
57212f080e7Smrj }
5737c478bd9Sstevel@tonic-gate
5747c478bd9Sstevel@tonic-gate /*
5757c478bd9Sstevel@tonic-gate * allocate array to track which major numbers we have printed warnings
5767c478bd9Sstevel@tonic-gate * for.
5777c478bd9Sstevel@tonic-gate */
5787c478bd9Sstevel@tonic-gate rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list),
5797c478bd9Sstevel@tonic-gate KM_SLEEP);
5807c478bd9Sstevel@tonic-gate
5817c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
5827c478bd9Sstevel@tonic-gate }
5837c478bd9Sstevel@tonic-gate
5847c478bd9Sstevel@tonic-gate
5857c478bd9Sstevel@tonic-gate /*
58612f080e7Smrj * rootnex_add_props()
58712f080e7Smrj *
5887c478bd9Sstevel@tonic-gate */
5897c478bd9Sstevel@tonic-gate static void
rootnex_add_props(dev_info_t * dip)59012f080e7Smrj rootnex_add_props(dev_info_t *dip)
5917c478bd9Sstevel@tonic-gate {
59212f080e7Smrj rootnex_intprop_t *rpp;
5937c478bd9Sstevel@tonic-gate int i;
5947c478bd9Sstevel@tonic-gate
59512f080e7Smrj /* Add static integer/boolean properties to the root node */
59612f080e7Smrj rpp = rootnex_intprp;
59712f080e7Smrj for (i = 0; i < NROOT_INTPROPS; i++) {
59812f080e7Smrj (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip,
59912f080e7Smrj rpp[i].prop_name, rpp[i].prop_value);
6007c478bd9Sstevel@tonic-gate }
60112f080e7Smrj }
6027c478bd9Sstevel@tonic-gate
6037c478bd9Sstevel@tonic-gate
6047c478bd9Sstevel@tonic-gate
6057c478bd9Sstevel@tonic-gate /*
60612f080e7Smrj * *************************
60712f080e7Smrj * ctlops related routines
60812f080e7Smrj * *************************
6097c478bd9Sstevel@tonic-gate */
6107c478bd9Sstevel@tonic-gate
6117c478bd9Sstevel@tonic-gate /*
61212f080e7Smrj * rootnex_ctlops()
61312f080e7Smrj *
6147c478bd9Sstevel@tonic-gate */
615a195726fSgovinda /*ARGSUSED*/
6167c478bd9Sstevel@tonic-gate static int
rootnex_ctlops(dev_info_t * dip,dev_info_t * rdip,ddi_ctl_enum_t ctlop,void * arg,void * result)61712f080e7Smrj rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop,
61812f080e7Smrj void *arg, void *result)
6197c478bd9Sstevel@tonic-gate {
62012f080e7Smrj int n, *ptr;
62112f080e7Smrj struct ddi_parent_private_data *pdp;
6227c478bd9Sstevel@tonic-gate
62312f080e7Smrj switch (ctlop) {
62412f080e7Smrj case DDI_CTLOPS_DMAPMAPC:
62512f080e7Smrj /*
62612f080e7Smrj * Return 'partial' to indicate that dma mapping
62712f080e7Smrj * has to be done in the main MMU.
62812f080e7Smrj */
62912f080e7Smrj return (DDI_DMA_PARTIAL);
6307c478bd9Sstevel@tonic-gate
63112f080e7Smrj case DDI_CTLOPS_BTOP:
63212f080e7Smrj /*
63312f080e7Smrj * Convert byte count input to physical page units.
63412f080e7Smrj * (byte counts that are not a page-size multiple
63512f080e7Smrj * are rounded down)
63612f080e7Smrj */
63712f080e7Smrj *(ulong_t *)result = btop(*(ulong_t *)arg);
63812f080e7Smrj return (DDI_SUCCESS);
6397c478bd9Sstevel@tonic-gate
64012f080e7Smrj case DDI_CTLOPS_PTOB:
64112f080e7Smrj /*
64212f080e7Smrj * Convert size in physical pages to bytes
64312f080e7Smrj */
64412f080e7Smrj *(ulong_t *)result = ptob(*(ulong_t *)arg);
64512f080e7Smrj return (DDI_SUCCESS);
6467c478bd9Sstevel@tonic-gate
64712f080e7Smrj case DDI_CTLOPS_BTOPR:
6487c478bd9Sstevel@tonic-gate /*
64912f080e7Smrj * Convert byte count input to physical page units
65012f080e7Smrj * (byte counts that are not a page-size multiple
65112f080e7Smrj * are rounded up)
6527c478bd9Sstevel@tonic-gate */
65312f080e7Smrj *(ulong_t *)result = btopr(*(ulong_t *)arg);
65412f080e7Smrj return (DDI_SUCCESS);
6557c478bd9Sstevel@tonic-gate
65612f080e7Smrj case DDI_CTLOPS_INITCHILD:
65712f080e7Smrj return (impl_ddi_sunbus_initchild(arg));
6587c478bd9Sstevel@tonic-gate
65912f080e7Smrj case DDI_CTLOPS_UNINITCHILD:
66012f080e7Smrj impl_ddi_sunbus_removechild(arg);
66112f080e7Smrj return (DDI_SUCCESS);
66212f080e7Smrj
66312f080e7Smrj case DDI_CTLOPS_REPORTDEV:
66412f080e7Smrj return (rootnex_ctl_reportdev(rdip));
66512f080e7Smrj
66612f080e7Smrj case DDI_CTLOPS_IOMIN:
6677c478bd9Sstevel@tonic-gate /*
66812f080e7Smrj * Nothing to do here but reflect back..
6697c478bd9Sstevel@tonic-gate */
67012f080e7Smrj return (DDI_SUCCESS);
6717c478bd9Sstevel@tonic-gate
67212f080e7Smrj case DDI_CTLOPS_REGSIZE:
67312f080e7Smrj case DDI_CTLOPS_NREGS:
67412f080e7Smrj break;
6757c478bd9Sstevel@tonic-gate
67612f080e7Smrj case DDI_CTLOPS_SIDDEV:
67712f080e7Smrj if (ndi_dev_is_prom_node(rdip))
67812f080e7Smrj return (DDI_SUCCESS);
67912f080e7Smrj if (ndi_dev_is_persistent_node(rdip))
68012f080e7Smrj return (DDI_SUCCESS);
68112f080e7Smrj return (DDI_FAILURE);
6827c478bd9Sstevel@tonic-gate
68312f080e7Smrj case DDI_CTLOPS_POWER:
68412f080e7Smrj return ((*pm_platform_power)((power_req_t *)arg));
6857c478bd9Sstevel@tonic-gate
686a195726fSgovinda case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */
68712f080e7Smrj case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */
68812f080e7Smrj case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */
68912f080e7Smrj case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */
690a195726fSgovinda case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */
691a195726fSgovinda case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */
69212f080e7Smrj if (!rootnex_state->r_reserved_msg_printed) {
69312f080e7Smrj rootnex_state->r_reserved_msg_printed = B_TRUE;
69412f080e7Smrj cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for "
69512f080e7Smrj "1 or more reserved/obsolete operations.");
69612f080e7Smrj }
69712f080e7Smrj return (DDI_FAILURE);
69812f080e7Smrj
69912f080e7Smrj default:
70012f080e7Smrj return (DDI_FAILURE);
7017c478bd9Sstevel@tonic-gate }
70212f080e7Smrj /*
70312f080e7Smrj * The rest are for "hardware" properties
70412f080e7Smrj */
70512f080e7Smrj if ((pdp = ddi_get_parent_data(rdip)) == NULL)
70612f080e7Smrj return (DDI_FAILURE);
7077c478bd9Sstevel@tonic-gate
70812f080e7Smrj if (ctlop == DDI_CTLOPS_NREGS) {
70912f080e7Smrj ptr = (int *)result;
71012f080e7Smrj *ptr = pdp->par_nreg;
71112f080e7Smrj } else {
71212f080e7Smrj off_t *size = (off_t *)result;
71312f080e7Smrj
71412f080e7Smrj ptr = (int *)arg;
71512f080e7Smrj n = *ptr;
71612f080e7Smrj if (n >= pdp->par_nreg) {
71712f080e7Smrj return (DDI_FAILURE);
71812f080e7Smrj }
71912f080e7Smrj *size = (off_t)pdp->par_reg[n].regspec_size;
72012f080e7Smrj }
7217c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
7227c478bd9Sstevel@tonic-gate }
7237c478bd9Sstevel@tonic-gate
72412f080e7Smrj
72512f080e7Smrj /*
72612f080e7Smrj * rootnex_ctl_reportdev()
72712f080e7Smrj *
72812f080e7Smrj */
7297c478bd9Sstevel@tonic-gate static int
rootnex_ctl_reportdev(dev_info_t * dev)73012f080e7Smrj rootnex_ctl_reportdev(dev_info_t *dev)
7317c478bd9Sstevel@tonic-gate {
73212f080e7Smrj int i, n, len, f_len = 0;
73312f080e7Smrj char *buf;
7347c478bd9Sstevel@tonic-gate
73512f080e7Smrj buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP);
73612f080e7Smrj f_len += snprintf(buf, REPORTDEV_BUFSIZE,
73712f080e7Smrj "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev));
73812f080e7Smrj len = strlen(buf);
7397c478bd9Sstevel@tonic-gate
74012f080e7Smrj for (i = 0; i < sparc_pd_getnreg(dev); i++) {
7417c478bd9Sstevel@tonic-gate
74212f080e7Smrj struct regspec *rp = sparc_pd_getreg(dev, i);
7437c478bd9Sstevel@tonic-gate
74412f080e7Smrj if (i == 0)
74512f080e7Smrj f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
74612f080e7Smrj ": ");
74712f080e7Smrj else
74812f080e7Smrj f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
74912f080e7Smrj " and ");
75012f080e7Smrj len = strlen(buf);
7517c478bd9Sstevel@tonic-gate
75212f080e7Smrj switch (rp->regspec_bustype) {
7537c478bd9Sstevel@tonic-gate
75412f080e7Smrj case BTEISA:
75512f080e7Smrj f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
75612f080e7Smrj "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr);
75712f080e7Smrj break;
75812f080e7Smrj
75912f080e7Smrj case BTISA:
76012f080e7Smrj f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
76112f080e7Smrj "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr);
76212f080e7Smrj break;
76312f080e7Smrj
76412f080e7Smrj default:
76512f080e7Smrj f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
76612f080e7Smrj "space %x offset %x",
76712f080e7Smrj rp->regspec_bustype, rp->regspec_addr);
76812f080e7Smrj break;
76912f080e7Smrj }
77012f080e7Smrj len = strlen(buf);
77112f080e7Smrj }
77212f080e7Smrj for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) {
77312f080e7Smrj int pri;
7747c478bd9Sstevel@tonic-gate
77512f080e7Smrj if (i != 0) {
77612f080e7Smrj f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
77712f080e7Smrj ",");
77812f080e7Smrj len = strlen(buf);
77912f080e7Smrj }
78012f080e7Smrj pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri);
78112f080e7Smrj f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
78212f080e7Smrj " sparc ipl %d", pri);
78312f080e7Smrj len = strlen(buf);
78412f080e7Smrj }
78512f080e7Smrj #ifdef DEBUG
78612f080e7Smrj if (f_len + 1 >= REPORTDEV_BUFSIZE) {
78712f080e7Smrj cmn_err(CE_NOTE, "next message is truncated: "
78812f080e7Smrj "printed length 1024, real length %d", f_len);
78912f080e7Smrj }
79012f080e7Smrj #endif /* DEBUG */
79112f080e7Smrj cmn_err(CE_CONT, "?%s\n", buf);
79212f080e7Smrj kmem_free(buf, REPORTDEV_BUFSIZE);
7937c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
7947c478bd9Sstevel@tonic-gate }
7957c478bd9Sstevel@tonic-gate
79612f080e7Smrj
79712f080e7Smrj /*
79812f080e7Smrj * ******************
79912f080e7Smrj * map related code
80012f080e7Smrj * ******************
80112f080e7Smrj */
80212f080e7Smrj
80312f080e7Smrj /*
80412f080e7Smrj * rootnex_map()
80512f080e7Smrj *
80612f080e7Smrj */
8077c478bd9Sstevel@tonic-gate static int
rootnex_map(dev_info_t * dip,dev_info_t * rdip,ddi_map_req_t * mp,off_t offset,off_t len,caddr_t * vaddrp)80812f080e7Smrj rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset,
80912f080e7Smrj off_t len, caddr_t *vaddrp)
8107c478bd9Sstevel@tonic-gate {
8111f0c5e61SRobert Mustacchi struct regspec *orp = NULL;
8121f0c5e61SRobert Mustacchi struct regspec64 rp = { 0 };
8137c478bd9Sstevel@tonic-gate ddi_map_req_t mr = *mp; /* Get private copy of request */
8147c478bd9Sstevel@tonic-gate
8157c478bd9Sstevel@tonic-gate mp = &mr;
8167c478bd9Sstevel@tonic-gate
8177c478bd9Sstevel@tonic-gate switch (mp->map_op) {
8187c478bd9Sstevel@tonic-gate case DDI_MO_MAP_LOCKED:
8197c478bd9Sstevel@tonic-gate case DDI_MO_UNMAP:
8207c478bd9Sstevel@tonic-gate case DDI_MO_MAP_HANDLE:
8217c478bd9Sstevel@tonic-gate break;
8227c478bd9Sstevel@tonic-gate default:
8237c478bd9Sstevel@tonic-gate #ifdef DDI_MAP_DEBUG
8247c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.",
8257c478bd9Sstevel@tonic-gate mp->map_op);
8267c478bd9Sstevel@tonic-gate #endif /* DDI_MAP_DEBUG */
8277c478bd9Sstevel@tonic-gate return (DDI_ME_UNIMPLEMENTED);
8287c478bd9Sstevel@tonic-gate }
8297c478bd9Sstevel@tonic-gate
8307c478bd9Sstevel@tonic-gate if (mp->map_flags & DDI_MF_USER_MAPPING) {
8317c478bd9Sstevel@tonic-gate #ifdef DDI_MAP_DEBUG
8327c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user.");
8337c478bd9Sstevel@tonic-gate #endif /* DDI_MAP_DEBUG */
8347c478bd9Sstevel@tonic-gate return (DDI_ME_UNIMPLEMENTED);
8357c478bd9Sstevel@tonic-gate }
8367c478bd9Sstevel@tonic-gate
8377c478bd9Sstevel@tonic-gate /*
8381f0c5e61SRobert Mustacchi * First, we need to get the original regspec out before we convert it
8391f0c5e61SRobert Mustacchi * to the extended format. If we have a register number, then we need to
8401f0c5e61SRobert Mustacchi * convert that to a regspec.
8417c478bd9Sstevel@tonic-gate */
8427c478bd9Sstevel@tonic-gate if (mp->map_type == DDI_MT_RNUMBER) {
8437c478bd9Sstevel@tonic-gate
8447c478bd9Sstevel@tonic-gate int rnumber = mp->map_obj.rnumber;
8457c478bd9Sstevel@tonic-gate #ifdef DDI_MAP_DEBUG
8467c478bd9Sstevel@tonic-gate static char *out_of_range =
8477c478bd9Sstevel@tonic-gate "rootnex_map: Out of range rnumber <%d>, device <%s>";
8487c478bd9Sstevel@tonic-gate #endif /* DDI_MAP_DEBUG */
8497c478bd9Sstevel@tonic-gate
8501f0c5e61SRobert Mustacchi orp = i_ddi_rnumber_to_regspec(rdip, rnumber);
8511f0c5e61SRobert Mustacchi if (orp == NULL) {
8527c478bd9Sstevel@tonic-gate #ifdef DDI_MAP_DEBUG
8537c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, out_of_range, rnumber,
8547c478bd9Sstevel@tonic-gate ddi_get_name(rdip));
8557c478bd9Sstevel@tonic-gate #endif /* DDI_MAP_DEBUG */
8567c478bd9Sstevel@tonic-gate return (DDI_ME_RNUMBER_RANGE);
8577c478bd9Sstevel@tonic-gate }
8581f0c5e61SRobert Mustacchi } else if (!(mp->map_flags & DDI_MF_EXT_REGSPEC)) {
8591f0c5e61SRobert Mustacchi orp = mp->map_obj.rp;
8601f0c5e61SRobert Mustacchi }
8617c478bd9Sstevel@tonic-gate
8621f0c5e61SRobert Mustacchi /*
8631f0c5e61SRobert Mustacchi * Ensure that we are always using a 64-bit extended regspec regardless
8641f0c5e61SRobert Mustacchi * of what was passed into us. If the child driver is using a 64-bit
8651f0c5e61SRobert Mustacchi * regspec, then we need to make sure that we copy this to the local
8661f0c5e61SRobert Mustacchi * regspec64, rp.
8671f0c5e61SRobert Mustacchi */
8681f0c5e61SRobert Mustacchi if (orp != NULL) {
8691f0c5e61SRobert Mustacchi rp.regspec_bustype = orp->regspec_bustype;
8701f0c5e61SRobert Mustacchi rp.regspec_addr = orp->regspec_addr;
8711f0c5e61SRobert Mustacchi rp.regspec_size = orp->regspec_size;
8721f0c5e61SRobert Mustacchi } else {
8731f0c5e61SRobert Mustacchi struct regspec64 *rp64;
8741f0c5e61SRobert Mustacchi rp64 = (struct regspec64 *)mp->map_obj.rp;
8751f0c5e61SRobert Mustacchi rp = *rp64;
8767c478bd9Sstevel@tonic-gate }
8777c478bd9Sstevel@tonic-gate
8781f0c5e61SRobert Mustacchi mp->map_type = DDI_MT_REGSPEC;
8791f0c5e61SRobert Mustacchi mp->map_flags |= DDI_MF_EXT_REGSPEC;
8801f0c5e61SRobert Mustacchi mp->map_obj.rp = (struct regspec *)&rp;
8811f0c5e61SRobert Mustacchi
8827c478bd9Sstevel@tonic-gate /*
8837c478bd9Sstevel@tonic-gate * Adjust offset and length correspnding to called values...
8847c478bd9Sstevel@tonic-gate * XXX: A non-zero length means override the one in the regspec
8857c478bd9Sstevel@tonic-gate * XXX: (regardless of what's in the parent's range?)
8867c478bd9Sstevel@tonic-gate */
8877c478bd9Sstevel@tonic-gate
88812f080e7Smrj #ifdef DDI_MAP_DEBUG
889843e1988Sjohnlev cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d "
890843e1988Sjohnlev "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
8911f0c5e61SRobert Mustacchi rp.regspec_bustype, rp.regspec_addr, rp.regspec_size, offset,
892843e1988Sjohnlev len, mp->map_handlep);
89312f080e7Smrj #endif /* DDI_MAP_DEBUG */
89412f080e7Smrj
89512f080e7Smrj /*
89612f080e7Smrj * I/O or memory mapping:
89712f080e7Smrj *
89812f080e7Smrj * <bustype=0, addr=x, len=x>: memory
89912f080e7Smrj * <bustype=1, addr=x, len=x>: i/o
90012f080e7Smrj * <bustype>1, addr=0, len=x>: x86-compatibility i/o
90112f080e7Smrj */
90212f080e7Smrj
9031f0c5e61SRobert Mustacchi if (rp.regspec_bustype > 1 && rp.regspec_addr != 0) {
90412f080e7Smrj cmn_err(CE_WARN, "<%s,%s> invalid register spec"
9051f0c5e61SRobert Mustacchi " <0x%" PRIx64 ", 0x%" PRIx64 ", 0x%" PRIx64 ">",
9061f0c5e61SRobert Mustacchi ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype,
9071f0c5e61SRobert Mustacchi rp.regspec_addr, rp.regspec_size);
90812f080e7Smrj return (DDI_ME_INVAL);
90912f080e7Smrj }
91012f080e7Smrj
9111f0c5e61SRobert Mustacchi if (rp.regspec_bustype > 1 && rp.regspec_addr == 0) {
91212f080e7Smrj /*
91312f080e7Smrj * compatibility i/o mapping
91412f080e7Smrj */
9151f0c5e61SRobert Mustacchi rp.regspec_bustype += offset;
91612f080e7Smrj } else {
91712f080e7Smrj /*
91812f080e7Smrj * Normal memory or i/o mapping
91912f080e7Smrj */
9201f0c5e61SRobert Mustacchi rp.regspec_addr += offset;
92112f080e7Smrj }
92212f080e7Smrj
92312f080e7Smrj if (len != 0)
9241f0c5e61SRobert Mustacchi rp.regspec_size = len;
92512f080e7Smrj
92612f080e7Smrj #ifdef DDI_MAP_DEBUG
9271f0c5e61SRobert Mustacchi cmn_err(CE_CONT, " <%s,%s> <0x%" PRIx64 ", 0x%" PRIx64
9281f0c5e61SRobert Mustacchi ", 0x%" PRId64 "> offset %d len %d handle 0x%x\n",
9291f0c5e61SRobert Mustacchi ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype,
9301f0c5e61SRobert Mustacchi rp.regspec_addr, rp.regspec_size, offset, len, mp->map_handlep);
93112f080e7Smrj #endif /* DDI_MAP_DEBUG */
93212f080e7Smrj
9331f0c5e61SRobert Mustacchi
93412f080e7Smrj /*
9351f0c5e61SRobert Mustacchi * The x86 root nexus does not have any notion of valid ranges of
9361f0c5e61SRobert Mustacchi * addresses. Its children have valid ranges, but because there are none
9371f0c5e61SRobert Mustacchi * for the nexus, we don't need to call i_ddi_apply_range(). Verify
9381f0c5e61SRobert Mustacchi * that is the case.
93912f080e7Smrj */
9401f0c5e61SRobert Mustacchi ASSERT0(sparc_pd_getnrng(dip));
94112f080e7Smrj
94212f080e7Smrj switch (mp->map_op) {
94312f080e7Smrj case DDI_MO_MAP_LOCKED:
94412f080e7Smrj
94512f080e7Smrj /*
94612f080e7Smrj * Set up the locked down kernel mapping to the regspec...
94712f080e7Smrj */
94812f080e7Smrj
94912f080e7Smrj return (rootnex_map_regspec(mp, vaddrp));
95012f080e7Smrj
95112f080e7Smrj case DDI_MO_UNMAP:
95212f080e7Smrj
95312f080e7Smrj /*
95412f080e7Smrj * Release mapping...
95512f080e7Smrj */
95612f080e7Smrj
95712f080e7Smrj return (rootnex_unmap_regspec(mp, vaddrp));
95812f080e7Smrj
95912f080e7Smrj case DDI_MO_MAP_HANDLE:
96012f080e7Smrj
96112f080e7Smrj return (rootnex_map_handle(mp));
96212f080e7Smrj
96312f080e7Smrj default:
96412f080e7Smrj return (DDI_ME_UNIMPLEMENTED);
96512f080e7Smrj }
96612f080e7Smrj }
96712f080e7Smrj
96812f080e7Smrj
96912f080e7Smrj /*
97012f080e7Smrj * rootnex_map_fault()
97112f080e7Smrj *
97212f080e7Smrj * fault in mappings for requestors
97312f080e7Smrj */
97412f080e7Smrj /*ARGSUSED*/
97512f080e7Smrj static int
rootnex_map_fault(dev_info_t * dip,dev_info_t * rdip,struct hat * hat,struct seg * seg,caddr_t addr,struct devpage * dp,pfn_t pfn,uint_t prot,uint_t lock)97612f080e7Smrj rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat,
97712f080e7Smrj struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot,
97812f080e7Smrj uint_t lock)
97912f080e7Smrj {
98012f080e7Smrj
98112f080e7Smrj #ifdef DDI_MAP_DEBUG
98212f080e7Smrj ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn);
98312f080e7Smrj ddi_map_debug(" Seg <%s>\n",
98412f080e7Smrj seg->s_ops == &segdev_ops ? "segdev" :
98512f080e7Smrj seg == &kvseg ? "segkmem" : "NONE!");
98612f080e7Smrj #endif /* DDI_MAP_DEBUG */
98712f080e7Smrj
98812f080e7Smrj /*
98912f080e7Smrj * This is all terribly broken, but it is a start
99012f080e7Smrj *
99112f080e7Smrj * XXX Note that this test means that segdev_ops
99212f080e7Smrj * must be exported from seg_dev.c.
99312f080e7Smrj * XXX What about devices with their own segment drivers?
99412f080e7Smrj */
99512f080e7Smrj if (seg->s_ops == &segdev_ops) {
996843e1988Sjohnlev struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
99712f080e7Smrj
99812f080e7Smrj if (hat == NULL) {
99912f080e7Smrj /*
100012f080e7Smrj * This is one plausible interpretation of
100112f080e7Smrj * a null hat i.e. use the first hat on the
100212f080e7Smrj * address space hat list which by convention is
100312f080e7Smrj * the hat of the system MMU. At alternative
100412f080e7Smrj * would be to panic .. this might well be better ..
100512f080e7Smrj */
1006dc32d872SJosef 'Jeff' Sipek ASSERT(AS_READ_HELD(seg->s_as));
100712f080e7Smrj hat = seg->s_as->a_hat;
100812f080e7Smrj cmn_err(CE_NOTE, "rootnex_map_fault: nil hat");
100912f080e7Smrj }
101012f080e7Smrj hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr,
101112f080e7Smrj (lock ? HAT_LOAD_LOCK : HAT_LOAD));
101212f080e7Smrj } else if (seg == &kvseg && dp == NULL) {
101312f080e7Smrj hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot,
101412f080e7Smrj HAT_LOAD_LOCK);
101512f080e7Smrj } else
101612f080e7Smrj return (DDI_FAILURE);
101712f080e7Smrj return (DDI_SUCCESS);
101812f080e7Smrj }
101912f080e7Smrj
102012f080e7Smrj
102112f080e7Smrj static int
rootnex_map_regspec(ddi_map_req_t * mp,caddr_t * vaddrp)102212f080e7Smrj rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
102312f080e7Smrj {
1024843e1988Sjohnlev rootnex_addr_t rbase;
102512f080e7Smrj void *cvaddr;
10261f0c5e61SRobert Mustacchi uint64_t npages, pgoffset;
10271f0c5e61SRobert Mustacchi struct regspec64 *rp;
102812f080e7Smrj ddi_acc_hdl_t *hp;
102912f080e7Smrj ddi_acc_impl_t *ap;
103012f080e7Smrj uint_t hat_acc_flags;
1031843e1988Sjohnlev paddr_t pbase;
103212f080e7Smrj
10331f0c5e61SRobert Mustacchi ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC);
10341f0c5e61SRobert Mustacchi rp = (struct regspec64 *)mp->map_obj.rp;
103512f080e7Smrj hp = mp->map_handlep;
103612f080e7Smrj
103712f080e7Smrj #ifdef DDI_MAP_DEBUG
103812f080e7Smrj ddi_map_debug(
103912f080e7Smrj "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n",
104012f080e7Smrj rp->regspec_bustype, rp->regspec_addr,
104112f080e7Smrj rp->regspec_size, mp->map_handlep);
104212f080e7Smrj #endif /* DDI_MAP_DEBUG */
104312f080e7Smrj
104412f080e7Smrj /*
104512f080e7Smrj * I/O or memory mapping
104612f080e7Smrj *
104712f080e7Smrj * <bustype=0, addr=x, len=x>: memory
104812f080e7Smrj * <bustype=1, addr=x, len=x>: i/o
104912f080e7Smrj * <bustype>1, addr=0, len=x>: x86-compatibility i/o
105012f080e7Smrj */
105112f080e7Smrj
105212f080e7Smrj if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
105312f080e7Smrj cmn_err(CE_WARN, "rootnex: invalid register spec"
10541f0c5e61SRobert Mustacchi " <0x%" PRIx64 ", 0x%" PRIx64", 0x%" PRIx64">",
10551f0c5e61SRobert Mustacchi rp->regspec_bustype, rp->regspec_addr, rp->regspec_size);
105612f080e7Smrj return (DDI_FAILURE);
105712f080e7Smrj }
105812f080e7Smrj
105912f080e7Smrj if (rp->regspec_bustype != 0) {
106012f080e7Smrj /*
106112f080e7Smrj * I/O space - needs a handle.
106212f080e7Smrj */
106312f080e7Smrj if (hp == NULL) {
106412f080e7Smrj return (DDI_FAILURE);
106512f080e7Smrj }
106612f080e7Smrj ap = (ddi_acc_impl_t *)hp->ah_platform_private;
106712f080e7Smrj ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE;
106812f080e7Smrj impl_acc_hdl_init(hp);
106912f080e7Smrj
107012f080e7Smrj if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
107112f080e7Smrj #ifdef DDI_MAP_DEBUG
1072843e1988Sjohnlev ddi_map_debug("rootnex_map_regspec: mmap() "
1073843e1988Sjohnlev "to I/O space is not supported.\n");
107412f080e7Smrj #endif /* DDI_MAP_DEBUG */
107512f080e7Smrj return (DDI_ME_INVAL);
107612f080e7Smrj } else {
107712f080e7Smrj /*
107812f080e7Smrj * 1275-compliant vs. compatibility i/o mapping
107912f080e7Smrj */
108012f080e7Smrj *vaddrp =
108112f080e7Smrj (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ?
1082843e1988Sjohnlev ((caddr_t)(uintptr_t)rp->regspec_bustype) :
1083843e1988Sjohnlev ((caddr_t)(uintptr_t)rp->regspec_addr);
1084843e1988Sjohnlev #ifdef __xpv
1085843e1988Sjohnlev if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1086843e1988Sjohnlev hp->ah_pfn = xen_assign_pfn(
1087843e1988Sjohnlev mmu_btop((ulong_t)rp->regspec_addr &
1088843e1988Sjohnlev MMU_PAGEMASK));
1089843e1988Sjohnlev } else {
1090843e1988Sjohnlev hp->ah_pfn = mmu_btop(
1091843e1988Sjohnlev (ulong_t)rp->regspec_addr & MMU_PAGEMASK);
1092843e1988Sjohnlev }
1093843e1988Sjohnlev #else
109400d0963fSdilpreet hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr &
1095843e1988Sjohnlev MMU_PAGEMASK);
1096843e1988Sjohnlev #endif
109700d0963fSdilpreet hp->ah_pnum = mmu_btopr(rp->regspec_size +
109800d0963fSdilpreet (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET);
109912f080e7Smrj }
110012f080e7Smrj
110112f080e7Smrj #ifdef DDI_MAP_DEBUG
110212f080e7Smrj ddi_map_debug(
110312f080e7Smrj "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n",
110412f080e7Smrj rp->regspec_size, *vaddrp);
110512f080e7Smrj #endif /* DDI_MAP_DEBUG */
110612f080e7Smrj return (DDI_SUCCESS);
110712f080e7Smrj }
110812f080e7Smrj
110912f080e7Smrj /*
111012f080e7Smrj * Memory space
111112f080e7Smrj */
111212f080e7Smrj
111312f080e7Smrj if (hp != NULL) {
111412f080e7Smrj /*
111512f080e7Smrj * hat layer ignores
111612f080e7Smrj * hp->ah_acc.devacc_attr_endian_flags.
111712f080e7Smrj */
111812f080e7Smrj switch (hp->ah_acc.devacc_attr_dataorder) {
111912f080e7Smrj case DDI_STRICTORDER_ACC:
112012f080e7Smrj hat_acc_flags = HAT_STRICTORDER;
112112f080e7Smrj break;
112212f080e7Smrj case DDI_UNORDERED_OK_ACC:
112312f080e7Smrj hat_acc_flags = HAT_UNORDERED_OK;
112412f080e7Smrj break;
112512f080e7Smrj case DDI_MERGING_OK_ACC:
112612f080e7Smrj hat_acc_flags = HAT_MERGING_OK;
112712f080e7Smrj break;
112812f080e7Smrj case DDI_LOADCACHING_OK_ACC:
112912f080e7Smrj hat_acc_flags = HAT_LOADCACHING_OK;
113012f080e7Smrj break;
113112f080e7Smrj case DDI_STORECACHING_OK_ACC:
113212f080e7Smrj hat_acc_flags = HAT_STORECACHING_OK;
113312f080e7Smrj break;
1134584b574aSToomas Soome default:
1135584b574aSToomas Soome return (DDI_ME_INVAL);
113612f080e7Smrj }
113712f080e7Smrj ap = (ddi_acc_impl_t *)hp->ah_platform_private;
113812f080e7Smrj ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
113912f080e7Smrj impl_acc_hdl_init(hp);
114012f080e7Smrj hp->ah_hat_flags = hat_acc_flags;
114112f080e7Smrj } else {
114212f080e7Smrj hat_acc_flags = HAT_STRICTORDER;
114312f080e7Smrj }
114412f080e7Smrj
1145843e1988Sjohnlev rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK);
1146843e1988Sjohnlev #ifdef __xpv
1147843e1988Sjohnlev /*
1148843e1988Sjohnlev * If we're dom0, we're using a real device so we need to translate
1149843e1988Sjohnlev * the MA to a PA.
1150843e1988Sjohnlev */
1151843e1988Sjohnlev if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1152843e1988Sjohnlev pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase)));
1153843e1988Sjohnlev } else {
1154843e1988Sjohnlev pbase = rbase;
1155843e1988Sjohnlev }
1156843e1988Sjohnlev #else
1157843e1988Sjohnlev pbase = rbase;
1158843e1988Sjohnlev #endif
1159843e1988Sjohnlev pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
116012f080e7Smrj
116112f080e7Smrj if (rp->regspec_size == 0) {
116212f080e7Smrj #ifdef DDI_MAP_DEBUG
116312f080e7Smrj ddi_map_debug("rootnex_map_regspec: zero regspec_size\n");
116412f080e7Smrj #endif /* DDI_MAP_DEBUG */
116512f080e7Smrj return (DDI_ME_INVAL);
116612f080e7Smrj }
116712f080e7Smrj
116812f080e7Smrj if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1169843e1988Sjohnlev /* extra cast to make gcc happy */
1170843e1988Sjohnlev *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase));
117112f080e7Smrj } else {
117212f080e7Smrj npages = mmu_btopr(rp->regspec_size + pgoffset);
117312f080e7Smrj
117412f080e7Smrj #ifdef DDI_MAP_DEBUG
1175843e1988Sjohnlev ddi_map_debug("rootnex_map_regspec: Mapping %d pages "
1176843e1988Sjohnlev "physical %llx", npages, pbase);
117712f080e7Smrj #endif /* DDI_MAP_DEBUG */
117812f080e7Smrj
117912f080e7Smrj cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP);
118012f080e7Smrj if (cvaddr == NULL)
118112f080e7Smrj return (DDI_ME_NORESOURCES);
118212f080e7Smrj
118312f080e7Smrj /*
118412f080e7Smrj * Now map in the pages we've allocated...
118512f080e7Smrj */
1186843e1988Sjohnlev hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages),
1187843e1988Sjohnlev mmu_btop(pbase), mp->map_prot | hat_acc_flags,
1188843e1988Sjohnlev HAT_LOAD_LOCK);
118912f080e7Smrj *vaddrp = (caddr_t)cvaddr + pgoffset;
119000d0963fSdilpreet
119100d0963fSdilpreet /* save away pfn and npages for FMA */
119200d0963fSdilpreet hp = mp->map_handlep;
119300d0963fSdilpreet if (hp) {
1194843e1988Sjohnlev hp->ah_pfn = mmu_btop(pbase);
119500d0963fSdilpreet hp->ah_pnum = npages;
119600d0963fSdilpreet }
119712f080e7Smrj }
119812f080e7Smrj
119912f080e7Smrj #ifdef DDI_MAP_DEBUG
120012f080e7Smrj ddi_map_debug("at virtual 0x%x\n", *vaddrp);
120112f080e7Smrj #endif /* DDI_MAP_DEBUG */
120212f080e7Smrj return (DDI_SUCCESS);
120312f080e7Smrj }
120412f080e7Smrj
120512f080e7Smrj
120612f080e7Smrj static int
rootnex_unmap_regspec(ddi_map_req_t * mp,caddr_t * vaddrp)120712f080e7Smrj rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
120812f080e7Smrj {
120912f080e7Smrj caddr_t addr = (caddr_t)*vaddrp;
12101f0c5e61SRobert Mustacchi uint64_t npages, pgoffset;
12111f0c5e61SRobert Mustacchi struct regspec64 *rp;
121212f080e7Smrj
121312f080e7Smrj if (mp->map_flags & DDI_MF_DEVICE_MAPPING)
121412f080e7Smrj return (0);
121512f080e7Smrj
12161f0c5e61SRobert Mustacchi ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC);
12171f0c5e61SRobert Mustacchi rp = (struct regspec64 *)mp->map_obj.rp;
121812f080e7Smrj
121912f080e7Smrj if (rp->regspec_size == 0) {
122012f080e7Smrj #ifdef DDI_MAP_DEBUG
122112f080e7Smrj ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n");
122212f080e7Smrj #endif /* DDI_MAP_DEBUG */
122312f080e7Smrj return (DDI_ME_INVAL);
122412f080e7Smrj }
122512f080e7Smrj
122612f080e7Smrj /*
122712f080e7Smrj * I/O or memory mapping:
122812f080e7Smrj *
122912f080e7Smrj * <bustype=0, addr=x, len=x>: memory
123012f080e7Smrj * <bustype=1, addr=x, len=x>: i/o
123112f080e7Smrj * <bustype>1, addr=0, len=x>: x86-compatibility i/o
123212f080e7Smrj */
123312f080e7Smrj if (rp->regspec_bustype != 0) {
123412f080e7Smrj /*
123512f080e7Smrj * This is I/O space, which requires no particular
123612f080e7Smrj * processing on unmap since it isn't mapped in the
123712f080e7Smrj * first place.
123812f080e7Smrj */
123912f080e7Smrj return (DDI_SUCCESS);
124012f080e7Smrj }
124112f080e7Smrj
124212f080e7Smrj /*
124312f080e7Smrj * Memory space
124412f080e7Smrj */
124512f080e7Smrj pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET;
124612f080e7Smrj npages = mmu_btopr(rp->regspec_size + pgoffset);
124712f080e7Smrj hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK);
124812f080e7Smrj device_arena_free(addr - pgoffset, ptob(npages));
124912f080e7Smrj
125012f080e7Smrj /*
125112f080e7Smrj * Destroy the pointer - the mapping has logically gone
125212f080e7Smrj */
125312f080e7Smrj *vaddrp = NULL;
125412f080e7Smrj
125512f080e7Smrj return (DDI_SUCCESS);
125612f080e7Smrj }
125712f080e7Smrj
125812f080e7Smrj static int
rootnex_map_handle(ddi_map_req_t * mp)125912f080e7Smrj rootnex_map_handle(ddi_map_req_t *mp)
126012f080e7Smrj {
1261843e1988Sjohnlev rootnex_addr_t rbase;
126212f080e7Smrj ddi_acc_hdl_t *hp;
12631f0c5e61SRobert Mustacchi uint64_t pgoffset;
12641f0c5e61SRobert Mustacchi struct regspec64 *rp;
1265843e1988Sjohnlev paddr_t pbase;
126612f080e7Smrj
12671f0c5e61SRobert Mustacchi rp = (struct regspec64 *)mp->map_obj.rp;
126812f080e7Smrj
126912f080e7Smrj #ifdef DDI_MAP_DEBUG
127012f080e7Smrj ddi_map_debug(
127112f080e7Smrj "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n",
127212f080e7Smrj rp->regspec_bustype, rp->regspec_addr,
127312f080e7Smrj rp->regspec_size, mp->map_handlep);
127412f080e7Smrj #endif /* DDI_MAP_DEBUG */
127512f080e7Smrj
127612f080e7Smrj /*
127712f080e7Smrj * I/O or memory mapping:
127812f080e7Smrj *
127912f080e7Smrj * <bustype=0, addr=x, len=x>: memory
128012f080e7Smrj * <bustype=1, addr=x, len=x>: i/o
128112f080e7Smrj * <bustype>1, addr=0, len=x>: x86-compatibility i/o
128212f080e7Smrj */
128312f080e7Smrj if (rp->regspec_bustype != 0) {
128412f080e7Smrj /*
128512f080e7Smrj * This refers to I/O space, and we don't support "mapping"
128612f080e7Smrj * I/O space to a user.
128712f080e7Smrj */
128812f080e7Smrj return (DDI_FAILURE);
128912f080e7Smrj }
129012f080e7Smrj
129112f080e7Smrj /*
129212f080e7Smrj * Set up the hat_flags for the mapping.
129312f080e7Smrj */
129412f080e7Smrj hp = mp->map_handlep;
129512f080e7Smrj
129612f080e7Smrj switch (hp->ah_acc.devacc_attr_endian_flags) {
129712f080e7Smrj case DDI_NEVERSWAP_ACC:
129812f080e7Smrj hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER;
129912f080e7Smrj break;
130012f080e7Smrj case DDI_STRUCTURE_LE_ACC:
130112f080e7Smrj hp->ah_hat_flags = HAT_STRUCTURE_LE;
130212f080e7Smrj break;
130312f080e7Smrj case DDI_STRUCTURE_BE_ACC:
130412f080e7Smrj return (DDI_FAILURE);
130512f080e7Smrj default:
130612f080e7Smrj return (DDI_REGS_ACC_CONFLICT);
130712f080e7Smrj }
130812f080e7Smrj
130912f080e7Smrj switch (hp->ah_acc.devacc_attr_dataorder) {
131012f080e7Smrj case DDI_STRICTORDER_ACC:
131112f080e7Smrj break;
131212f080e7Smrj case DDI_UNORDERED_OK_ACC:
131312f080e7Smrj hp->ah_hat_flags |= HAT_UNORDERED_OK;
131412f080e7Smrj break;
131512f080e7Smrj case DDI_MERGING_OK_ACC:
131612f080e7Smrj hp->ah_hat_flags |= HAT_MERGING_OK;
131712f080e7Smrj break;
131812f080e7Smrj case DDI_LOADCACHING_OK_ACC:
131912f080e7Smrj hp->ah_hat_flags |= HAT_LOADCACHING_OK;
132012f080e7Smrj break;
132112f080e7Smrj case DDI_STORECACHING_OK_ACC:
132212f080e7Smrj hp->ah_hat_flags |= HAT_STORECACHING_OK;
132312f080e7Smrj break;
132412f080e7Smrj default:
132512f080e7Smrj return (DDI_FAILURE);
132612f080e7Smrj }
132712f080e7Smrj
1328843e1988Sjohnlev rbase = (rootnex_addr_t)rp->regspec_addr &
1329843e1988Sjohnlev (~(rootnex_addr_t)MMU_PAGEOFFSET);
1330843e1988Sjohnlev pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
133112f080e7Smrj
133212f080e7Smrj if (rp->regspec_size == 0)
133312f080e7Smrj return (DDI_ME_INVAL);
133412f080e7Smrj
1335843e1988Sjohnlev #ifdef __xpv
1336843e1988Sjohnlev /*
1337843e1988Sjohnlev * If we're dom0, we're using a real device so we need to translate
1338843e1988Sjohnlev * the MA to a PA.
1339843e1988Sjohnlev */
1340843e1988Sjohnlev if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1341843e1988Sjohnlev pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) |
1342843e1988Sjohnlev (rbase & MMU_PAGEOFFSET);
1343843e1988Sjohnlev } else {
1344843e1988Sjohnlev pbase = rbase;
1345843e1988Sjohnlev }
1346843e1988Sjohnlev #else
1347843e1988Sjohnlev pbase = rbase;
1348843e1988Sjohnlev #endif
1349843e1988Sjohnlev
1350843e1988Sjohnlev hp->ah_pfn = mmu_btop(pbase);
135112f080e7Smrj hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset);
135212f080e7Smrj
135312f080e7Smrj return (DDI_SUCCESS);
135412f080e7Smrj }
135512f080e7Smrj
135612f080e7Smrj
135712f080e7Smrj
135812f080e7Smrj /*
135912f080e7Smrj * ************************
136012f080e7Smrj * interrupt related code
136112f080e7Smrj * ************************
136212f080e7Smrj */
136312f080e7Smrj
136412f080e7Smrj /*
136512f080e7Smrj * rootnex_intr_ops()
136612f080e7Smrj * bus_intr_op() function for interrupt support
136712f080e7Smrj */
136812f080e7Smrj /* ARGSUSED */
136912f080e7Smrj static int
rootnex_intr_ops(dev_info_t * pdip,dev_info_t * rdip,ddi_intr_op_t intr_op,ddi_intr_handle_impl_t * hdlp,void * result)137012f080e7Smrj rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
137112f080e7Smrj ddi_intr_handle_impl_t *hdlp, void *result)
137212f080e7Smrj {
137312f080e7Smrj struct intrspec *ispec;
137412f080e7Smrj
137512f080e7Smrj DDI_INTR_NEXDBG((CE_CONT,
137612f080e7Smrj "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n",
137712f080e7Smrj (void *)pdip, (void *)rdip, intr_op, (void *)hdlp));
137812f080e7Smrj
137912f080e7Smrj /* Process the interrupt operation */
138012f080e7Smrj switch (intr_op) {
138112f080e7Smrj case DDI_INTROP_GETCAP:
138212f080e7Smrj /* First check with pcplusmp */
138312f080e7Smrj if (psm_intr_ops == NULL)
138412f080e7Smrj return (DDI_FAILURE);
138512f080e7Smrj
138612f080e7Smrj if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) {
138712f080e7Smrj *(int *)result = 0;
138812f080e7Smrj return (DDI_FAILURE);
138912f080e7Smrj }
139012f080e7Smrj break;
139112f080e7Smrj case DDI_INTROP_SETCAP:
139212f080e7Smrj if (psm_intr_ops == NULL)
139312f080e7Smrj return (DDI_FAILURE);
139412f080e7Smrj
139512f080e7Smrj if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result))
139612f080e7Smrj return (DDI_FAILURE);
139712f080e7Smrj break;
139812f080e7Smrj case DDI_INTROP_ALLOC:
13997ff178cdSJimmy Vetayases ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
14007ff178cdSJimmy Vetayases return (rootnex_alloc_intr_fixed(rdip, hdlp, result));
140112f080e7Smrj case DDI_INTROP_FREE:
14027ff178cdSJimmy Vetayases ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
14037ff178cdSJimmy Vetayases return (rootnex_free_intr_fixed(rdip, hdlp));
140412f080e7Smrj case DDI_INTROP_GETPRI:
140512f080e7Smrj if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
140612f080e7Smrj return (DDI_FAILURE);
140712f080e7Smrj *(int *)result = ispec->intrspec_pri;
140812f080e7Smrj break;
140912f080e7Smrj case DDI_INTROP_SETPRI:
141012f080e7Smrj /* Validate the interrupt priority passed to us */
141112f080e7Smrj if (*(int *)result > LOCK_LEVEL)
141212f080e7Smrj return (DDI_FAILURE);
141312f080e7Smrj
141412f080e7Smrj /* Ensure that PSM is all initialized and ispec is ok */
141512f080e7Smrj if ((psm_intr_ops == NULL) ||
141612f080e7Smrj ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL))
141712f080e7Smrj return (DDI_FAILURE);
141812f080e7Smrj
141912f080e7Smrj /* Change the priority */
142012f080e7Smrj if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) ==
142112f080e7Smrj PSM_FAILURE)
142212f080e7Smrj return (DDI_FAILURE);
142312f080e7Smrj
142412f080e7Smrj /* update the ispec with the new priority */
142512f080e7Smrj ispec->intrspec_pri = *(int *)result;
142612f080e7Smrj break;
142712f080e7Smrj case DDI_INTROP_ADDISR:
142812f080e7Smrj if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
142912f080e7Smrj return (DDI_FAILURE);
143012f080e7Smrj ispec->intrspec_func = hdlp->ih_cb_func;
143112f080e7Smrj break;
143212f080e7Smrj case DDI_INTROP_REMISR:
143312f080e7Smrj if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
143412f080e7Smrj return (DDI_FAILURE);
143512f080e7Smrj ispec->intrspec_func = (uint_t (*)()) 0;
143612f080e7Smrj break;
143712f080e7Smrj case DDI_INTROP_ENABLE:
143812f080e7Smrj if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
143912f080e7Smrj return (DDI_FAILURE);
144012f080e7Smrj
144112f080e7Smrj /* Call psmi to translate irq with the dip */
144212f080e7Smrj if (psm_intr_ops == NULL)
144312f080e7Smrj return (DDI_FAILURE);
144412f080e7Smrj
14457a364d25Sschwartz ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
144686a9c507SGuoli Shu if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR,
144786a9c507SGuoli Shu (int *)&hdlp->ih_vector) == PSM_FAILURE)
144886a9c507SGuoli Shu return (DDI_FAILURE);
144912f080e7Smrj
145012f080e7Smrj /* Add the interrupt handler */
145112f080e7Smrj if (!add_avintr((void *)hdlp, ispec->intrspec_pri,
145212f080e7Smrj hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector,
14537a364d25Sschwartz hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip))
145412f080e7Smrj return (DDI_FAILURE);
145512f080e7Smrj break;
145612f080e7Smrj case DDI_INTROP_DISABLE:
145712f080e7Smrj if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
145812f080e7Smrj return (DDI_FAILURE);
145912f080e7Smrj
146012f080e7Smrj /* Call psm_ops() to translate irq with the dip */
146112f080e7Smrj if (psm_intr_ops == NULL)
146212f080e7Smrj return (DDI_FAILURE);
146312f080e7Smrj
14647a364d25Sschwartz ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
146512f080e7Smrj (void) (*psm_intr_ops)(rdip, hdlp,
146612f080e7Smrj PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector);
146712f080e7Smrj
146812f080e7Smrj /* Remove the interrupt handler */
146912f080e7Smrj rem_avintr((void *)hdlp, ispec->intrspec_pri,
147012f080e7Smrj hdlp->ih_cb_func, hdlp->ih_vector);
147112f080e7Smrj break;
147212f080e7Smrj case DDI_INTROP_SETMASK:
147312f080e7Smrj if (psm_intr_ops == NULL)
147412f080e7Smrj return (DDI_FAILURE);
147512f080e7Smrj
147612f080e7Smrj if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL))
147712f080e7Smrj return (DDI_FAILURE);
147812f080e7Smrj break;
147912f080e7Smrj case DDI_INTROP_CLRMASK:
148012f080e7Smrj if (psm_intr_ops == NULL)
148112f080e7Smrj return (DDI_FAILURE);
148212f080e7Smrj
148312f080e7Smrj if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL))
148412f080e7Smrj return (DDI_FAILURE);
148512f080e7Smrj break;
148612f080e7Smrj case DDI_INTROP_GETPENDING:
148712f080e7Smrj if (psm_intr_ops == NULL)
148812f080e7Smrj return (DDI_FAILURE);
148912f080e7Smrj
149012f080e7Smrj if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING,
149112f080e7Smrj result)) {
149212f080e7Smrj *(int *)result = 0;
149312f080e7Smrj return (DDI_FAILURE);
149412f080e7Smrj }
149512f080e7Smrj break;
1496a54f81fbSanish case DDI_INTROP_NAVAIL:
149712f080e7Smrj case DDI_INTROP_NINTRS:
1498a54f81fbSanish *(int *)result = i_ddi_get_intx_nintrs(rdip);
1499a54f81fbSanish if (*(int *)result == 0) {
150012f080e7Smrj /*
150112f080e7Smrj * Special case for 'pcic' driver' only. This driver
150212f080e7Smrj * driver is a child of 'isa' and 'rootnex' drivers.
150312f080e7Smrj *
150412f080e7Smrj * See detailed comments on this in the function
150512f080e7Smrj * rootnex_get_ispec().
150612f080e7Smrj *
150712f080e7Smrj * Children of 'pcic' send 'NINITR' request all the
150812f080e7Smrj * way to rootnex driver. But, the 'pdp->par_nintr'
150912f080e7Smrj * field may not initialized. So, we fake it here
151012f080e7Smrj * to return 1 (a la what PCMCIA nexus does).
151112f080e7Smrj */
151212f080e7Smrj if (strcmp(ddi_get_name(rdip), "pcic") == 0)
151312f080e7Smrj *(int *)result = 1;
1514a54f81fbSanish else
1515a54f81fbSanish return (DDI_FAILURE);
151612f080e7Smrj }
151712f080e7Smrj break;
151812f080e7Smrj case DDI_INTROP_SUPPORTED_TYPES:
1519a54f81fbSanish *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */
152012f080e7Smrj break;
152112f080e7Smrj default:
152212f080e7Smrj return (DDI_FAILURE);
152312f080e7Smrj }
152412f080e7Smrj
152512f080e7Smrj return (DDI_SUCCESS);
152612f080e7Smrj }
152712f080e7Smrj
152812f080e7Smrj
152912f080e7Smrj /*
153012f080e7Smrj * rootnex_get_ispec()
153112f080e7Smrj * convert an interrupt number to an interrupt specification.
153212f080e7Smrj * The interrupt number determines which interrupt spec will be
153312f080e7Smrj * returned if more than one exists.
153412f080e7Smrj *
153512f080e7Smrj * Look into the parent private data area of the 'rdip' to find out
153612f080e7Smrj * the interrupt specification. First check to make sure there is
153712f080e7Smrj * one that matchs "inumber" and then return a pointer to it.
153812f080e7Smrj *
153912f080e7Smrj * Return NULL if one could not be found.
154012f080e7Smrj *
154112f080e7Smrj * NOTE: This is needed for rootnex_intr_ops()
154212f080e7Smrj */
154312f080e7Smrj static struct intrspec *
rootnex_get_ispec(dev_info_t * rdip,int inum)154412f080e7Smrj rootnex_get_ispec(dev_info_t *rdip, int inum)
154512f080e7Smrj {
154612f080e7Smrj struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip);
154712f080e7Smrj
154812f080e7Smrj /*
154912f080e7Smrj * Special case handling for drivers that provide their own
155012f080e7Smrj * intrspec structures instead of relying on the DDI framework.
155112f080e7Smrj *
155212f080e7Smrj * A broken hardware driver in ON could potentially provide its
155312f080e7Smrj * own intrspec structure, instead of relying on the hardware.
155412f080e7Smrj * If these drivers are children of 'rootnex' then we need to
155512f080e7Smrj * continue to provide backward compatibility to them here.
155612f080e7Smrj *
155712f080e7Smrj * Following check is a special case for 'pcic' driver which
155812f080e7Smrj * was found to have broken hardwre andby provides its own intrspec.
155912f080e7Smrj *
156012f080e7Smrj * Verbatim comments from this driver are shown here:
156112f080e7Smrj * "Don't use the ddi_add_intr since we don't have a
156212f080e7Smrj * default intrspec in all cases."
156312f080e7Smrj *
156412f080e7Smrj * Since an 'ispec' may not be always created for it,
156512f080e7Smrj * check for that and create one if so.
156612f080e7Smrj *
156712f080e7Smrj * NOTE: Currently 'pcic' is the only driver found to do this.
156812f080e7Smrj */
156912f080e7Smrj if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
157012f080e7Smrj pdp->par_nintr = 1;
157112f080e7Smrj pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) *
157212f080e7Smrj pdp->par_nintr, KM_SLEEP);
157312f080e7Smrj }
157412f080e7Smrj
157512f080e7Smrj /* Validate the interrupt number */
157612f080e7Smrj if (inum >= pdp->par_nintr)
157712f080e7Smrj return (NULL);
157812f080e7Smrj
157912f080e7Smrj /* Get the interrupt structure pointer and return that */
158012f080e7Smrj return ((struct intrspec *)&pdp->par_intr[inum]);
158112f080e7Smrj }
158212f080e7Smrj
15837ff178cdSJimmy Vetayases /*
15847ff178cdSJimmy Vetayases * Allocate interrupt vector for FIXED (legacy) type.
15857ff178cdSJimmy Vetayases */
15867ff178cdSJimmy Vetayases static int
rootnex_alloc_intr_fixed(dev_info_t * rdip,ddi_intr_handle_impl_t * hdlp,void * result)15877ff178cdSJimmy Vetayases rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp,
15887ff178cdSJimmy Vetayases void *result)
15897ff178cdSJimmy Vetayases {
15907ff178cdSJimmy Vetayases struct intrspec *ispec;
15917ff178cdSJimmy Vetayases ddi_intr_handle_impl_t info_hdl;
15927ff178cdSJimmy Vetayases int ret;
15937ff178cdSJimmy Vetayases int free_phdl = 0;
15947ff178cdSJimmy Vetayases apic_get_type_t type_info;
15957ff178cdSJimmy Vetayases
15967ff178cdSJimmy Vetayases if (psm_intr_ops == NULL)
15977ff178cdSJimmy Vetayases return (DDI_FAILURE);
15987ff178cdSJimmy Vetayases
15997ff178cdSJimmy Vetayases if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
16007ff178cdSJimmy Vetayases return (DDI_FAILURE);
16017ff178cdSJimmy Vetayases
16027ff178cdSJimmy Vetayases /*
16037ff178cdSJimmy Vetayases * If the PSM module is "APIX" then pass the request for it
16047ff178cdSJimmy Vetayases * to allocate the vector now.
16057ff178cdSJimmy Vetayases */
16067ff178cdSJimmy Vetayases bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
16077ff178cdSJimmy Vetayases info_hdl.ih_private = &type_info;
16087ff178cdSJimmy Vetayases if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
16097ff178cdSJimmy Vetayases PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
16107ff178cdSJimmy Vetayases if (hdlp->ih_private == NULL) { /* allocate phdl structure */
16117ff178cdSJimmy Vetayases free_phdl = 1;
16127ff178cdSJimmy Vetayases i_ddi_alloc_intr_phdl(hdlp);
16137ff178cdSJimmy Vetayases }
16147ff178cdSJimmy Vetayases ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
16157ff178cdSJimmy Vetayases ret = (*psm_intr_ops)(rdip, hdlp,
16167ff178cdSJimmy Vetayases PSM_INTR_OP_ALLOC_VECTORS, result);
16177ff178cdSJimmy Vetayases if (free_phdl) { /* free up the phdl structure */
16187ff178cdSJimmy Vetayases free_phdl = 0;
16197ff178cdSJimmy Vetayases i_ddi_free_intr_phdl(hdlp);
16207ff178cdSJimmy Vetayases hdlp->ih_private = NULL;
16217ff178cdSJimmy Vetayases }
16227ff178cdSJimmy Vetayases } else {
16237ff178cdSJimmy Vetayases /*
16247ff178cdSJimmy Vetayases * No APIX module; fall back to the old scheme where the
16257ff178cdSJimmy Vetayases * interrupt vector is allocated during ddi_enable_intr() call.
16267ff178cdSJimmy Vetayases */
16277ff178cdSJimmy Vetayases hdlp->ih_pri = ispec->intrspec_pri;
16287ff178cdSJimmy Vetayases *(int *)result = hdlp->ih_scratch1;
16297ff178cdSJimmy Vetayases ret = DDI_SUCCESS;
16307ff178cdSJimmy Vetayases }
16317ff178cdSJimmy Vetayases
16327ff178cdSJimmy Vetayases return (ret);
16337ff178cdSJimmy Vetayases }
16347ff178cdSJimmy Vetayases
16357ff178cdSJimmy Vetayases /*
16367ff178cdSJimmy Vetayases * Free up interrupt vector for FIXED (legacy) type.
16377ff178cdSJimmy Vetayases */
16387ff178cdSJimmy Vetayases static int
rootnex_free_intr_fixed(dev_info_t * rdip,ddi_intr_handle_impl_t * hdlp)16397ff178cdSJimmy Vetayases rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp)
16407ff178cdSJimmy Vetayases {
16417ff178cdSJimmy Vetayases struct intrspec *ispec;
16427ff178cdSJimmy Vetayases struct ddi_parent_private_data *pdp;
16437ff178cdSJimmy Vetayases ddi_intr_handle_impl_t info_hdl;
16447ff178cdSJimmy Vetayases int ret;
16457ff178cdSJimmy Vetayases apic_get_type_t type_info;
16467ff178cdSJimmy Vetayases
16477ff178cdSJimmy Vetayases if (psm_intr_ops == NULL)
16487ff178cdSJimmy Vetayases return (DDI_FAILURE);
16497ff178cdSJimmy Vetayases
16507ff178cdSJimmy Vetayases /*
16517ff178cdSJimmy Vetayases * If the PSM module is "APIX" then pass the request for it
16527ff178cdSJimmy Vetayases * to free up the vector now.
16537ff178cdSJimmy Vetayases */
16547ff178cdSJimmy Vetayases bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
16557ff178cdSJimmy Vetayases info_hdl.ih_private = &type_info;
16567ff178cdSJimmy Vetayases if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
16577ff178cdSJimmy Vetayases PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
16587ff178cdSJimmy Vetayases if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
16597ff178cdSJimmy Vetayases return (DDI_FAILURE);
16607ff178cdSJimmy Vetayases ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
16617ff178cdSJimmy Vetayases ret = (*psm_intr_ops)(rdip, hdlp,
16627ff178cdSJimmy Vetayases PSM_INTR_OP_FREE_VECTORS, NULL);
16637ff178cdSJimmy Vetayases } else {
16647ff178cdSJimmy Vetayases /*
16657ff178cdSJimmy Vetayases * No APIX module; fall back to the old scheme where
16667ff178cdSJimmy Vetayases * the interrupt vector was already freed during
16677ff178cdSJimmy Vetayases * ddi_disable_intr() call.
16687ff178cdSJimmy Vetayases */
16697ff178cdSJimmy Vetayases ret = DDI_SUCCESS;
16707ff178cdSJimmy Vetayases }
16717ff178cdSJimmy Vetayases
16727ff178cdSJimmy Vetayases pdp = ddi_get_parent_data(rdip);
16737ff178cdSJimmy Vetayases
16747ff178cdSJimmy Vetayases /*
16757ff178cdSJimmy Vetayases * Special case for 'pcic' driver' only.
16767ff178cdSJimmy Vetayases * If an intrspec was created for it, clean it up here
16777ff178cdSJimmy Vetayases * See detailed comments on this in the function
16787ff178cdSJimmy Vetayases * rootnex_get_ispec().
16797ff178cdSJimmy Vetayases */
16807ff178cdSJimmy Vetayases if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
16817ff178cdSJimmy Vetayases kmem_free(pdp->par_intr, sizeof (struct intrspec) *
16827ff178cdSJimmy Vetayases pdp->par_nintr);
16837ff178cdSJimmy Vetayases /*
16847ff178cdSJimmy Vetayases * Set it to zero; so that
16857ff178cdSJimmy Vetayases * DDI framework doesn't free it again
16867ff178cdSJimmy Vetayases */
16877ff178cdSJimmy Vetayases pdp->par_intr = NULL;
16887ff178cdSJimmy Vetayases pdp->par_nintr = 0;
16897ff178cdSJimmy Vetayases }
16907ff178cdSJimmy Vetayases
16917ff178cdSJimmy Vetayases return (ret);
16927ff178cdSJimmy Vetayases }
16937ff178cdSJimmy Vetayases
169412f080e7Smrj
16957c478bd9Sstevel@tonic-gate /*
169612f080e7Smrj * ******************
169712f080e7Smrj * dma related code
169812f080e7Smrj * ******************
169912f080e7Smrj */
170012f080e7Smrj
17017c478bd9Sstevel@tonic-gate /*ARGSUSED*/
17027c478bd9Sstevel@tonic-gate static int
rootnex_coredma_allochdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_attr_t * attr,int (* waitfp)(caddr_t),caddr_t arg,ddi_dma_handle_t * handlep)170320906b23SVikram Hegde rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
170420906b23SVikram Hegde ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
170520906b23SVikram Hegde ddi_dma_handle_t *handlep)
17067c478bd9Sstevel@tonic-gate {
170712f080e7Smrj uint64_t maxsegmentsize_ll;
170812f080e7Smrj uint_t maxsegmentsize;
170912f080e7Smrj ddi_dma_impl_t *hp;
171012f080e7Smrj rootnex_dma_t *dma;
171112f080e7Smrj uint64_t count_max;
171212f080e7Smrj uint64_t seg;
171312f080e7Smrj int kmflag;
171412f080e7Smrj int e;
17157c478bd9Sstevel@tonic-gate
171612f080e7Smrj
171712f080e7Smrj /* convert our sleep flags */
171812f080e7Smrj if (waitfp == DDI_DMA_SLEEP) {
171912f080e7Smrj kmflag = KM_SLEEP;
172012f080e7Smrj } else {
172112f080e7Smrj kmflag = KM_NOSLEEP;
172212f080e7Smrj }
17237c478bd9Sstevel@tonic-gate
17247c478bd9Sstevel@tonic-gate /*
172512f080e7Smrj * We try to do only one memory allocation here. We'll do a little
172612f080e7Smrj * pointer manipulation later. If the bind ends up taking more than
172712f080e7Smrj * our prealloc's space, we'll have to allocate more memory in the
172812f080e7Smrj * bind operation. Not great, but much better than before and the
172912f080e7Smrj * best we can do with the current bind interfaces.
173012f080e7Smrj */
173112f080e7Smrj hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag);
173250200e77SFrank Van Der Linden if (hp == NULL)
173312f080e7Smrj return (DDI_DMA_NORESOURCES);
173412f080e7Smrj
173512f080e7Smrj /* Do our pointer manipulation now, align the structures */
173612f080e7Smrj hp->dmai_private = (void *)(((uintptr_t)hp +
173712f080e7Smrj (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7);
173812f080e7Smrj dma = (rootnex_dma_t *)hp->dmai_private;
173912f080e7Smrj dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma +
174012f080e7Smrj sizeof (rootnex_dma_t) + 0x7) & ~0x7);
174112f080e7Smrj
174212f080e7Smrj /* setup the handle */
174312f080e7Smrj rootnex_clean_dmahdl(hp);
1744567c0b92SStephen Hanson hp->dmai_error.err_fep = NULL;
1745567c0b92SStephen Hanson hp->dmai_error.err_cf = NULL;
174612f080e7Smrj dma->dp_dip = rdip;
174750200e77SFrank Van Der Linden dma->dp_sglinfo.si_flags = attr->dma_attr_flags;
174812f080e7Smrj dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo;
174950200e77SFrank Van Der Linden
175050200e77SFrank Van Der Linden /*
175150200e77SFrank Van Der Linden * The BOUNCE_ON_SEG workaround is not needed when an IOMMU
175250200e77SFrank Van Der Linden * is being used. Set the upper limit to the seg value.
175350200e77SFrank Van Der Linden * There will be enough DVMA space to always get addresses
175450200e77SFrank Van Der Linden * that will match the constraints.
175550200e77SFrank Van Der Linden */
175650200e77SFrank Van Der Linden if (IOMMU_USED(rdip) &&
175750200e77SFrank Van Der Linden (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) {
175850200e77SFrank Van Der Linden dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg;
175950200e77SFrank Van Der Linden dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG;
176050200e77SFrank Van Der Linden } else
176150200e77SFrank Van Der Linden dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi;
176250200e77SFrank Van Der Linden
176312f080e7Smrj hp->dmai_minxfer = attr->dma_attr_minxfer;
176412f080e7Smrj hp->dmai_burstsizes = attr->dma_attr_burstsizes;
176512f080e7Smrj hp->dmai_rdip = rdip;
176612f080e7Smrj hp->dmai_attr = *attr;
176712f080e7Smrj
176850200e77SFrank Van Der Linden if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr)
176950200e77SFrank Van Der Linden dma->dp_sglinfo.si_cancross = B_FALSE;
177050200e77SFrank Van Der Linden else
177150200e77SFrank Van Der Linden dma->dp_sglinfo.si_cancross = B_TRUE;
177250200e77SFrank Van Der Linden
177312f080e7Smrj /* we don't need to worry about the SPL since we do a tryenter */
177412f080e7Smrj mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL);
177512f080e7Smrj
177612f080e7Smrj /*
177712f080e7Smrj * Figure out our maximum segment size. If the segment size is greater
177812f080e7Smrj * than 4G, we will limit it to (4G - 1) since the max size of a dma
177912f080e7Smrj * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and
178012f080e7Smrj * dma_attr_count_max are size-1 type values.
17817c478bd9Sstevel@tonic-gate *
178212f080e7Smrj * Maximum segment size is the largest physically contiguous chunk of
178312f080e7Smrj * memory that we can return from a bind (i.e. the maximum size of a
178412f080e7Smrj * single cookie).
17857c478bd9Sstevel@tonic-gate */
17867c478bd9Sstevel@tonic-gate
178712f080e7Smrj /* handle the rollover cases */
178812f080e7Smrj seg = attr->dma_attr_seg + 1;
178912f080e7Smrj if (seg < attr->dma_attr_seg) {
179012f080e7Smrj seg = attr->dma_attr_seg;
179112f080e7Smrj }
179212f080e7Smrj count_max = attr->dma_attr_count_max + 1;
179312f080e7Smrj if (count_max < attr->dma_attr_count_max) {
179412f080e7Smrj count_max = attr->dma_attr_count_max;
179512f080e7Smrj }
179612f080e7Smrj
179712f080e7Smrj /*
179812f080e7Smrj * granularity may or may not be a power of two. If it isn't, we can't
179912f080e7Smrj * use a simple mask.
180012f080e7Smrj */
1801de710d24SJosef 'Jeff' Sipek if (!ISP2(attr->dma_attr_granular)) {
180212f080e7Smrj dma->dp_granularity_power_2 = B_FALSE;
180312f080e7Smrj } else {
180412f080e7Smrj dma->dp_granularity_power_2 = B_TRUE;
180512f080e7Smrj }
180612f080e7Smrj
180712f080e7Smrj /*
180812f080e7Smrj * maxxfer should be a whole multiple of granularity. If we're going to
180912f080e7Smrj * break up a window because we're greater than maxxfer, we might as
181012f080e7Smrj * well make sure it's maxxfer is a whole multiple so we don't have to
181112f080e7Smrj * worry about triming the window later on for this case.
181212f080e7Smrj */
181312f080e7Smrj if (attr->dma_attr_granular > 1) {
181412f080e7Smrj if (dma->dp_granularity_power_2) {
181512f080e7Smrj dma->dp_maxxfer = attr->dma_attr_maxxfer -
181612f080e7Smrj (attr->dma_attr_maxxfer &
181712f080e7Smrj (attr->dma_attr_granular - 1));
181812f080e7Smrj } else {
181912f080e7Smrj dma->dp_maxxfer = attr->dma_attr_maxxfer -
182012f080e7Smrj (attr->dma_attr_maxxfer % attr->dma_attr_granular);
18217c478bd9Sstevel@tonic-gate }
182212f080e7Smrj } else {
182312f080e7Smrj dma->dp_maxxfer = attr->dma_attr_maxxfer;
182412f080e7Smrj }
182512f080e7Smrj
182612f080e7Smrj maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer);
182712f080e7Smrj maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max);
182812f080e7Smrj if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) {
182912f080e7Smrj maxsegmentsize = 0xFFFFFFFF;
183012f080e7Smrj } else {
183112f080e7Smrj maxsegmentsize = maxsegmentsize_ll;
183212f080e7Smrj }
183312f080e7Smrj dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize;
183412f080e7Smrj dma->dp_sglinfo.si_segmask = attr->dma_attr_seg;
183512f080e7Smrj
183612f080e7Smrj /* check the ddi_dma_attr arg to make sure it makes a little sense */
183712f080e7Smrj if (rootnex_alloc_check_parms) {
183812f080e7Smrj e = rootnex_valid_alloc_parms(attr, maxsegmentsize);
183912f080e7Smrj if (e != DDI_SUCCESS) {
1840ef4ab52fSFrank Van Der Linden ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]);
184112f080e7Smrj (void) rootnex_dma_freehdl(dip, rdip,
184212f080e7Smrj (ddi_dma_handle_t)hp);
184312f080e7Smrj return (e);
184412f080e7Smrj }
184512f080e7Smrj }
184612f080e7Smrj
184712f080e7Smrj *handlep = (ddi_dma_handle_t)hp;
184812f080e7Smrj
18490b7ba611SMark Johnson ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
18500b7ba611SMark Johnson ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t,
185112f080e7Smrj rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
185212f080e7Smrj
18537c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
18547c478bd9Sstevel@tonic-gate }
18557c478bd9Sstevel@tonic-gate
18567c478bd9Sstevel@tonic-gate
18577c478bd9Sstevel@tonic-gate /*
185820906b23SVikram Hegde * rootnex_dma_allochdl()
185920906b23SVikram Hegde * called from ddi_dma_alloc_handle().
18607c478bd9Sstevel@tonic-gate */
186120906b23SVikram Hegde static int
rootnex_dma_allochdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_attr_t * attr,int (* waitfp)(caddr_t),caddr_t arg,ddi_dma_handle_t * handlep)186220906b23SVikram Hegde rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
186320906b23SVikram Hegde int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep)
186420906b23SVikram Hegde {
186550200e77SFrank Van Der Linden int retval = DDI_SUCCESS;
186686ef0a63SRichard Lowe #if !defined(__xpv)
186720906b23SVikram Hegde
186850200e77SFrank Van Der Linden if (IOMMU_UNITIALIZED(rdip)) {
186950200e77SFrank Van Der Linden retval = iommulib_nex_open(dip, rdip);
187020906b23SVikram Hegde
187150200e77SFrank Van Der Linden if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP)
187250200e77SFrank Van Der Linden return (retval);
187320906b23SVikram Hegde }
187420906b23SVikram Hegde
187550200e77SFrank Van Der Linden if (IOMMU_UNUSED(rdip)) {
187650200e77SFrank Van Der Linden retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
187750200e77SFrank Van Der Linden handlep);
187850200e77SFrank Van Der Linden } else {
187950200e77SFrank Van Der Linden retval = iommulib_nexdma_allochdl(dip, rdip, attr,
188050200e77SFrank Van Der Linden waitfp, arg, handlep);
188150200e77SFrank Van Der Linden }
188220906b23SVikram Hegde #else
1883567c0b92SStephen Hanson retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1884567c0b92SStephen Hanson handlep);
188520906b23SVikram Hegde #endif
188650200e77SFrank Van Der Linden switch (retval) {
188750200e77SFrank Van Der Linden case DDI_DMA_NORESOURCES:
188850200e77SFrank Van Der Linden if (waitfp != DDI_DMA_DONTWAIT) {
188950200e77SFrank Van Der Linden ddi_set_callback(waitfp, arg,
189050200e77SFrank Van Der Linden &rootnex_state->r_dvma_call_list_id);
189150200e77SFrank Van Der Linden }
189250200e77SFrank Van Der Linden break;
189350200e77SFrank Van Der Linden case DDI_SUCCESS:
1894567c0b92SStephen Hanson ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL);
189550200e77SFrank Van Der Linden break;
189650200e77SFrank Van Der Linden default:
189750200e77SFrank Van Der Linden break;
189850200e77SFrank Van Der Linden }
1899567c0b92SStephen Hanson return (retval);
190020906b23SVikram Hegde }
190120906b23SVikram Hegde
190212f080e7Smrj /*ARGSUSED*/
190312f080e7Smrj static int
rootnex_coredma_freehdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle)190420906b23SVikram Hegde rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
190520906b23SVikram Hegde ddi_dma_handle_t handle)
190612f080e7Smrj {
190712f080e7Smrj ddi_dma_impl_t *hp;
190812f080e7Smrj rootnex_dma_t *dma;
19097c478bd9Sstevel@tonic-gate
19107c478bd9Sstevel@tonic-gate
191112f080e7Smrj hp = (ddi_dma_impl_t *)handle;
191212f080e7Smrj dma = (rootnex_dma_t *)hp->dmai_private;
191312f080e7Smrj
191412f080e7Smrj /* unbind should have been called first */
191512f080e7Smrj ASSERT(!dma->dp_inuse);
191612f080e7Smrj
191712f080e7Smrj mutex_destroy(&dma->dp_mutex);
191812f080e7Smrj kmem_cache_free(rootnex_state->r_dmahdl_cache, hp);
191912f080e7Smrj
19200b7ba611SMark Johnson ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
19210b7ba611SMark Johnson ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t,
192212f080e7Smrj rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
192312f080e7Smrj
192412f080e7Smrj return (DDI_SUCCESS);
192512f080e7Smrj }
19267c478bd9Sstevel@tonic-gate
19277c478bd9Sstevel@tonic-gate /*
192820906b23SVikram Hegde * rootnex_dma_freehdl()
192920906b23SVikram Hegde * called from ddi_dma_free_handle().
19307c478bd9Sstevel@tonic-gate */
193120906b23SVikram Hegde static int
rootnex_dma_freehdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle)193220906b23SVikram Hegde rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
193320906b23SVikram Hegde {
193450200e77SFrank Van Der Linden int ret;
193550200e77SFrank Van Der Linden
1936567c0b92SStephen Hanson ndi_fmc_remove(rdip, DMA_HANDLE, handle);
193786ef0a63SRichard Lowe #if !defined(__xpv)
193850200e77SFrank Van Der Linden if (IOMMU_USED(rdip))
193950200e77SFrank Van Der Linden ret = iommulib_nexdma_freehdl(dip, rdip, handle);
194050200e77SFrank Van Der Linden else
194120906b23SVikram Hegde #endif
194250200e77SFrank Van Der Linden ret = rootnex_coredma_freehdl(dip, rdip, handle);
194350200e77SFrank Van Der Linden
194450200e77SFrank Van Der Linden if (rootnex_state->r_dvma_call_list_id)
194550200e77SFrank Van Der Linden ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
194650200e77SFrank Van Der Linden
194750200e77SFrank Van Der Linden return (ret);
194820906b23SVikram Hegde }
194920906b23SVikram Hegde
195012f080e7Smrj /*ARGSUSED*/
19517c478bd9Sstevel@tonic-gate static int
rootnex_coredma_bindhdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle,struct ddi_dma_req * dmareq,ddi_dma_cookie_t * cookiep,uint_t * ccountp)195220906b23SVikram Hegde rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
195320906b23SVikram Hegde ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
195420906b23SVikram Hegde ddi_dma_cookie_t *cookiep, uint_t *ccountp)
19557c478bd9Sstevel@tonic-gate {
195612f080e7Smrj rootnex_sglinfo_t *sinfo;
195750200e77SFrank Van Der Linden ddi_dma_obj_t *dmao;
195886ef0a63SRichard Lowe #if !defined(__xpv)
195950200e77SFrank Van Der Linden struct dvmaseg *dvs;
196050200e77SFrank Van Der Linden ddi_dma_cookie_t *cookie;
1961ee7536e8SFrank Van Der Linden #endif
196212f080e7Smrj ddi_dma_attr_t *attr;
19637c478bd9Sstevel@tonic-gate ddi_dma_impl_t *hp;
196412f080e7Smrj rootnex_dma_t *dma;
196512f080e7Smrj int kmflag;
196612f080e7Smrj int e;
196750200e77SFrank Van Der Linden uint_t ncookies;
19687c478bd9Sstevel@tonic-gate
196912f080e7Smrj hp = (ddi_dma_impl_t *)handle;
197012f080e7Smrj dma = (rootnex_dma_t *)hp->dmai_private;
1971ee7536e8SFrank Van Der Linden dmao = &dma->dp_dma;
197212f080e7Smrj sinfo = &dma->dp_sglinfo;
197312f080e7Smrj attr = &hp->dmai_attr;
197412f080e7Smrj
197596992ee7SEthindra Ramamurthy /* convert the sleep flags */
197694f1124eSVikram Hegde if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
197796992ee7SEthindra Ramamurthy dma->dp_sleep_flags = kmflag = KM_SLEEP;
197894f1124eSVikram Hegde } else {
197996992ee7SEthindra Ramamurthy dma->dp_sleep_flags = kmflag = KM_NOSLEEP;
198094f1124eSVikram Hegde }
198194f1124eSVikram Hegde
198212f080e7Smrj hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS;
19837c478bd9Sstevel@tonic-gate
19847c478bd9Sstevel@tonic-gate /*
198512f080e7Smrj * This is useful for debugging a driver. Not as useful in a production
198612f080e7Smrj * system. The only time this will fail is if you have a driver bug.
19877c478bd9Sstevel@tonic-gate */
198812f080e7Smrj if (rootnex_bind_check_inuse) {
198912f080e7Smrj /*
199012f080e7Smrj * No one else should ever have this lock unless someone else
199112f080e7Smrj * is trying to use this handle. So contention on the lock
199212f080e7Smrj * is the same as inuse being set.
199312f080e7Smrj */
199412f080e7Smrj e = mutex_tryenter(&dma->dp_mutex);
199512f080e7Smrj if (e == 0) {
1996ef4ab52fSFrank Van Der Linden ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
199712f080e7Smrj return (DDI_DMA_INUSE);
199812f080e7Smrj }
199912f080e7Smrj if (dma->dp_inuse) {
200012f080e7Smrj mutex_exit(&dma->dp_mutex);
2001ef4ab52fSFrank Van Der Linden ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
200212f080e7Smrj return (DDI_DMA_INUSE);
200312f080e7Smrj }
200412f080e7Smrj dma->dp_inuse = B_TRUE;
200512f080e7Smrj mutex_exit(&dma->dp_mutex);
20067c478bd9Sstevel@tonic-gate }
200712f080e7Smrj
200812f080e7Smrj /* save away the original bind info */
200912f080e7Smrj dma->dp_dma = dmareq->dmar_object;
20107c478bd9Sstevel@tonic-gate
201186ef0a63SRichard Lowe #if !defined(__xpv)
201250200e77SFrank Van Der Linden if (IOMMU_USED(rdip)) {
201350200e77SFrank Van Der Linden dmao = &dma->dp_dvma;
201450200e77SFrank Van Der Linden e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao);
201550200e77SFrank Van Der Linden switch (e) {
201650200e77SFrank Van Der Linden case DDI_SUCCESS:
201750200e77SFrank Van Der Linden if (sinfo->si_cancross ||
201850200e77SFrank Van Der Linden dmao->dmao_obj.dvma_obj.dv_nseg != 1 ||
201950200e77SFrank Van Der Linden dmao->dmao_size > sinfo->si_max_cookie_size) {
202050200e77SFrank Van Der Linden dma->dp_dvma_used = B_TRUE;
202150200e77SFrank Van Der Linden break;
202250200e77SFrank Van Der Linden }
202350200e77SFrank Van Der Linden sinfo->si_sgl_size = 1;
202450200e77SFrank Van Der Linden hp->dmai_rflags |= DMP_NOSYNC;
202550200e77SFrank Van Der Linden
202650200e77SFrank Van Der Linden dma->dp_dvma_used = B_TRUE;
202750200e77SFrank Van Der Linden dma->dp_need_to_free_cookie = B_FALSE;
202850200e77SFrank Van Der Linden
202950200e77SFrank Van Der Linden dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0];
203050200e77SFrank Van Der Linden cookie = hp->dmai_cookie = dma->dp_cookies =
203150200e77SFrank Van Der Linden (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
203250200e77SFrank Van Der Linden cookie->dmac_laddress = dvs->dvs_start +
203350200e77SFrank Van Der Linden dmao->dmao_obj.dvma_obj.dv_off;
203450200e77SFrank Van Der Linden cookie->dmac_size = dvs->dvs_len;
203550200e77SFrank Van Der Linden cookie->dmac_type = 0;
203650200e77SFrank Van Der Linden
203750200e77SFrank Van Der Linden ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *,
203850200e77SFrank Van Der Linden rdip);
203950200e77SFrank Van Der Linden goto fast;
204050200e77SFrank Van Der Linden case DDI_ENOTSUP:
204150200e77SFrank Van Der Linden break;
204250200e77SFrank Van Der Linden default:
204350200e77SFrank Van Der Linden rootnex_clean_dmahdl(hp);
204450200e77SFrank Van Der Linden return (e);
204550200e77SFrank Van Der Linden }
204686c1f4dcSVikram Hegde }
204720906b23SVikram Hegde #endif
204886c1f4dcSVikram Hegde
20497c478bd9Sstevel@tonic-gate /*
205050200e77SFrank Van Der Linden * Figure out a rough estimate of what maximum number of pages
205150200e77SFrank Van Der Linden * this buffer could use (a high estimate of course).
20527c478bd9Sstevel@tonic-gate */
205312f080e7Smrj sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1;
20547c478bd9Sstevel@tonic-gate
205550200e77SFrank Van Der Linden if (dma->dp_dvma_used) {
205650200e77SFrank Van Der Linden /*
205750200e77SFrank Van Der Linden * The number of physical pages is the worst case.
205850200e77SFrank Van Der Linden *
205950200e77SFrank Van Der Linden * For DVMA, the worst case is the length divided
206050200e77SFrank Van Der Linden * by the maximum cookie length, plus 1. Add to that
206150200e77SFrank Van Der Linden * the number of segment boundaries potentially crossed, and
206250200e77SFrank Van Der Linden * the additional number of DVMA segments that was returned.
206350200e77SFrank Van Der Linden *
206450200e77SFrank Van Der Linden * In the normal case, for modern devices, si_cancross will
206550200e77SFrank Van Der Linden * be false, and dv_nseg will be 1, and the fast path will
206650200e77SFrank Van Der Linden * have been taken above.
206750200e77SFrank Van Der Linden */
206850200e77SFrank Van Der Linden ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size)
206950200e77SFrank Van Der Linden + 1;
207050200e77SFrank Van Der Linden if (sinfo->si_cancross)
207150200e77SFrank Van Der Linden ncookies +=
207250200e77SFrank Van Der Linden (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1;
207350200e77SFrank Van Der Linden ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1);
207450200e77SFrank Van Der Linden
207550200e77SFrank Van Der Linden sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies);
207650200e77SFrank Van Der Linden }
207750200e77SFrank Van Der Linden
20787c478bd9Sstevel@tonic-gate /*
207912f080e7Smrj * We'll use the pre-allocated cookies for any bind that will *always*
208012f080e7Smrj * fit (more important to be consistent, we don't want to create
208112f080e7Smrj * additional degenerate cases).
20827c478bd9Sstevel@tonic-gate */
208312f080e7Smrj if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) {
208412f080e7Smrj dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
208512f080e7Smrj dma->dp_need_to_free_cookie = B_FALSE;
2086ef4ab52fSFrank Van Der Linden ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip,
208712f080e7Smrj uint_t, sinfo->si_max_pages);
20887c478bd9Sstevel@tonic-gate
208912f080e7Smrj /*
209012f080e7Smrj * For anything larger than that, we'll go ahead and allocate the
209112f080e7Smrj * maximum number of pages we expect to see. Hopefuly, we won't be
209212f080e7Smrj * seeing this path in the fast path for high performance devices very
209312f080e7Smrj * frequently.
209412f080e7Smrj *
209512f080e7Smrj * a ddi bind interface that allowed the driver to provide storage to
209612f080e7Smrj * the bind interface would speed this case up.
209712f080e7Smrj */
209812f080e7Smrj } else {
209912f080e7Smrj /*
210012f080e7Smrj * Save away how much memory we allocated. If we're doing a
210112f080e7Smrj * nosleep, the alloc could fail...
210212f080e7Smrj */
210312f080e7Smrj dma->dp_cookie_size = sinfo->si_max_pages *
210412f080e7Smrj sizeof (ddi_dma_cookie_t);
210512f080e7Smrj dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag);
210612f080e7Smrj if (dma->dp_cookies == NULL) {
2107ef4ab52fSFrank Van Der Linden ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
210812f080e7Smrj rootnex_clean_dmahdl(hp);
210912f080e7Smrj return (DDI_DMA_NORESOURCES);
21107c478bd9Sstevel@tonic-gate }
211112f080e7Smrj dma->dp_need_to_free_cookie = B_TRUE;
2112ef4ab52fSFrank Van Der Linden ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip,
2113ef4ab52fSFrank Van Der Linden uint_t, sinfo->si_max_pages);
21147c478bd9Sstevel@tonic-gate }
211512f080e7Smrj hp->dmai_cookie = dma->dp_cookies;
211612f080e7Smrj
21177c478bd9Sstevel@tonic-gate /*
211812f080e7Smrj * Get the real sgl. rootnex_get_sgl will fill in cookie array while
21193a634bfcSVikram Hegde * looking at the constraints in the dma structure. It will then put
21203a634bfcSVikram Hegde * some additional state about the sgl in the dma struct (i.e. is
21213a634bfcSVikram Hegde * the sgl clean, or do we need to do some munging; how many pages
21223a634bfcSVikram Hegde * need to be copied, etc.)
21237c478bd9Sstevel@tonic-gate */
212450200e77SFrank Van Der Linden if (dma->dp_dvma_used)
212550200e77SFrank Van Der Linden rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
212650200e77SFrank Van Der Linden else
212750200e77SFrank Van Der Linden rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
212812f080e7Smrj
21293a634bfcSVikram Hegde out:
213086c1f4dcSVikram Hegde ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages);
213112f080e7Smrj /* if we don't need a copy buffer, we don't need to sync */
213212f080e7Smrj if (sinfo->si_copybuf_req == 0) {
213312f080e7Smrj hp->dmai_rflags |= DMP_NOSYNC;
213412f080e7Smrj }
21357c478bd9Sstevel@tonic-gate
21367c478bd9Sstevel@tonic-gate /*
213712f080e7Smrj * if we don't need the copybuf and we don't need to do a partial, we
213812f080e7Smrj * hit the fast path. All the high performance devices should be trying
213912f080e7Smrj * to hit this path. To hit this path, a device should be able to reach
214012f080e7Smrj * all of memory, shouldn't try to bind more than it can transfer, and
214112f080e7Smrj * the buffer shouldn't require more cookies than the driver/device can
214212f080e7Smrj * handle [sgllen]).
2143877f730dSGarrett D'Amore *
2144877f730dSGarrett D'Amore * Note that negative values of dma_attr_sgllen are supposed
2145877f730dSGarrett D'Amore * to mean unlimited, but we just cast them to mean a
2146877f730dSGarrett D'Amore * "ridiculous large limit". This saves some extra checks on
2147877f730dSGarrett D'Amore * hot paths.
21487c478bd9Sstevel@tonic-gate */
214912f080e7Smrj if ((sinfo->si_copybuf_req == 0) &&
2150877f730dSGarrett D'Amore (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) &&
21516667d029SHans Rosenfeld (dmao->dmao_size <= dma->dp_maxxfer)) {
215250200e77SFrank Van Der Linden fast:
215385c8e0e8Sstephh /*
215485c8e0e8Sstephh * If the driver supports FMA, insert the handle in the FMA DMA
215585c8e0e8Sstephh * handle cache.
215685c8e0e8Sstephh */
2157567c0b92SStephen Hanson if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
215885c8e0e8Sstephh hp->dmai_error.err_cf = rootnex_dma_check;
215985c8e0e8Sstephh
216012f080e7Smrj /*
216112f080e7Smrj * copy out the first cookie and ccountp, set the cookie
216212f080e7Smrj * pointer to the second cookie. The first cookie is passed
216312f080e7Smrj * back on the stack. Additional cookies are accessed via
216412f080e7Smrj * ddi_dma_nextcookie()
216512f080e7Smrj */
216612f080e7Smrj *cookiep = dma->dp_cookies[0];
216712f080e7Smrj *ccountp = sinfo->si_sgl_size;
216812f080e7Smrj hp->dmai_cookie++;
216912f080e7Smrj hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
21709a63ec27SRobert Mustacchi hp->dmai_ncookies = *ccountp;
21719a63ec27SRobert Mustacchi hp->dmai_curcookie = 1;
2172ef4ab52fSFrank Van Der Linden ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
217350200e77SFrank Van Der Linden ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip,
21743a634bfcSVikram Hegde uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS],
217550200e77SFrank Van Der Linden uint_t, dmao->dmao_size, uint_t, *ccountp);
21763a634bfcSVikram Hegde
21773a634bfcSVikram Hegde
217812f080e7Smrj return (DDI_DMA_MAPPED);
217912f080e7Smrj }
21807c478bd9Sstevel@tonic-gate
218112f080e7Smrj /*
218212f080e7Smrj * go to the slow path, we may need to alloc more memory, create
218312f080e7Smrj * multiple windows, and munge up a sgl to make the device happy.
218412f080e7Smrj */
218550200e77SFrank Van Der Linden
218650200e77SFrank Van Der Linden /*
218750200e77SFrank Van Der Linden * With the IOMMU mapobject method used, we should never hit
218850200e77SFrank Van Der Linden * the slow path. If we do, something is seriously wrong.
218950200e77SFrank Van Der Linden * Clean up and return an error.
219050200e77SFrank Van Der Linden */
219150200e77SFrank Van Der Linden
219286ef0a63SRichard Lowe #if !defined(__xpv)
21935800d066SAlbert Lee
219450200e77SFrank Van Der Linden if (dma->dp_dvma_used) {
219550200e77SFrank Van Der Linden (void) iommulib_nexdma_unmapobject(dip, rdip, handle,
219650200e77SFrank Van Der Linden &dma->dp_dvma);
219750200e77SFrank Van Der Linden e = DDI_DMA_NOMAPPING;
219850200e77SFrank Van Der Linden } else {
21995800d066SAlbert Lee #endif
220050200e77SFrank Van Der Linden e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma,
220150200e77SFrank Van Der Linden kmflag);
220286ef0a63SRichard Lowe #if !defined(__xpv)
220350200e77SFrank Van Der Linden }
22045800d066SAlbert Lee #endif
220512f080e7Smrj if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
220612f080e7Smrj if (dma->dp_need_to_free_cookie) {
220712f080e7Smrj kmem_free(dma->dp_cookies, dma->dp_cookie_size);
220812f080e7Smrj }
2209ef4ab52fSFrank Van Der Linden ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
221012f080e7Smrj rootnex_clean_dmahdl(hp); /* must be after free cookie */
221112f080e7Smrj return (e);
221212f080e7Smrj }
22137c478bd9Sstevel@tonic-gate
221485c8e0e8Sstephh /*
221585c8e0e8Sstephh * If the driver supports FMA, insert the handle in the FMA DMA handle
221685c8e0e8Sstephh * cache.
221785c8e0e8Sstephh */
2218567c0b92SStephen Hanson if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
221985c8e0e8Sstephh hp->dmai_error.err_cf = rootnex_dma_check;
222085c8e0e8Sstephh
222112f080e7Smrj /* if the first window uses the copy buffer, sync it for the device */
222212f080e7Smrj if ((dma->dp_window[dma->dp_current_win].wd_dosync) &&
222312f080e7Smrj (hp->dmai_rflags & DDI_DMA_WRITE)) {
222494f1124eSVikram Hegde (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
222512f080e7Smrj DDI_DMA_SYNC_FORDEV);
222612f080e7Smrj }
22277c478bd9Sstevel@tonic-gate
22287c478bd9Sstevel@tonic-gate /*
222912f080e7Smrj * copy out the first cookie and ccountp, set the cookie pointer to the
223012f080e7Smrj * second cookie. Make sure the partial flag is set/cleared correctly.
223112f080e7Smrj * If we have a partial map (i.e. multiple windows), the number of
223212f080e7Smrj * cookies we return is the number of cookies in the first window.
22337c478bd9Sstevel@tonic-gate */
223412f080e7Smrj if (e == DDI_DMA_MAPPED) {
223512f080e7Smrj hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
223612f080e7Smrj *ccountp = sinfo->si_sgl_size;
22373a634bfcSVikram Hegde hp->dmai_nwin = 1;
223812f080e7Smrj } else {
223912f080e7Smrj hp->dmai_rflags |= DDI_DMA_PARTIAL;
224012f080e7Smrj *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt;
224112f080e7Smrj ASSERT(hp->dmai_nwin <= dma->dp_max_win);
224212f080e7Smrj }
224312f080e7Smrj *cookiep = dma->dp_cookies[0];
224412f080e7Smrj hp->dmai_cookie++;
22459a63ec27SRobert Mustacchi hp->dmai_ncookies = *ccountp;
22469a63ec27SRobert Mustacchi hp->dmai_curcookie = 1;
22477c478bd9Sstevel@tonic-gate
22480b7ba611SMark Johnson ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
224950200e77SFrank Van Der Linden ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t,
225012f080e7Smrj rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
225150200e77SFrank Van Der Linden dmao->dmao_size, uint_t, *ccountp);
225212f080e7Smrj return (e);
22537c478bd9Sstevel@tonic-gate }
22547c478bd9Sstevel@tonic-gate
225512f080e7Smrj /*
225620906b23SVikram Hegde * rootnex_dma_bindhdl()
225720906b23SVikram Hegde * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle().
225812f080e7Smrj */
225920906b23SVikram Hegde static int
rootnex_dma_bindhdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle,struct ddi_dma_req * dmareq,ddi_dma_cookie_t * cookiep,uint_t * ccountp)226020906b23SVikram Hegde rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
226120906b23SVikram Hegde ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
226220906b23SVikram Hegde ddi_dma_cookie_t *cookiep, uint_t *ccountp)
226320906b23SVikram Hegde {
226450200e77SFrank Van Der Linden int ret;
226586ef0a63SRichard Lowe #if !defined(__xpv)
226650200e77SFrank Van Der Linden if (IOMMU_USED(rdip))
226750200e77SFrank Van Der Linden ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq,
226850200e77SFrank Van Der Linden cookiep, ccountp);
226950200e77SFrank Van Der Linden else
227020906b23SVikram Hegde #endif
227150200e77SFrank Van Der Linden ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq,
227250200e77SFrank Van Der Linden cookiep, ccountp);
227350200e77SFrank Van Der Linden
227450200e77SFrank Van Der Linden if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) {
227550200e77SFrank Van Der Linden ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg,
227650200e77SFrank Van Der Linden &rootnex_state->r_dvma_call_list_id);
227750200e77SFrank Van Der Linden }
227850200e77SFrank Van Der Linden
227950200e77SFrank Van Der Linden return (ret);
228020906b23SVikram Hegde }
228120906b23SVikram Hegde
22823a634bfcSVikram Hegde
22833a634bfcSVikram Hegde
228412f080e7Smrj /*ARGSUSED*/
22857c478bd9Sstevel@tonic-gate static int
rootnex_coredma_unbindhdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle)228620906b23SVikram Hegde rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
228712f080e7Smrj ddi_dma_handle_t handle)
22887c478bd9Sstevel@tonic-gate {
228912f080e7Smrj ddi_dma_impl_t *hp;
229012f080e7Smrj rootnex_dma_t *dma;
229112f080e7Smrj int e;
22927c478bd9Sstevel@tonic-gate
229312f080e7Smrj hp = (ddi_dma_impl_t *)handle;
229412f080e7Smrj dma = (rootnex_dma_t *)hp->dmai_private;
229512f080e7Smrj
229612f080e7Smrj /* make sure the buffer wasn't free'd before calling unbind */
229712f080e7Smrj if (rootnex_unbind_verify_buffer) {
229812f080e7Smrj e = rootnex_verify_buffer(dma);
229912f080e7Smrj if (e != DDI_SUCCESS) {
230012f080e7Smrj ASSERT(0);
230112f080e7Smrj return (DDI_FAILURE);
23027c478bd9Sstevel@tonic-gate }
23037c478bd9Sstevel@tonic-gate }
230412f080e7Smrj
230512f080e7Smrj /* sync the current window before unbinding the buffer */
230612f080e7Smrj if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync &&
230712f080e7Smrj (hp->dmai_rflags & DDI_DMA_READ)) {
230894f1124eSVikram Hegde (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
230912f080e7Smrj DDI_DMA_SYNC_FORCPU);
231012f080e7Smrj }
231112f080e7Smrj
23127c478bd9Sstevel@tonic-gate /*
231312f080e7Smrj * cleanup and copy buffer or window state. if we didn't use the copy
231412f080e7Smrj * buffer or windows, there won't be much to do :-)
23157c478bd9Sstevel@tonic-gate */
231612f080e7Smrj rootnex_teardown_copybuf(dma);
231712f080e7Smrj rootnex_teardown_windows(dma);
23187c478bd9Sstevel@tonic-gate
231986ef0a63SRichard Lowe #if !defined(__xpv)
23202cedd8f0SHans Rosenfeld if (IOMMU_USED(rdip) && dma->dp_dvma_used)
232150200e77SFrank Van Der Linden (void) iommulib_nexdma_unmapobject(dip, rdip, handle,
232250200e77SFrank Van Der Linden &dma->dp_dvma);
23235800d066SAlbert Lee #endif
232486c1f4dcSVikram Hegde
232512f080e7Smrj /*
232612f080e7Smrj * If we had to allocate space to for the worse case sgl (it didn't
232712f080e7Smrj * fit into our pre-allocate buffer), free that up now
232812f080e7Smrj */
232912f080e7Smrj if (dma->dp_need_to_free_cookie) {
233012f080e7Smrj kmem_free(dma->dp_cookies, dma->dp_cookie_size);
23317c478bd9Sstevel@tonic-gate }
23327c478bd9Sstevel@tonic-gate
23337c478bd9Sstevel@tonic-gate /*
233412f080e7Smrj * clean up the handle so it's ready for the next bind (i.e. if the
233512f080e7Smrj * handle is reused).
23367c478bd9Sstevel@tonic-gate */
233712f080e7Smrj rootnex_clean_dmahdl(hp);
2338567c0b92SStephen Hanson hp->dmai_error.err_cf = NULL;
233912f080e7Smrj
23400b7ba611SMark Johnson ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
23410b7ba611SMark Johnson ROOTNEX_DPROBE1(rootnex__unbind, uint64_t,
234212f080e7Smrj rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
234312f080e7Smrj
234412f080e7Smrj return (DDI_SUCCESS);
234512f080e7Smrj }
234612f080e7Smrj
234720906b23SVikram Hegde /*
234820906b23SVikram Hegde * rootnex_dma_unbindhdl()
234920906b23SVikram Hegde * called from ddi_dma_unbind_handle()
235020906b23SVikram Hegde */
235120906b23SVikram Hegde /*ARGSUSED*/
235220906b23SVikram Hegde static int
rootnex_dma_unbindhdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle)235320906b23SVikram Hegde rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
235420906b23SVikram Hegde ddi_dma_handle_t handle)
235520906b23SVikram Hegde {
235650200e77SFrank Van Der Linden int ret;
235750200e77SFrank Van Der Linden
235886ef0a63SRichard Lowe #if !defined(__xpv)
235950200e77SFrank Van Der Linden if (IOMMU_USED(rdip))
236050200e77SFrank Van Der Linden ret = iommulib_nexdma_unbindhdl(dip, rdip, handle);
236150200e77SFrank Van Der Linden else
236220906b23SVikram Hegde #endif
236350200e77SFrank Van Der Linden ret = rootnex_coredma_unbindhdl(dip, rdip, handle);
236450200e77SFrank Van Der Linden
236550200e77SFrank Van Der Linden if (rootnex_state->r_dvma_call_list_id)
236650200e77SFrank Van Der Linden ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
236750200e77SFrank Van Der Linden
236850200e77SFrank Van Der Linden return (ret);
236920906b23SVikram Hegde }
237020906b23SVikram Hegde
237186ef0a63SRichard Lowe #if !defined(__xpv)
237294f1124eSVikram Hegde
237394f1124eSVikram Hegde static int
rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle)237494f1124eSVikram Hegde rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle)
237594f1124eSVikram Hegde {
237694f1124eSVikram Hegde ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
237794f1124eSVikram Hegde rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
237894f1124eSVikram Hegde
237994f1124eSVikram Hegde if (dma->dp_sleep_flags != KM_SLEEP &&
238094f1124eSVikram Hegde dma->dp_sleep_flags != KM_NOSLEEP)
238194f1124eSVikram Hegde cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle");
238294f1124eSVikram Hegde return (dma->dp_sleep_flags);
238394f1124eSVikram Hegde }
238420906b23SVikram Hegde /*ARGSUSED*/
238520906b23SVikram Hegde static void
rootnex_coredma_reset_cookies(dev_info_t * dip,ddi_dma_handle_t handle)238620906b23SVikram Hegde rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
238720906b23SVikram Hegde {
238820906b23SVikram Hegde ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
238920906b23SVikram Hegde rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
239094f1124eSVikram Hegde rootnex_window_t *window;
239120906b23SVikram Hegde
239294f1124eSVikram Hegde if (dma->dp_window) {
239394f1124eSVikram Hegde window = &dma->dp_window[dma->dp_current_win];
239494f1124eSVikram Hegde hp->dmai_cookie = window->wd_first_cookie;
239594f1124eSVikram Hegde } else {
239694f1124eSVikram Hegde hp->dmai_cookie = dma->dp_cookies;
239794f1124eSVikram Hegde }
239820906b23SVikram Hegde hp->dmai_cookie++;
23999a63ec27SRobert Mustacchi hp->dmai_curcookie = 1;
240020906b23SVikram Hegde }
240120906b23SVikram Hegde
240220906b23SVikram Hegde /*ARGSUSED*/
240320906b23SVikram Hegde static int
rootnex_coredma_get_cookies(dev_info_t * dip,ddi_dma_handle_t handle,ddi_dma_cookie_t ** cookiepp,uint_t * ccountp)240420906b23SVikram Hegde rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
240594f1124eSVikram Hegde ddi_dma_cookie_t **cookiepp, uint_t *ccountp)
240620906b23SVikram Hegde {
240794f1124eSVikram Hegde int i;
240894f1124eSVikram Hegde int km_flags;
240920906b23SVikram Hegde ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
241020906b23SVikram Hegde rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
241194f1124eSVikram Hegde rootnex_window_t *window;
241294f1124eSVikram Hegde ddi_dma_cookie_t *cp;
241394f1124eSVikram Hegde ddi_dma_cookie_t *cookie;
241420906b23SVikram Hegde
241594f1124eSVikram Hegde ASSERT(*cookiepp == NULL);
241694f1124eSVikram Hegde ASSERT(*ccountp == 0);
241720906b23SVikram Hegde
241894f1124eSVikram Hegde if (dma->dp_window) {
241994f1124eSVikram Hegde window = &dma->dp_window[dma->dp_current_win];
242094f1124eSVikram Hegde cp = window->wd_first_cookie;
242194f1124eSVikram Hegde *ccountp = window->wd_cookie_cnt;
242220906b23SVikram Hegde } else {
242394f1124eSVikram Hegde cp = dma->dp_cookies;
242420906b23SVikram Hegde *ccountp = dma->dp_sglinfo.si_sgl_size;
242520906b23SVikram Hegde }
242620906b23SVikram Hegde
242794f1124eSVikram Hegde km_flags = rootnex_coredma_get_sleep_flags(handle);
242894f1124eSVikram Hegde cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags);
242994f1124eSVikram Hegde if (cookie == NULL) {
243094f1124eSVikram Hegde return (DDI_DMA_NORESOURCES);
243194f1124eSVikram Hegde }
243294f1124eSVikram Hegde
243394f1124eSVikram Hegde for (i = 0; i < *ccountp; i++) {
243494f1124eSVikram Hegde cookie[i].dmac_notused = cp[i].dmac_notused;
243594f1124eSVikram Hegde cookie[i].dmac_type = cp[i].dmac_type;
243694f1124eSVikram Hegde cookie[i].dmac_address = cp[i].dmac_address;
243794f1124eSVikram Hegde cookie[i].dmac_size = cp[i].dmac_size;
243894f1124eSVikram Hegde }
243994f1124eSVikram Hegde
244094f1124eSVikram Hegde *cookiepp = cookie;
244194f1124eSVikram Hegde
244294f1124eSVikram Hegde return (DDI_SUCCESS);
244394f1124eSVikram Hegde }
244494f1124eSVikram Hegde
244594f1124eSVikram Hegde /*ARGSUSED*/
244694f1124eSVikram Hegde static int
rootnex_coredma_set_cookies(dev_info_t * dip,ddi_dma_handle_t handle,ddi_dma_cookie_t * cookiep,uint_t ccount)244794f1124eSVikram Hegde rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
244894f1124eSVikram Hegde ddi_dma_cookie_t *cookiep, uint_t ccount)
244994f1124eSVikram Hegde {
245094f1124eSVikram Hegde ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
245194f1124eSVikram Hegde rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
245294f1124eSVikram Hegde rootnex_window_t *window;
245394f1124eSVikram Hegde ddi_dma_cookie_t *cur_cookiep;
245494f1124eSVikram Hegde
245594f1124eSVikram Hegde ASSERT(cookiep);
245694f1124eSVikram Hegde ASSERT(ccount != 0);
245794f1124eSVikram Hegde ASSERT(dma->dp_need_to_switch_cookies == B_FALSE);
245894f1124eSVikram Hegde
245994f1124eSVikram Hegde if (dma->dp_window) {
246094f1124eSVikram Hegde window = &dma->dp_window[dma->dp_current_win];
246194f1124eSVikram Hegde dma->dp_saved_cookies = window->wd_first_cookie;
246294f1124eSVikram Hegde window->wd_first_cookie = cookiep;
246394f1124eSVikram Hegde ASSERT(ccount == window->wd_cookie_cnt);
246494f1124eSVikram Hegde cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
246594f1124eSVikram Hegde + window->wd_first_cookie;
246694f1124eSVikram Hegde } else {
246794f1124eSVikram Hegde dma->dp_saved_cookies = dma->dp_cookies;
246894f1124eSVikram Hegde dma->dp_cookies = cookiep;
246994f1124eSVikram Hegde ASSERT(ccount == dma->dp_sglinfo.si_sgl_size);
247094f1124eSVikram Hegde cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
247194f1124eSVikram Hegde + dma->dp_cookies;
247294f1124eSVikram Hegde }
247394f1124eSVikram Hegde
247494f1124eSVikram Hegde dma->dp_need_to_switch_cookies = B_TRUE;
247594f1124eSVikram Hegde hp->dmai_cookie = cur_cookiep;
247694f1124eSVikram Hegde
247794f1124eSVikram Hegde return (DDI_SUCCESS);
247894f1124eSVikram Hegde }
247994f1124eSVikram Hegde
248094f1124eSVikram Hegde /*ARGSUSED*/
248194f1124eSVikram Hegde static int
rootnex_coredma_clear_cookies(dev_info_t * dip,ddi_dma_handle_t handle)248294f1124eSVikram Hegde rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
248394f1124eSVikram Hegde {
248494f1124eSVikram Hegde ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
248594f1124eSVikram Hegde rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
248694f1124eSVikram Hegde rootnex_window_t *window;
248794f1124eSVikram Hegde ddi_dma_cookie_t *cur_cookiep;
248894f1124eSVikram Hegde ddi_dma_cookie_t *cookie_array;
248994f1124eSVikram Hegde uint_t ccount;
249094f1124eSVikram Hegde
249194f1124eSVikram Hegde /* check if cookies have not been switched */
249294f1124eSVikram Hegde if (dma->dp_need_to_switch_cookies == B_FALSE)
249394f1124eSVikram Hegde return (DDI_SUCCESS);
249494f1124eSVikram Hegde
249594f1124eSVikram Hegde ASSERT(dma->dp_saved_cookies);
249694f1124eSVikram Hegde
249794f1124eSVikram Hegde if (dma->dp_window) {
249894f1124eSVikram Hegde window = &dma->dp_window[dma->dp_current_win];
249994f1124eSVikram Hegde cookie_array = window->wd_first_cookie;
250094f1124eSVikram Hegde window->wd_first_cookie = dma->dp_saved_cookies;
250194f1124eSVikram Hegde dma->dp_saved_cookies = NULL;
250294f1124eSVikram Hegde ccount = window->wd_cookie_cnt;
250394f1124eSVikram Hegde cur_cookiep = (hp->dmai_cookie - cookie_array)
250494f1124eSVikram Hegde + window->wd_first_cookie;
250594f1124eSVikram Hegde } else {
250694f1124eSVikram Hegde cookie_array = dma->dp_cookies;
250794f1124eSVikram Hegde dma->dp_cookies = dma->dp_saved_cookies;
250894f1124eSVikram Hegde dma->dp_saved_cookies = NULL;
250994f1124eSVikram Hegde ccount = dma->dp_sglinfo.si_sgl_size;
251094f1124eSVikram Hegde cur_cookiep = (hp->dmai_cookie - cookie_array)
251194f1124eSVikram Hegde + dma->dp_cookies;
251294f1124eSVikram Hegde }
251394f1124eSVikram Hegde
251494f1124eSVikram Hegde kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
251594f1124eSVikram Hegde
251694f1124eSVikram Hegde hp->dmai_cookie = cur_cookiep;
251794f1124eSVikram Hegde
251894f1124eSVikram Hegde dma->dp_need_to_switch_cookies = B_FALSE;
251920906b23SVikram Hegde
252020906b23SVikram Hegde return (DDI_SUCCESS);
252120906b23SVikram Hegde }
252294f1124eSVikram Hegde
25235dfdb46bSVikram Hegde #endif
252412f080e7Smrj
252550200e77SFrank Van Der Linden static struct as *
rootnex_get_as(ddi_dma_obj_t * dmao)252650200e77SFrank Van Der Linden rootnex_get_as(ddi_dma_obj_t *dmao)
252750200e77SFrank Van Der Linden {
252850200e77SFrank Van Der Linden struct as *asp;
252950200e77SFrank Van Der Linden
253050200e77SFrank Van Der Linden switch (dmao->dmao_type) {
253150200e77SFrank Van Der Linden case DMA_OTYP_VADDR:
253250200e77SFrank Van Der Linden case DMA_OTYP_BUFVADDR:
253350200e77SFrank Van Der Linden asp = dmao->dmao_obj.virt_obj.v_as;
253450200e77SFrank Van Der Linden if (asp == NULL)
253550200e77SFrank Van Der Linden asp = &kas;
253650200e77SFrank Van Der Linden break;
253750200e77SFrank Van Der Linden default:
253850200e77SFrank Van Der Linden asp = NULL;
253950200e77SFrank Van Der Linden break;
254050200e77SFrank Van Der Linden }
254150200e77SFrank Van Der Linden return (asp);
254250200e77SFrank Van Der Linden }
254350200e77SFrank Van Der Linden
254412f080e7Smrj /*
254512f080e7Smrj * rootnex_verify_buffer()
254612f080e7Smrj * verify buffer wasn't free'd
254712f080e7Smrj */
254812f080e7Smrj static int
rootnex_verify_buffer(rootnex_dma_t * dma)254912f080e7Smrj rootnex_verify_buffer(rootnex_dma_t *dma)
255012f080e7Smrj {
255112f080e7Smrj page_t **pplist;
255212f080e7Smrj caddr_t vaddr;
255312f080e7Smrj uint_t pcnt;
255412f080e7Smrj uint_t poff;
255512f080e7Smrj page_t *pp;
255600d0963fSdilpreet char b;
255712f080e7Smrj int i;
25587c478bd9Sstevel@tonic-gate
255912f080e7Smrj /* Figure out how many pages this buffer occupies */
256012f080e7Smrj if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) {
256112f080e7Smrj poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET;
256212f080e7Smrj } else {
256312f080e7Smrj vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr;
256412f080e7Smrj poff = (uintptr_t)vaddr & MMU_PAGEOFFSET;
256512f080e7Smrj }
256612f080e7Smrj pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff);
25677c478bd9Sstevel@tonic-gate
256812f080e7Smrj switch (dma->dp_dma.dmao_type) {
256912f080e7Smrj case DMA_OTYP_PAGES:
25707c478bd9Sstevel@tonic-gate /*
257112f080e7Smrj * for a linked list of pp's walk through them to make sure
257212f080e7Smrj * they're locked and not free.
25737c478bd9Sstevel@tonic-gate */
257412f080e7Smrj pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp;
257512f080e7Smrj for (i = 0; i < pcnt; i++) {
257612f080e7Smrj if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) {
257712f080e7Smrj return (DDI_FAILURE);
257812f080e7Smrj }
257912f080e7Smrj pp = pp->p_next;
25807c478bd9Sstevel@tonic-gate }
258112f080e7Smrj break;
25827c478bd9Sstevel@tonic-gate
258312f080e7Smrj case DMA_OTYP_VADDR:
258412f080e7Smrj case DMA_OTYP_BUFVADDR:
258512f080e7Smrj pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv;
25867c478bd9Sstevel@tonic-gate /*
258712f080e7Smrj * for an array of pp's walk through them to make sure they're
258812f080e7Smrj * not free. It's possible that they may not be locked.
25897c478bd9Sstevel@tonic-gate */
259012f080e7Smrj if (pplist) {
259112f080e7Smrj for (i = 0; i < pcnt; i++) {
259212f080e7Smrj if (PP_ISFREE(pplist[i])) {
259312f080e7Smrj return (DDI_FAILURE);
259412f080e7Smrj }
259512f080e7Smrj }
25967c478bd9Sstevel@tonic-gate
259712f080e7Smrj /* For a virtual address, try to peek at each page */
259812f080e7Smrj } else {
259950200e77SFrank Van Der Linden if (rootnex_get_as(&dma->dp_dma) == &kas) {
260012f080e7Smrj for (i = 0; i < pcnt; i++) {
260100d0963fSdilpreet if (ddi_peek8(NULL, vaddr, &b) ==
260200d0963fSdilpreet DDI_FAILURE)
260312f080e7Smrj return (DDI_FAILURE);
260400d0963fSdilpreet vaddr += MMU_PAGESIZE;
260512f080e7Smrj }
260612f080e7Smrj }
26077c478bd9Sstevel@tonic-gate }
260812f080e7Smrj break;
260912f080e7Smrj
261012f080e7Smrj default:
261150200e77SFrank Van Der Linden cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object");
261212f080e7Smrj break;
26137c478bd9Sstevel@tonic-gate }
26147c478bd9Sstevel@tonic-gate
261512f080e7Smrj return (DDI_SUCCESS);
26167c478bd9Sstevel@tonic-gate }
26177c478bd9Sstevel@tonic-gate
261812f080e7Smrj
261912f080e7Smrj /*
262012f080e7Smrj * rootnex_clean_dmahdl()
262112f080e7Smrj * Clean the dma handle. This should be called on a handle alloc and an
262212f080e7Smrj * unbind handle. Set the handle state to the default settings.
262312f080e7Smrj */
262412f080e7Smrj static void
rootnex_clean_dmahdl(ddi_dma_impl_t * hp)262512f080e7Smrj rootnex_clean_dmahdl(ddi_dma_impl_t *hp)
26267c478bd9Sstevel@tonic-gate {
262712f080e7Smrj rootnex_dma_t *dma;
262812f080e7Smrj
262912f080e7Smrj
263012f080e7Smrj dma = (rootnex_dma_t *)hp->dmai_private;
263112f080e7Smrj
263212f080e7Smrj hp->dmai_nwin = 0;
263312f080e7Smrj dma->dp_current_cookie = 0;
263412f080e7Smrj dma->dp_copybuf_size = 0;
263512f080e7Smrj dma->dp_window = NULL;
263612f080e7Smrj dma->dp_cbaddr = NULL;
263712f080e7Smrj dma->dp_inuse = B_FALSE;
263850200e77SFrank Van Der Linden dma->dp_dvma_used = B_FALSE;
263912f080e7Smrj dma->dp_need_to_free_cookie = B_FALSE;
264094f1124eSVikram Hegde dma->dp_need_to_switch_cookies = B_FALSE;
264194f1124eSVikram Hegde dma->dp_saved_cookies = NULL;
264294f1124eSVikram Hegde dma->dp_sleep_flags = KM_PANIC;
264312f080e7Smrj dma->dp_need_to_free_window = B_FALSE;
264412f080e7Smrj dma->dp_partial_required = B_FALSE;
264512f080e7Smrj dma->dp_trim_required = B_FALSE;
264612f080e7Smrj dma->dp_sglinfo.si_copybuf_req = 0;
264712f080e7Smrj
264812f080e7Smrj /* FMA related initialization */
264912f080e7Smrj hp->dmai_fault = 0;
265012f080e7Smrj hp->dmai_fault_check = NULL;
265112f080e7Smrj hp->dmai_fault_notify = NULL;
265212f080e7Smrj hp->dmai_error.err_ena = 0;
265312f080e7Smrj hp->dmai_error.err_status = DDI_FM_OK;
265412f080e7Smrj hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
265512f080e7Smrj hp->dmai_error.err_ontrap = NULL;
26569a63ec27SRobert Mustacchi
26579a63ec27SRobert Mustacchi /* Cookie tracking */
26589a63ec27SRobert Mustacchi hp->dmai_ncookies = 0;
26599a63ec27SRobert Mustacchi hp->dmai_curcookie = 0;
26607c478bd9Sstevel@tonic-gate }
26617c478bd9Sstevel@tonic-gate
266212f080e7Smrj
266312f080e7Smrj /*
266412f080e7Smrj * rootnex_valid_alloc_parms()
266512f080e7Smrj * Called in ddi_dma_alloc_handle path to validate its parameters.
266612f080e7Smrj */
26677c478bd9Sstevel@tonic-gate static int
rootnex_valid_alloc_parms(ddi_dma_attr_t * attr,uint_t maxsegmentsize)266812f080e7Smrj rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize)
26697c478bd9Sstevel@tonic-gate {
267012f080e7Smrj if ((attr->dma_attr_seg < MMU_PAGEOFFSET) ||
267112f080e7Smrj (attr->dma_attr_count_max < MMU_PAGEOFFSET) ||
267212f080e7Smrj (attr->dma_attr_granular > MMU_PAGESIZE) ||
267312f080e7Smrj (attr->dma_attr_maxxfer < MMU_PAGESIZE)) {
267412f080e7Smrj return (DDI_DMA_BADATTR);
267512f080e7Smrj }
26767c478bd9Sstevel@tonic-gate
267712f080e7Smrj if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) {
267812f080e7Smrj return (DDI_DMA_BADATTR);
267912f080e7Smrj }
268012f080e7Smrj
268112f080e7Smrj if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET ||
268212f080e7Smrj MMU_PAGESIZE & (attr->dma_attr_granular - 1) ||
2683877f730dSGarrett D'Amore attr->dma_attr_sgllen == 0) {
268412f080e7Smrj return (DDI_DMA_BADATTR);
268512f080e7Smrj }
268612f080e7Smrj
268712f080e7Smrj /* We should be able to DMA into every byte offset in a page */
268812f080e7Smrj if (maxsegmentsize < MMU_PAGESIZE) {
268912f080e7Smrj return (DDI_DMA_BADATTR);
26907c478bd9Sstevel@tonic-gate }
269112f080e7Smrj
269207c6692fSMark Johnson /* if we're bouncing on seg, seg must be <= addr_hi */
269307c6692fSMark Johnson if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) &&
269407c6692fSMark Johnson (attr->dma_attr_seg > attr->dma_attr_addr_hi)) {
269507c6692fSMark Johnson return (DDI_DMA_BADATTR);
269607c6692fSMark Johnson }
269712f080e7Smrj return (DDI_SUCCESS);
26987c478bd9Sstevel@tonic-gate }
26997c478bd9Sstevel@tonic-gate
270007c6692fSMark Johnson /*
270107c6692fSMark Johnson * rootnex_need_bounce_seg()
270207c6692fSMark Johnson * check to see if the buffer lives on both side of the seg.
270307c6692fSMark Johnson */
270407c6692fSMark Johnson static boolean_t
rootnex_need_bounce_seg(ddi_dma_obj_t * dmar_object,rootnex_sglinfo_t * sglinfo)270507c6692fSMark Johnson rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo)
270607c6692fSMark Johnson {
270707c6692fSMark Johnson ddi_dma_atyp_t buftype;
270807c6692fSMark Johnson rootnex_addr_t raddr;
270907c6692fSMark Johnson boolean_t lower_addr;
271007c6692fSMark Johnson boolean_t upper_addr;
271107c6692fSMark Johnson uint64_t offset;
271207c6692fSMark Johnson page_t **pplist;
271307c6692fSMark Johnson uint64_t paddr;
271407c6692fSMark Johnson uint32_t psize;
271507c6692fSMark Johnson uint32_t size;
271607c6692fSMark Johnson caddr_t vaddr;
271707c6692fSMark Johnson uint_t pcnt;
271807c6692fSMark Johnson page_t *pp;
271907c6692fSMark Johnson
2720584b574aSToomas Soome pp = NULL;
272107c6692fSMark Johnson /* shortcuts */
272207c6692fSMark Johnson pplist = dmar_object->dmao_obj.virt_obj.v_priv;
272307c6692fSMark Johnson vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
272407c6692fSMark Johnson buftype = dmar_object->dmao_type;
272507c6692fSMark Johnson size = dmar_object->dmao_size;
272607c6692fSMark Johnson
272707c6692fSMark Johnson lower_addr = B_FALSE;
272807c6692fSMark Johnson upper_addr = B_FALSE;
272907c6692fSMark Johnson pcnt = 0;
273007c6692fSMark Johnson
273107c6692fSMark Johnson /*
273207c6692fSMark Johnson * Process the first page to handle the initial offset of the buffer.
273307c6692fSMark Johnson * We'll use the base address we get later when we loop through all
273407c6692fSMark Johnson * the pages.
273507c6692fSMark Johnson */
273607c6692fSMark Johnson if (buftype == DMA_OTYP_PAGES) {
273707c6692fSMark Johnson pp = dmar_object->dmao_obj.pp_obj.pp_pp;
273807c6692fSMark Johnson offset = dmar_object->dmao_obj.pp_obj.pp_offset &
273907c6692fSMark Johnson MMU_PAGEOFFSET;
274007c6692fSMark Johnson paddr = pfn_to_pa(pp->p_pagenum) + offset;
274107c6692fSMark Johnson psize = MIN(size, (MMU_PAGESIZE - offset));
274207c6692fSMark Johnson pp = pp->p_next;
274307c6692fSMark Johnson sglinfo->si_asp = NULL;
274407c6692fSMark Johnson } else if (pplist != NULL) {
274507c6692fSMark Johnson offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
274607c6692fSMark Johnson sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
274707c6692fSMark Johnson if (sglinfo->si_asp == NULL) {
274807c6692fSMark Johnson sglinfo->si_asp = &kas;
274907c6692fSMark Johnson }
275007c6692fSMark Johnson paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
275107c6692fSMark Johnson paddr += offset;
275207c6692fSMark Johnson psize = MIN(size, (MMU_PAGESIZE - offset));
275307c6692fSMark Johnson pcnt++;
275407c6692fSMark Johnson } else {
275507c6692fSMark Johnson offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
275607c6692fSMark Johnson sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
275707c6692fSMark Johnson if (sglinfo->si_asp == NULL) {
275807c6692fSMark Johnson sglinfo->si_asp = &kas;
275907c6692fSMark Johnson }
276007c6692fSMark Johnson paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
276107c6692fSMark Johnson paddr += offset;
276207c6692fSMark Johnson psize = MIN(size, (MMU_PAGESIZE - offset));
276307c6692fSMark Johnson vaddr += psize;
276407c6692fSMark Johnson }
276507c6692fSMark Johnson
276650200e77SFrank Van Der Linden raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
276707c6692fSMark Johnson
276807c6692fSMark Johnson if ((raddr + psize) > sglinfo->si_segmask) {
276907c6692fSMark Johnson upper_addr = B_TRUE;
277007c6692fSMark Johnson } else {
277107c6692fSMark Johnson lower_addr = B_TRUE;
277207c6692fSMark Johnson }
277307c6692fSMark Johnson size -= psize;
277407c6692fSMark Johnson
277507c6692fSMark Johnson /*
277607c6692fSMark Johnson * Walk through the rest of the pages in the buffer. Track to see
277707c6692fSMark Johnson * if we have pages on both sides of the segment boundary.
277807c6692fSMark Johnson */
277907c6692fSMark Johnson while (size > 0) {
278007c6692fSMark Johnson /* partial or full page */
278107c6692fSMark Johnson psize = MIN(size, MMU_PAGESIZE);
278207c6692fSMark Johnson
278307c6692fSMark Johnson if (buftype == DMA_OTYP_PAGES) {
278407c6692fSMark Johnson /* get the paddr from the page_t */
278507c6692fSMark Johnson ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
278607c6692fSMark Johnson paddr = pfn_to_pa(pp->p_pagenum);
278707c6692fSMark Johnson pp = pp->p_next;
278807c6692fSMark Johnson } else if (pplist != NULL) {
278907c6692fSMark Johnson /* index into the array of page_t's to get the paddr */
279007c6692fSMark Johnson ASSERT(!PP_ISFREE(pplist[pcnt]));
279107c6692fSMark Johnson paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
279207c6692fSMark Johnson pcnt++;
279307c6692fSMark Johnson } else {
279407c6692fSMark Johnson /* call into the VM to get the paddr */
279507c6692fSMark Johnson paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
279607c6692fSMark Johnson vaddr));
279707c6692fSMark Johnson vaddr += psize;
279807c6692fSMark Johnson }
279907c6692fSMark Johnson
280050200e77SFrank Van Der Linden raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
280107c6692fSMark Johnson
280207c6692fSMark Johnson if ((raddr + psize) > sglinfo->si_segmask) {
280307c6692fSMark Johnson upper_addr = B_TRUE;
280407c6692fSMark Johnson } else {
280507c6692fSMark Johnson lower_addr = B_TRUE;
280607c6692fSMark Johnson }
280707c6692fSMark Johnson /*
280807c6692fSMark Johnson * if the buffer lives both above and below the segment
280907c6692fSMark Johnson * boundary, or the current page is the page immediately
281007c6692fSMark Johnson * after the segment, we will use a copy/bounce buffer for
281107c6692fSMark Johnson * all pages > seg.
281207c6692fSMark Johnson */
281307c6692fSMark Johnson if ((lower_addr && upper_addr) ||
281407c6692fSMark Johnson (raddr == (sglinfo->si_segmask + 1))) {
281507c6692fSMark Johnson return (B_TRUE);
281607c6692fSMark Johnson }
281707c6692fSMark Johnson
281807c6692fSMark Johnson size -= psize;
281907c6692fSMark Johnson }
282007c6692fSMark Johnson
282107c6692fSMark Johnson return (B_FALSE);
282207c6692fSMark Johnson }
282307c6692fSMark Johnson
282412f080e7Smrj /*
282512f080e7Smrj * rootnex_get_sgl()
282612f080e7Smrj * Called in bind fastpath to get the sgl. Most of this will be replaced
282712f080e7Smrj * with a call to the vm layer when vm2.0 comes around...
282812f080e7Smrj */
282912f080e7Smrj static void
rootnex_get_sgl(ddi_dma_obj_t * dmar_object,ddi_dma_cookie_t * sgl,rootnex_sglinfo_t * sglinfo)283012f080e7Smrj rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
283112f080e7Smrj rootnex_sglinfo_t *sglinfo)
28327c478bd9Sstevel@tonic-gate {
283312f080e7Smrj ddi_dma_atyp_t buftype;
2834843e1988Sjohnlev rootnex_addr_t raddr;
283512f080e7Smrj uint64_t last_page;
283612f080e7Smrj uint64_t offset;
283712f080e7Smrj uint64_t addrhi;
283812f080e7Smrj uint64_t addrlo;
283912f080e7Smrj uint64_t maxseg;
284012f080e7Smrj page_t **pplist;
284112f080e7Smrj uint64_t paddr;
284212f080e7Smrj uint32_t psize;
284312f080e7Smrj uint32_t size;
284412f080e7Smrj caddr_t vaddr;
284512f080e7Smrj uint_t pcnt;
284612f080e7Smrj page_t *pp;
284712f080e7Smrj uint_t cnt;
284812f080e7Smrj
2849584b574aSToomas Soome pp = NULL;
285012f080e7Smrj /* shortcuts */
285112f080e7Smrj pplist = dmar_object->dmao_obj.virt_obj.v_priv;
285212f080e7Smrj vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
285312f080e7Smrj maxseg = sglinfo->si_max_cookie_size;
285412f080e7Smrj buftype = dmar_object->dmao_type;
285512f080e7Smrj addrhi = sglinfo->si_max_addr;
285612f080e7Smrj addrlo = sglinfo->si_min_addr;
285712f080e7Smrj size = dmar_object->dmao_size;
285812f080e7Smrj
285912f080e7Smrj pcnt = 0;
286012f080e7Smrj cnt = 0;
28617c478bd9Sstevel@tonic-gate
286207c6692fSMark Johnson
286307c6692fSMark Johnson /*
286407c6692fSMark Johnson * check to see if we need to use the copy buffer for pages over
286507c6692fSMark Johnson * the segment attr.
286607c6692fSMark Johnson */
286707c6692fSMark Johnson sglinfo->si_bounce_on_seg = B_FALSE;
286807c6692fSMark Johnson if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) {
286907c6692fSMark Johnson sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg(
287007c6692fSMark Johnson dmar_object, sglinfo);
287107c6692fSMark Johnson }
287207c6692fSMark Johnson
287312f080e7Smrj /*
287412f080e7Smrj * if we were passed down a linked list of pages, i.e. pointer to
287512f080e7Smrj * page_t, use this to get our physical address and buf offset.
287612f080e7Smrj */
287712f080e7Smrj if (buftype == DMA_OTYP_PAGES) {
287812f080e7Smrj pp = dmar_object->dmao_obj.pp_obj.pp_pp;
287912f080e7Smrj ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
288012f080e7Smrj offset = dmar_object->dmao_obj.pp_obj.pp_offset &
288112f080e7Smrj MMU_PAGEOFFSET;
2882843e1988Sjohnlev paddr = pfn_to_pa(pp->p_pagenum) + offset;
288312f080e7Smrj psize = MIN(size, (MMU_PAGESIZE - offset));
288412f080e7Smrj pp = pp->p_next;
288512f080e7Smrj sglinfo->si_asp = NULL;
288612f080e7Smrj
288712f080e7Smrj /*
288812f080e7Smrj * We weren't passed down a linked list of pages, but if we were passed
288912f080e7Smrj * down an array of pages, use this to get our physical address and buf
289012f080e7Smrj * offset.
289112f080e7Smrj */
289212f080e7Smrj } else if (pplist != NULL) {
289312f080e7Smrj ASSERT((buftype == DMA_OTYP_VADDR) ||
289412f080e7Smrj (buftype == DMA_OTYP_BUFVADDR));
289512f080e7Smrj
289612f080e7Smrj offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
289712f080e7Smrj sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
289812f080e7Smrj if (sglinfo->si_asp == NULL) {
289912f080e7Smrj sglinfo->si_asp = &kas;
290012f080e7Smrj }
290112f080e7Smrj
290212f080e7Smrj ASSERT(!PP_ISFREE(pplist[pcnt]));
2903843e1988Sjohnlev paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
290412f080e7Smrj paddr += offset;
290512f080e7Smrj psize = MIN(size, (MMU_PAGESIZE - offset));
290612f080e7Smrj pcnt++;
290712f080e7Smrj
290812f080e7Smrj /*
290912f080e7Smrj * All we have is a virtual address, we'll need to call into the VM
291012f080e7Smrj * to get the physical address.
291112f080e7Smrj */
291212f080e7Smrj } else {
291312f080e7Smrj ASSERT((buftype == DMA_OTYP_VADDR) ||
291412f080e7Smrj (buftype == DMA_OTYP_BUFVADDR));
291512f080e7Smrj
291612f080e7Smrj offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
291712f080e7Smrj sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
291812f080e7Smrj if (sglinfo->si_asp == NULL) {
291912f080e7Smrj sglinfo->si_asp = &kas;
292012f080e7Smrj }
292112f080e7Smrj
2922843e1988Sjohnlev paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
292312f080e7Smrj paddr += offset;
292412f080e7Smrj psize = MIN(size, (MMU_PAGESIZE - offset));
292512f080e7Smrj vaddr += psize;
292612f080e7Smrj }
292712f080e7Smrj
292850200e77SFrank Van Der Linden raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2929843e1988Sjohnlev
293012f080e7Smrj /*
293112f080e7Smrj * Setup the first cookie with the physical address of the page and the
293212f080e7Smrj * size of the page (which takes into account the initial offset into
293312f080e7Smrj * the page.
293412f080e7Smrj */
2935843e1988Sjohnlev sgl[cnt].dmac_laddress = raddr;
293612f080e7Smrj sgl[cnt].dmac_size = psize;
293712f080e7Smrj sgl[cnt].dmac_type = 0;
293812f080e7Smrj
293912f080e7Smrj /*
294012f080e7Smrj * Save away the buffer offset into the page. We'll need this later in
294112f080e7Smrj * the copy buffer code to help figure out the page index within the
294212f080e7Smrj * buffer and the offset into the current page.
294312f080e7Smrj */
294412f080e7Smrj sglinfo->si_buf_offset = offset;
294512f080e7Smrj
294612f080e7Smrj /*
294707c6692fSMark Johnson * If we are using the copy buffer for anything over the segment
294807c6692fSMark Johnson * boundary, and this page is over the segment boundary.
294907c6692fSMark Johnson * OR
295007c6692fSMark Johnson * if the DMA engine can't reach the physical address.
295112f080e7Smrj */
295207c6692fSMark Johnson if (((sglinfo->si_bounce_on_seg) &&
295307c6692fSMark Johnson ((raddr + psize) > sglinfo->si_segmask)) ||
295407c6692fSMark Johnson ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
295507c6692fSMark Johnson /*
295607c6692fSMark Johnson * Increase how much copy buffer we use. We always increase by
295707c6692fSMark Johnson * pagesize so we don't have to worry about converting offsets.
295807c6692fSMark Johnson * Set a flag in the cookies dmac_type to indicate that it uses
295907c6692fSMark Johnson * the copy buffer. If this isn't the last cookie, go to the
296007c6692fSMark Johnson * next cookie (since we separate each page which uses the copy
296107c6692fSMark Johnson * buffer in case the copy buffer is not physically contiguous.
296207c6692fSMark Johnson */
296312f080e7Smrj sglinfo->si_copybuf_req += MMU_PAGESIZE;
296412f080e7Smrj sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
296512f080e7Smrj if ((cnt + 1) < sglinfo->si_max_pages) {
296612f080e7Smrj cnt++;
296712f080e7Smrj sgl[cnt].dmac_laddress = 0;
296812f080e7Smrj sgl[cnt].dmac_size = 0;
296912f080e7Smrj sgl[cnt].dmac_type = 0;
297012f080e7Smrj }
297112f080e7Smrj }
297212f080e7Smrj
297312f080e7Smrj /*
297412f080e7Smrj * save this page's physical address so we can figure out if the next
297512f080e7Smrj * page is physically contiguous. Keep decrementing size until we are
297612f080e7Smrj * done with the buffer.
297712f080e7Smrj */
2978843e1988Sjohnlev last_page = raddr & MMU_PAGEMASK;
297912f080e7Smrj size -= psize;
298012f080e7Smrj
298112f080e7Smrj while (size > 0) {
298212f080e7Smrj /* Get the size for this page (i.e. partial or full page) */
298312f080e7Smrj psize = MIN(size, MMU_PAGESIZE);
298412f080e7Smrj
298512f080e7Smrj if (buftype == DMA_OTYP_PAGES) {
298612f080e7Smrj /* get the paddr from the page_t */
298712f080e7Smrj ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2988843e1988Sjohnlev paddr = pfn_to_pa(pp->p_pagenum);
298912f080e7Smrj pp = pp->p_next;
299012f080e7Smrj } else if (pplist != NULL) {
299112f080e7Smrj /* index into the array of page_t's to get the paddr */
299212f080e7Smrj ASSERT(!PP_ISFREE(pplist[pcnt]));
2993843e1988Sjohnlev paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
299412f080e7Smrj pcnt++;
299512f080e7Smrj } else {
299612f080e7Smrj /* call into the VM to get the paddr */
2997843e1988Sjohnlev paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
299812f080e7Smrj vaddr));
299912f080e7Smrj vaddr += psize;
300012f080e7Smrj }
300112f080e7Smrj
300250200e77SFrank Van Der Linden raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
300307c6692fSMark Johnson
300407c6692fSMark Johnson /*
300507c6692fSMark Johnson * If we are using the copy buffer for anything over the
300607c6692fSMark Johnson * segment boundary, and this page is over the segment
300707c6692fSMark Johnson * boundary.
300807c6692fSMark Johnson * OR
300907c6692fSMark Johnson * if the DMA engine can't reach the physical address.
301007c6692fSMark Johnson */
301107c6692fSMark Johnson if (((sglinfo->si_bounce_on_seg) &&
301207c6692fSMark Johnson ((raddr + psize) > sglinfo->si_segmask)) ||
301307c6692fSMark Johnson ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
301407c6692fSMark Johnson
301512f080e7Smrj sglinfo->si_copybuf_req += MMU_PAGESIZE;
301612f080e7Smrj
301712f080e7Smrj /*
301812f080e7Smrj * if there is something in the current cookie, go to
301912f080e7Smrj * the next one. We only want one page in a cookie which
302012f080e7Smrj * uses the copybuf since the copybuf doesn't have to
302112f080e7Smrj * be physically contiguous.
302212f080e7Smrj */
302312f080e7Smrj if (sgl[cnt].dmac_size != 0) {
302412f080e7Smrj cnt++;
302512f080e7Smrj }
3026843e1988Sjohnlev sgl[cnt].dmac_laddress = raddr;
302712f080e7Smrj sgl[cnt].dmac_size = psize;
302812f080e7Smrj sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
302912f080e7Smrj /* if this isn't the last cookie, go to the next one */
303012f080e7Smrj if ((cnt + 1) < sglinfo->si_max_pages) {
303112f080e7Smrj cnt++;
303212f080e7Smrj sgl[cnt].dmac_laddress = 0;
303312f080e7Smrj sgl[cnt].dmac_size = 0;
303412f080e7Smrj sgl[cnt].dmac_type = 0;
303512f080e7Smrj }
30367c478bd9Sstevel@tonic-gate
303712f080e7Smrj /*
303812f080e7Smrj * this page didn't need the copy buffer, if it's not physically
303912f080e7Smrj * contiguous, or it would put us over a segment boundary, or it
304012f080e7Smrj * puts us over the max cookie size, or the current sgl doesn't
304112f080e7Smrj * have anything in it.
304212f080e7Smrj */
3043843e1988Sjohnlev } else if (((last_page + MMU_PAGESIZE) != raddr) ||
3044843e1988Sjohnlev !(raddr & sglinfo->si_segmask) ||
304512f080e7Smrj ((sgl[cnt].dmac_size + psize) > maxseg) ||
304612f080e7Smrj (sgl[cnt].dmac_size == 0)) {
304712f080e7Smrj /*
304812f080e7Smrj * if we're not already in a new cookie, go to the next
304912f080e7Smrj * cookie.
305012f080e7Smrj */
305112f080e7Smrj if (sgl[cnt].dmac_size != 0) {
305212f080e7Smrj cnt++;
305312f080e7Smrj }
30547c478bd9Sstevel@tonic-gate
305512f080e7Smrj /* save the cookie information */
3056843e1988Sjohnlev sgl[cnt].dmac_laddress = raddr;
305712f080e7Smrj sgl[cnt].dmac_size = psize;
305812f080e7Smrj sgl[cnt].dmac_type = 0;
305912f080e7Smrj
30607c478bd9Sstevel@tonic-gate /*
306112f080e7Smrj * this page didn't need the copy buffer, it is physically
306212f080e7Smrj * contiguous with the last page, and it's <= the max cookie
306312f080e7Smrj * size.
30647c478bd9Sstevel@tonic-gate */
306512f080e7Smrj } else {
306612f080e7Smrj sgl[cnt].dmac_size += psize;
30677c478bd9Sstevel@tonic-gate
306812f080e7Smrj /*
306923a869ddSRyan Zezeski * If this cookie is used up, and more cookies
307023a869ddSRyan Zezeski * are available, then move onto the next one.
307112f080e7Smrj */
307223a869ddSRyan Zezeski if ((sgl[cnt].dmac_size == maxseg) &&
307312f080e7Smrj ((cnt + 1) < sglinfo->si_max_pages)) {
307412f080e7Smrj cnt++;
307512f080e7Smrj sgl[cnt].dmac_laddress = 0;
307612f080e7Smrj sgl[cnt].dmac_size = 0;
307712f080e7Smrj sgl[cnt].dmac_type = 0;
307812f080e7Smrj }
30797c478bd9Sstevel@tonic-gate }
30807c478bd9Sstevel@tonic-gate
30817c478bd9Sstevel@tonic-gate /*
308212f080e7Smrj * save this page's physical address so we can figure out if the
308312f080e7Smrj * next page is physically contiguous. Keep decrementing size
308412f080e7Smrj * until we are done with the buffer.
30857c478bd9Sstevel@tonic-gate */
3086843e1988Sjohnlev last_page = raddr;
308712f080e7Smrj size -= psize;
308812f080e7Smrj }
30897c478bd9Sstevel@tonic-gate
309012f080e7Smrj /* we're done, save away how many cookies the sgl has */
309112f080e7Smrj if (sgl[cnt].dmac_size == 0) {
309212f080e7Smrj ASSERT(cnt < sglinfo->si_max_pages);
309312f080e7Smrj sglinfo->si_sgl_size = cnt;
309412f080e7Smrj } else {
309512f080e7Smrj sglinfo->si_sgl_size = cnt + 1;
30967c478bd9Sstevel@tonic-gate }
30977c478bd9Sstevel@tonic-gate }
30987c478bd9Sstevel@tonic-gate
309950200e77SFrank Van Der Linden static void
rootnex_dvma_get_sgl(ddi_dma_obj_t * dmar_object,ddi_dma_cookie_t * sgl,rootnex_sglinfo_t * sglinfo)310050200e77SFrank Van Der Linden rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
310150200e77SFrank Van Der Linden rootnex_sglinfo_t *sglinfo)
310250200e77SFrank Van Der Linden {
310350200e77SFrank Van Der Linden uint64_t offset;
310450200e77SFrank Van Der Linden uint64_t maxseg;
310550200e77SFrank Van Der Linden uint64_t dvaddr;
310650200e77SFrank Van Der Linden struct dvmaseg *dvs;
310750200e77SFrank Van Der Linden uint64_t paddr;
310850200e77SFrank Van Der Linden uint32_t psize, ssize;
310950200e77SFrank Van Der Linden uint32_t size;
311050200e77SFrank Van Der Linden uint_t cnt;
311150200e77SFrank Van Der Linden int physcontig;
311250200e77SFrank Van Der Linden
311350200e77SFrank Van Der Linden ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR);
311450200e77SFrank Van Der Linden
311550200e77SFrank Van Der Linden /* shortcuts */
311650200e77SFrank Van Der Linden maxseg = sglinfo->si_max_cookie_size;
311750200e77SFrank Van Der Linden size = dmar_object->dmao_size;
311850200e77SFrank Van Der Linden
311950200e77SFrank Van Der Linden cnt = 0;
312050200e77SFrank Van Der Linden sglinfo->si_bounce_on_seg = B_FALSE;
312150200e77SFrank Van Der Linden
312250200e77SFrank Van Der Linden dvs = dmar_object->dmao_obj.dvma_obj.dv_seg;
312350200e77SFrank Van Der Linden offset = dmar_object->dmao_obj.dvma_obj.dv_off;
312450200e77SFrank Van Der Linden ssize = dvs->dvs_len;
312550200e77SFrank Van Der Linden paddr = dvs->dvs_start;
312650200e77SFrank Van Der Linden paddr += offset;
312750200e77SFrank Van Der Linden psize = MIN(ssize, (maxseg - offset));
312850200e77SFrank Van Der Linden dvaddr = paddr + psize;
312950200e77SFrank Van Der Linden ssize -= psize;
313050200e77SFrank Van Der Linden
313150200e77SFrank Van Der Linden sgl[cnt].dmac_laddress = paddr;
313250200e77SFrank Van Der Linden sgl[cnt].dmac_size = psize;
313350200e77SFrank Van Der Linden sgl[cnt].dmac_type = 0;
313450200e77SFrank Van Der Linden
313550200e77SFrank Van Der Linden size -= psize;
313650200e77SFrank Van Der Linden while (size > 0) {
313750200e77SFrank Van Der Linden if (ssize == 0) {
313850200e77SFrank Van Der Linden dvs++;
313950200e77SFrank Van Der Linden ssize = dvs->dvs_len;
314050200e77SFrank Van Der Linden dvaddr = dvs->dvs_start;
314150200e77SFrank Van Der Linden physcontig = 0;
314250200e77SFrank Van Der Linden } else
314350200e77SFrank Van Der Linden physcontig = 1;
314450200e77SFrank Van Der Linden
314550200e77SFrank Van Der Linden paddr = dvaddr;
314650200e77SFrank Van Der Linden psize = MIN(ssize, maxseg);
314750200e77SFrank Van Der Linden dvaddr += psize;
314850200e77SFrank Van Der Linden ssize -= psize;
314950200e77SFrank Van Der Linden
315050200e77SFrank Van Der Linden if (!physcontig || !(paddr & sglinfo->si_segmask) ||
315150200e77SFrank Van Der Linden ((sgl[cnt].dmac_size + psize) > maxseg) ||
315250200e77SFrank Van Der Linden (sgl[cnt].dmac_size == 0)) {
315350200e77SFrank Van Der Linden /*
315450200e77SFrank Van Der Linden * if we're not already in a new cookie, go to the next
315550200e77SFrank Van Der Linden * cookie.
315650200e77SFrank Van Der Linden */
315750200e77SFrank Van Der Linden if (sgl[cnt].dmac_size != 0) {
315850200e77SFrank Van Der Linden cnt++;
315950200e77SFrank Van Der Linden }
316050200e77SFrank Van Der Linden
316150200e77SFrank Van Der Linden /* save the cookie information */
316250200e77SFrank Van Der Linden sgl[cnt].dmac_laddress = paddr;
316350200e77SFrank Van Der Linden sgl[cnt].dmac_size = psize;
316450200e77SFrank Van Der Linden sgl[cnt].dmac_type = 0;
316550200e77SFrank Van Der Linden } else {
316650200e77SFrank Van Der Linden sgl[cnt].dmac_size += psize;
316750200e77SFrank Van Der Linden
316850200e77SFrank Van Der Linden /*
316923a869ddSRyan Zezeski * If this cookie is used up, and more cookies
317023a869ddSRyan Zezeski * are available, then move onto the next one.
317150200e77SFrank Van Der Linden */
317223a869ddSRyan Zezeski if ((sgl[cnt].dmac_size == maxseg) &&
317350200e77SFrank Van Der Linden ((cnt + 1) < sglinfo->si_max_pages)) {
317450200e77SFrank Van Der Linden cnt++;
317550200e77SFrank Van Der Linden sgl[cnt].dmac_laddress = 0;
317650200e77SFrank Van Der Linden sgl[cnt].dmac_size = 0;
317750200e77SFrank Van Der Linden sgl[cnt].dmac_type = 0;
317850200e77SFrank Van Der Linden }
317950200e77SFrank Van Der Linden }
318050200e77SFrank Van Der Linden size -= psize;
318150200e77SFrank Van Der Linden }
318250200e77SFrank Van Der Linden
318350200e77SFrank Van Der Linden /* we're done, save away how many cookies the sgl has */
318450200e77SFrank Van Der Linden if (sgl[cnt].dmac_size == 0) {
318550200e77SFrank Van Der Linden sglinfo->si_sgl_size = cnt;
318650200e77SFrank Van Der Linden } else {
318750200e77SFrank Van Der Linden sglinfo->si_sgl_size = cnt + 1;
318850200e77SFrank Van Der Linden }
318950200e77SFrank Van Der Linden }
319050200e77SFrank Van Der Linden
31917c478bd9Sstevel@tonic-gate /*
319212f080e7Smrj * rootnex_bind_slowpath()
319312f080e7Smrj * Call in the bind path if the calling driver can't use the sgl without
319412f080e7Smrj * modifying it. We either need to use the copy buffer and/or we will end up
319512f080e7Smrj * with a partial bind.
31967c478bd9Sstevel@tonic-gate */
319712f080e7Smrj static int
rootnex_bind_slowpath(ddi_dma_impl_t * hp,struct ddi_dma_req * dmareq,rootnex_dma_t * dma,ddi_dma_attr_t * attr,ddi_dma_obj_t * dmao,int kmflag)319812f080e7Smrj rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
319950200e77SFrank Van Der Linden rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
32007c478bd9Sstevel@tonic-gate {
320112f080e7Smrj rootnex_sglinfo_t *sinfo;
320212f080e7Smrj rootnex_window_t *window;
320312f080e7Smrj ddi_dma_cookie_t *cookie;
320412f080e7Smrj size_t copybuf_used;
320512f080e7Smrj size_t dmac_size;
320612f080e7Smrj boolean_t partial;
320712f080e7Smrj off_t cur_offset;
320812f080e7Smrj page_t *cur_pp;
320912f080e7Smrj major_t mnum;
321012f080e7Smrj int e;
321112f080e7Smrj int i;
321212f080e7Smrj
321312f080e7Smrj
321412f080e7Smrj sinfo = &dma->dp_sglinfo;
321512f080e7Smrj copybuf_used = 0;
321612f080e7Smrj partial = B_FALSE;
32177c478bd9Sstevel@tonic-gate
32187c478bd9Sstevel@tonic-gate /*
321912f080e7Smrj * If we're using the copybuf, set the copybuf state in dma struct.
322012f080e7Smrj * Needs to be first since it sets the copy buffer size.
32217c478bd9Sstevel@tonic-gate */
322212f080e7Smrj if (sinfo->si_copybuf_req != 0) {
322312f080e7Smrj e = rootnex_setup_copybuf(hp, dmareq, dma, attr);
322412f080e7Smrj if (e != DDI_SUCCESS) {
322512f080e7Smrj return (e);
322612f080e7Smrj }
322712f080e7Smrj } else {
322812f080e7Smrj dma->dp_copybuf_size = 0;
322912f080e7Smrj }
32307c478bd9Sstevel@tonic-gate
32317c478bd9Sstevel@tonic-gate /*
323212f080e7Smrj * Figure out if we need to do a partial mapping. If so, figure out
323312f080e7Smrj * if we need to trim the buffers when we munge the sgl.
32347c478bd9Sstevel@tonic-gate */
323512f080e7Smrj if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) ||
323650200e77SFrank Van Der Linden (dmao->dmao_size > dma->dp_maxxfer) ||
3237877f730dSGarrett D'Amore ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) {
323812f080e7Smrj dma->dp_partial_required = B_TRUE;
323912f080e7Smrj if (attr->dma_attr_granular != 1) {
324012f080e7Smrj dma->dp_trim_required = B_TRUE;
324112f080e7Smrj }
324212f080e7Smrj } else {
324312f080e7Smrj dma->dp_partial_required = B_FALSE;
324412f080e7Smrj dma->dp_trim_required = B_FALSE;
324512f080e7Smrj }
32467c478bd9Sstevel@tonic-gate
324712f080e7Smrj /* If we need to do a partial bind, make sure the driver supports it */
324812f080e7Smrj if (dma->dp_partial_required &&
324912f080e7Smrj !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
32507c478bd9Sstevel@tonic-gate
325112f080e7Smrj mnum = ddi_driver_major(dma->dp_dip);
325212f080e7Smrj /*
325312f080e7Smrj * patchable which allows us to print one warning per major
325412f080e7Smrj * number.
325512f080e7Smrj */
325612f080e7Smrj if ((rootnex_bind_warn) &&
325712f080e7Smrj ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) {
325812f080e7Smrj rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING;
325912f080e7Smrj cmn_err(CE_WARN, "!%s: coding error detected, the "
326012f080e7Smrj "driver is using ddi_dma_attr(9S) incorrectly. "
326112f080e7Smrj "There is a small risk of data corruption in "
326212f080e7Smrj "particular with large I/Os. The driver should be "
326312f080e7Smrj "replaced with a corrected version for proper "
326412f080e7Smrj "system operation. To disable this warning, add "
326512f080e7Smrj "'set rootnex:rootnex_bind_warn=0' to "
3266*bbf21555SRichard Lowe "/etc/system(5).", ddi_driver_name(dma->dp_dip));
326712f080e7Smrj }
326812f080e7Smrj return (DDI_DMA_TOOBIG);
326912f080e7Smrj }
32707c478bd9Sstevel@tonic-gate
327112f080e7Smrj /*
327212f080e7Smrj * we might need multiple windows, setup state to handle them. In this
327312f080e7Smrj * code path, we will have at least one window.
327412f080e7Smrj */
327550200e77SFrank Van Der Linden e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag);
327612f080e7Smrj if (e != DDI_SUCCESS) {
327712f080e7Smrj rootnex_teardown_copybuf(dma);
327812f080e7Smrj return (e);
327912f080e7Smrj }
32807c478bd9Sstevel@tonic-gate
328112f080e7Smrj window = &dma->dp_window[0];
328212f080e7Smrj cookie = &dma->dp_cookies[0];
328312f080e7Smrj cur_offset = 0;
328412f080e7Smrj rootnex_init_win(hp, dma, window, cookie, cur_offset);
328550200e77SFrank Van Der Linden if (dmao->dmao_type == DMA_OTYP_PAGES) {
328612f080e7Smrj cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp;
32877c478bd9Sstevel@tonic-gate }
328812f080e7Smrj
328912f080e7Smrj /* loop though all the cookies we got back from get_sgl() */
329012f080e7Smrj for (i = 0; i < sinfo->si_sgl_size; i++) {
329112f080e7Smrj /*
329212f080e7Smrj * If we're using the copy buffer, check this cookie and setup
329312f080e7Smrj * its associated copy buffer state. If this cookie uses the
329412f080e7Smrj * copy buffer, make sure we sync this window during dma_sync.
329512f080e7Smrj */
329612f080e7Smrj if (dma->dp_copybuf_size > 0) {
329750200e77SFrank Van Der Linden rootnex_setup_cookie(dmao, dma, cookie,
329812f080e7Smrj cur_offset, ©buf_used, &cur_pp);
329912f080e7Smrj if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
330012f080e7Smrj window->wd_dosync = B_TRUE;
33017c478bd9Sstevel@tonic-gate }
33027c478bd9Sstevel@tonic-gate }
33037c478bd9Sstevel@tonic-gate
330412f080e7Smrj /*
330512f080e7Smrj * save away the cookie size, since it could be modified in
330612f080e7Smrj * the windowing code.
330712f080e7Smrj */
330812f080e7Smrj dmac_size = cookie->dmac_size;
330912f080e7Smrj
331012f080e7Smrj /* if we went over max copybuf size */
331112f080e7Smrj if (dma->dp_copybuf_size &&
331212f080e7Smrj (copybuf_used > dma->dp_copybuf_size)) {
331312f080e7Smrj partial = B_TRUE;
331412f080e7Smrj e = rootnex_copybuf_window_boundary(hp, dma, &window,
331512f080e7Smrj cookie, cur_offset, ©buf_used);
331612f080e7Smrj if (e != DDI_SUCCESS) {
331712f080e7Smrj rootnex_teardown_copybuf(dma);
331812f080e7Smrj rootnex_teardown_windows(dma);
331912f080e7Smrj return (e);
332012f080e7Smrj }
33217c478bd9Sstevel@tonic-gate
33227c478bd9Sstevel@tonic-gate /*
332312f080e7Smrj * if the coookie uses the copy buffer, make sure the
332412f080e7Smrj * new window we just moved to is set to sync.
33257c478bd9Sstevel@tonic-gate */
332612f080e7Smrj if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
332712f080e7Smrj window->wd_dosync = B_TRUE;
332812f080e7Smrj }
3329ef4ab52fSFrank Van Der Linden ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *,
333012f080e7Smrj dma->dp_dip);
333112f080e7Smrj
333212f080e7Smrj /* if the cookie cnt == max sgllen, move to the next window */
3333877f730dSGarrett D'Amore } else if (window->wd_cookie_cnt >=
3334877f730dSGarrett D'Amore (unsigned)attr->dma_attr_sgllen) {
333512f080e7Smrj partial = B_TRUE;
333612f080e7Smrj ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen);
333712f080e7Smrj e = rootnex_sgllen_window_boundary(hp, dma, &window,
333812f080e7Smrj cookie, attr, cur_offset);
333912f080e7Smrj if (e != DDI_SUCCESS) {
334012f080e7Smrj rootnex_teardown_copybuf(dma);
334112f080e7Smrj rootnex_teardown_windows(dma);
334212f080e7Smrj return (e);
334312f080e7Smrj }
33447c478bd9Sstevel@tonic-gate
33457c478bd9Sstevel@tonic-gate /*
334612f080e7Smrj * if the coookie uses the copy buffer, make sure the
334712f080e7Smrj * new window we just moved to is set to sync.
33487c478bd9Sstevel@tonic-gate */
334912f080e7Smrj if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
335012f080e7Smrj window->wd_dosync = B_TRUE;
335112f080e7Smrj }
3352ef4ab52fSFrank Van Der Linden ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *,
335312f080e7Smrj dma->dp_dip);
335412f080e7Smrj
335512f080e7Smrj /* else if we will be over maxxfer */
335612f080e7Smrj } else if ((window->wd_size + dmac_size) >
335712f080e7Smrj dma->dp_maxxfer) {
335812f080e7Smrj partial = B_TRUE;
335912f080e7Smrj e = rootnex_maxxfer_window_boundary(hp, dma, &window,
336012f080e7Smrj cookie);
336112f080e7Smrj if (e != DDI_SUCCESS) {
336212f080e7Smrj rootnex_teardown_copybuf(dma);
336312f080e7Smrj rootnex_teardown_windows(dma);
336412f080e7Smrj return (e);
336512f080e7Smrj }
33667c478bd9Sstevel@tonic-gate
33677c478bd9Sstevel@tonic-gate /*
336812f080e7Smrj * if the coookie uses the copy buffer, make sure the
336912f080e7Smrj * new window we just moved to is set to sync.
33707c478bd9Sstevel@tonic-gate */
337112f080e7Smrj if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
337212f080e7Smrj window->wd_dosync = B_TRUE;
337312f080e7Smrj }
3374ef4ab52fSFrank Van Der Linden ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *,
337512f080e7Smrj dma->dp_dip);
33767c478bd9Sstevel@tonic-gate
337712f080e7Smrj /* else this cookie fits in the current window */
337812f080e7Smrj } else {
337912f080e7Smrj window->wd_cookie_cnt++;
338012f080e7Smrj window->wd_size += dmac_size;
33817c478bd9Sstevel@tonic-gate }
33827c478bd9Sstevel@tonic-gate
338312f080e7Smrj /* track our offset into the buffer, go to the next cookie */
338450200e77SFrank Van Der Linden ASSERT(dmac_size <= dmao->dmao_size);
338512f080e7Smrj ASSERT(cookie->dmac_size <= dmac_size);
338612f080e7Smrj cur_offset += dmac_size;
338712f080e7Smrj cookie++;
338812f080e7Smrj }
33897c478bd9Sstevel@tonic-gate
339012f080e7Smrj /* if we ended up with a zero sized window in the end, clean it up */
339112f080e7Smrj if (window->wd_size == 0) {
339212f080e7Smrj hp->dmai_nwin--;
339312f080e7Smrj window--;
33947c478bd9Sstevel@tonic-gate }
33957c478bd9Sstevel@tonic-gate
339612f080e7Smrj ASSERT(window->wd_trim.tr_trim_last == B_FALSE);
339712f080e7Smrj
339812f080e7Smrj if (!partial) {
339912f080e7Smrj return (DDI_DMA_MAPPED);
34007c478bd9Sstevel@tonic-gate }
340112f080e7Smrj
340212f080e7Smrj ASSERT(dma->dp_partial_required);
340312f080e7Smrj return (DDI_DMA_PARTIAL_MAP);
34047c478bd9Sstevel@tonic-gate }
34057c478bd9Sstevel@tonic-gate
34067c478bd9Sstevel@tonic-gate /*
340712f080e7Smrj * rootnex_setup_copybuf()
340812f080e7Smrj * Called in bind slowpath. Figures out if we're going to use the copy
340912f080e7Smrj * buffer, and if we do, sets up the basic state to handle it.
34107c478bd9Sstevel@tonic-gate */
341112f080e7Smrj static int
rootnex_setup_copybuf(ddi_dma_impl_t * hp,struct ddi_dma_req * dmareq,rootnex_dma_t * dma,ddi_dma_attr_t * attr)341212f080e7Smrj rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
341312f080e7Smrj rootnex_dma_t *dma, ddi_dma_attr_t *attr)
34147c478bd9Sstevel@tonic-gate {
341512f080e7Smrj rootnex_sglinfo_t *sinfo;
341612f080e7Smrj ddi_dma_attr_t lattr;
341712f080e7Smrj size_t max_copybuf;
341812f080e7Smrj int cansleep;
341912f080e7Smrj int e;
34207c478bd9Sstevel@tonic-gate
342150200e77SFrank Van Der Linden ASSERT(!dma->dp_dvma_used);
342212f080e7Smrj
342312f080e7Smrj sinfo = &dma->dp_sglinfo;
34247c478bd9Sstevel@tonic-gate
342536945f79Smrj /* read this first so it's consistent through the routine */
342636945f79Smrj max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK;
34277c478bd9Sstevel@tonic-gate
342812f080e7Smrj /* We need to call into the rootnex on ddi_dma_sync() */
342912f080e7Smrj hp->dmai_rflags &= ~DMP_NOSYNC;
34307c478bd9Sstevel@tonic-gate
343112f080e7Smrj /* make sure the copybuf size <= the max size */
343212f080e7Smrj dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf);
343312f080e7Smrj ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0);
34347c478bd9Sstevel@tonic-gate
34357c478bd9Sstevel@tonic-gate
343612f080e7Smrj /* convert the sleep flags */
343712f080e7Smrj if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
343812f080e7Smrj cansleep = 1;
343912f080e7Smrj } else {
344012f080e7Smrj cansleep = 0;
34417c478bd9Sstevel@tonic-gate }
34427c478bd9Sstevel@tonic-gate
34437c478bd9Sstevel@tonic-gate /*
3444d21b39ddSmrj * Allocate the actual copy buffer. This needs to fit within the DMA
3445d21b39ddSmrj * engine limits, so we can't use kmem_alloc... We don't need
3446d21b39ddSmrj * contiguous memory (sgllen) since we will be forcing windows on
3447d21b39ddSmrj * sgllen anyway.
34487c478bd9Sstevel@tonic-gate */
344912f080e7Smrj lattr = *attr;
345012f080e7Smrj lattr.dma_attr_align = MMU_PAGESIZE;
3451877f730dSGarrett D'Amore lattr.dma_attr_sgllen = -1; /* no limit */
345207c6692fSMark Johnson /*
345307c6692fSMark Johnson * if we're using the copy buffer because of seg, use that for our
345407c6692fSMark Johnson * upper address limit.
345507c6692fSMark Johnson */
345607c6692fSMark Johnson if (sinfo->si_bounce_on_seg) {
345707c6692fSMark Johnson lattr.dma_attr_addr_hi = lattr.dma_attr_seg;
345807c6692fSMark Johnson }
345912f080e7Smrj e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep,
346012f080e7Smrj 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL);
346112f080e7Smrj if (e != DDI_SUCCESS) {
346212f080e7Smrj return (DDI_DMA_NORESOURCES);
346312f080e7Smrj }
34647c478bd9Sstevel@tonic-gate
3465ef4ab52fSFrank Van Der Linden ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip,
346612f080e7Smrj size_t, dma->dp_copybuf_size);
34677c478bd9Sstevel@tonic-gate
346812f080e7Smrj return (DDI_SUCCESS);
346912f080e7Smrj }
34707c478bd9Sstevel@tonic-gate
34717c478bd9Sstevel@tonic-gate
347212f080e7Smrj /*
347312f080e7Smrj * rootnex_setup_windows()
347412f080e7Smrj * Called in bind slowpath to setup the window state. We always have windows
347512f080e7Smrj * in the slowpath. Even if the window count = 1.
347612f080e7Smrj */
347712f080e7Smrj static int
rootnex_setup_windows(ddi_dma_impl_t * hp,rootnex_dma_t * dma,ddi_dma_attr_t * attr,ddi_dma_obj_t * dmao,int kmflag)347812f080e7Smrj rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
347950200e77SFrank Van Der Linden ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
348012f080e7Smrj {
348112f080e7Smrj rootnex_window_t *windowp;
348212f080e7Smrj rootnex_sglinfo_t *sinfo;
348312f080e7Smrj size_t copy_state_size;
348412f080e7Smrj size_t win_state_size;
348512f080e7Smrj size_t state_available;
348612f080e7Smrj size_t space_needed;
348712f080e7Smrj uint_t copybuf_win;
348812f080e7Smrj uint_t maxxfer_win;
348912f080e7Smrj size_t space_used;
349012f080e7Smrj uint_t sglwin;
34917c478bd9Sstevel@tonic-gate
34927c478bd9Sstevel@tonic-gate
349312f080e7Smrj sinfo = &dma->dp_sglinfo;
34947c478bd9Sstevel@tonic-gate
349512f080e7Smrj dma->dp_current_win = 0;
349612f080e7Smrj hp->dmai_nwin = 0;
349712f080e7Smrj
349812f080e7Smrj /* If we don't need to do a partial, we only have one window */
349912f080e7Smrj if (!dma->dp_partial_required) {
350012f080e7Smrj dma->dp_max_win = 1;
350112f080e7Smrj
350212f080e7Smrj /*
350312f080e7Smrj * we need multiple windows, need to figure out the worse case number
350412f080e7Smrj * of windows.
350512f080e7Smrj */
350612f080e7Smrj } else {
350712f080e7Smrj /*
350812f080e7Smrj * if we need windows because we need more copy buffer that
350912f080e7Smrj * we allow, the worse case number of windows we could need
351012f080e7Smrj * here would be (copybuf space required / copybuf space that
351112f080e7Smrj * we have) plus one for remainder, and plus 2 to handle the
351212f080e7Smrj * extra pages on the trim for the first and last pages of the
351312f080e7Smrj * buffer (a page is the minimum window size so under the right
351412f080e7Smrj * attr settings, you could have a window for each page).
351512f080e7Smrj * The last page will only be hit here if the size is not a
351612f080e7Smrj * multiple of the granularity (which theoretically shouldn't
351712f080e7Smrj * be the case but never has been enforced, so we could have
351812f080e7Smrj * broken things without it).
351912f080e7Smrj */
352012f080e7Smrj if (sinfo->si_copybuf_req > dma->dp_copybuf_size) {
352112f080e7Smrj ASSERT(dma->dp_copybuf_size > 0);
352212f080e7Smrj copybuf_win = (sinfo->si_copybuf_req /
352312f080e7Smrj dma->dp_copybuf_size) + 1 + 2;
352412f080e7Smrj } else {
352512f080e7Smrj copybuf_win = 0;
352612f080e7Smrj }
35277c478bd9Sstevel@tonic-gate
35287c478bd9Sstevel@tonic-gate /*
352912f080e7Smrj * if we need windows because we have more cookies than the H/W
353012f080e7Smrj * can handle, the number of windows we would need here would
3531b57cd2d3SMark Johnson * be (cookie count / cookies count H/W supports minus 1[for
3532b57cd2d3SMark Johnson * trim]) plus one for remainder.
35337c478bd9Sstevel@tonic-gate */
3534877f730dSGarrett D'Amore if ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size) {
3535b57cd2d3SMark Johnson sglwin = (sinfo->si_sgl_size /
3536b57cd2d3SMark Johnson (attr->dma_attr_sgllen - 1)) + 1;
353712f080e7Smrj } else {
353812f080e7Smrj sglwin = 0;
35397c478bd9Sstevel@tonic-gate }
354012f080e7Smrj
354112f080e7Smrj /*
354212f080e7Smrj * if we need windows because we're binding more memory than the
354312f080e7Smrj * H/W can transfer at once, the number of windows we would need
354412f080e7Smrj * here would be (xfer count / max xfer H/W supports) plus one
354512f080e7Smrj * for remainder, and plus 2 to handle the extra pages on the
354612f080e7Smrj * trim (see above comment about trim)
354712f080e7Smrj */
354850200e77SFrank Van Der Linden if (dmao->dmao_size > dma->dp_maxxfer) {
354950200e77SFrank Van Der Linden maxxfer_win = (dmao->dmao_size /
355012f080e7Smrj dma->dp_maxxfer) + 1 + 2;
355112f080e7Smrj } else {
355212f080e7Smrj maxxfer_win = 0;
35537c478bd9Sstevel@tonic-gate }
355412f080e7Smrj dma->dp_max_win = copybuf_win + sglwin + maxxfer_win;
355512f080e7Smrj ASSERT(dma->dp_max_win > 0);
35567c478bd9Sstevel@tonic-gate }
355712f080e7Smrj win_state_size = dma->dp_max_win * sizeof (rootnex_window_t);
35587c478bd9Sstevel@tonic-gate
35597c478bd9Sstevel@tonic-gate /*
356012f080e7Smrj * Get space for window and potential copy buffer state. Before we
356112f080e7Smrj * go and allocate memory, see if we can get away with using what's
356212f080e7Smrj * left in the pre-allocted state or the dynamically allocated sgl.
35637c478bd9Sstevel@tonic-gate */
356412f080e7Smrj space_used = (uintptr_t)(sinfo->si_sgl_size *
356512f080e7Smrj sizeof (ddi_dma_cookie_t));
356612f080e7Smrj
356712f080e7Smrj /* if we dynamically allocated space for the cookies */
356812f080e7Smrj if (dma->dp_need_to_free_cookie) {
356912f080e7Smrj /* if we have more space in the pre-allocted buffer, use it */
357012f080e7Smrj ASSERT(space_used <= dma->dp_cookie_size);
357112f080e7Smrj if ((dma->dp_cookie_size - space_used) <=
357212f080e7Smrj rootnex_state->r_prealloc_size) {
357312f080e7Smrj state_available = rootnex_state->r_prealloc_size;
357412f080e7Smrj windowp = (rootnex_window_t *)dma->dp_prealloc_buffer;
35757c478bd9Sstevel@tonic-gate
357612f080e7Smrj /*
357712f080e7Smrj * else, we have more free space in the dynamically allocated
357812f080e7Smrj * buffer, i.e. the buffer wasn't worse case fragmented so we
357912f080e7Smrj * didn't need a lot of cookies.
358012f080e7Smrj */
358112f080e7Smrj } else {
358212f080e7Smrj state_available = dma->dp_cookie_size - space_used;
358312f080e7Smrj windowp = (rootnex_window_t *)
358412f080e7Smrj &dma->dp_cookies[sinfo->si_sgl_size];
358512f080e7Smrj }
35867c478bd9Sstevel@tonic-gate
358712f080e7Smrj /* we used the pre-alloced buffer */
358812f080e7Smrj } else {
358912f080e7Smrj ASSERT(space_used <= rootnex_state->r_prealloc_size);
359012f080e7Smrj state_available = rootnex_state->r_prealloc_size - space_used;
359112f080e7Smrj windowp = (rootnex_window_t *)
359212f080e7Smrj &dma->dp_cookies[sinfo->si_sgl_size];
359312f080e7Smrj }
35947c478bd9Sstevel@tonic-gate
359512f080e7Smrj /*
359612f080e7Smrj * figure out how much state we need to track the copy buffer. Add an
359712f080e7Smrj * addition 8 bytes for pointer alignemnt later.
359812f080e7Smrj */
359912f080e7Smrj if (dma->dp_copybuf_size > 0) {
360012f080e7Smrj copy_state_size = sinfo->si_max_pages *
360112f080e7Smrj sizeof (rootnex_pgmap_t);
360212f080e7Smrj } else {
360312f080e7Smrj copy_state_size = 0;
360412f080e7Smrj }
360512f080e7Smrj /* add an additional 8 bytes for pointer alignment */
360612f080e7Smrj space_needed = win_state_size + copy_state_size + 0x8;
36077c478bd9Sstevel@tonic-gate
360812f080e7Smrj /* if we have enough space already, use it */
360912f080e7Smrj if (state_available >= space_needed) {
361012f080e7Smrj dma->dp_window = windowp;
361112f080e7Smrj dma->dp_need_to_free_window = B_FALSE;
36127c478bd9Sstevel@tonic-gate
361312f080e7Smrj /* not enough space, need to allocate more. */
361412f080e7Smrj } else {
361512f080e7Smrj dma->dp_window = kmem_alloc(space_needed, kmflag);
361612f080e7Smrj if (dma->dp_window == NULL) {
361712f080e7Smrj return (DDI_DMA_NORESOURCES);
361812f080e7Smrj }
361912f080e7Smrj dma->dp_need_to_free_window = B_TRUE;
362012f080e7Smrj dma->dp_window_size = space_needed;
3621ef4ab52fSFrank Van Der Linden ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *,
362212f080e7Smrj dma->dp_dip, size_t, space_needed);
362312f080e7Smrj }
36247c478bd9Sstevel@tonic-gate
362512f080e7Smrj /*
362612f080e7Smrj * we allocate copy buffer state and window state at the same time.
362712f080e7Smrj * setup our copy buffer state pointers. Make sure it's aligned.
362812f080e7Smrj */
362912f080e7Smrj if (dma->dp_copybuf_size > 0) {
363012f080e7Smrj dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t)
363112f080e7Smrj &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7);
36327c478bd9Sstevel@tonic-gate
363312f080e7Smrj } else {
363412f080e7Smrj dma->dp_pgmap = NULL;
36357c478bd9Sstevel@tonic-gate }
363612f080e7Smrj
36377c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
36387c478bd9Sstevel@tonic-gate }
36397c478bd9Sstevel@tonic-gate
364012f080e7Smrj
364112f080e7Smrj /*
364212f080e7Smrj * rootnex_teardown_copybuf()
364312f080e7Smrj * cleans up after rootnex_setup_copybuf()
364412f080e7Smrj */
364512f080e7Smrj static void
rootnex_teardown_copybuf(rootnex_dma_t * dma)364612f080e7Smrj rootnex_teardown_copybuf(rootnex_dma_t *dma)
36477c478bd9Sstevel@tonic-gate {
364812f080e7Smrj /* if we allocated a copy buffer, free it */
364912f080e7Smrj if (dma->dp_cbaddr != NULL) {
36507b93957cSeota i_ddi_mem_free(dma->dp_cbaddr, NULL);
365112f080e7Smrj }
365212f080e7Smrj }
36537c478bd9Sstevel@tonic-gate
36547c478bd9Sstevel@tonic-gate
365512f080e7Smrj /*
365612f080e7Smrj * rootnex_teardown_windows()
365712f080e7Smrj * cleans up after rootnex_setup_windows()
365812f080e7Smrj */
365912f080e7Smrj static void
rootnex_teardown_windows(rootnex_dma_t * dma)366012f080e7Smrj rootnex_teardown_windows(rootnex_dma_t *dma)
366112f080e7Smrj {
366212f080e7Smrj /*
366312f080e7Smrj * if we had to allocate window state on the last bind (because we
366412f080e7Smrj * didn't have enough pre-allocated space in the handle), free it.
366512f080e7Smrj */
366612f080e7Smrj if (dma->dp_need_to_free_window) {
366712f080e7Smrj kmem_free(dma->dp_window, dma->dp_window_size);
36687c478bd9Sstevel@tonic-gate }
36697c478bd9Sstevel@tonic-gate }
36707c478bd9Sstevel@tonic-gate
367112f080e7Smrj
367212f080e7Smrj /*
367312f080e7Smrj * rootnex_init_win()
367412f080e7Smrj * Called in bind slow path during creation of a new window. Initializes
367512f080e7Smrj * window state to default values.
367612f080e7Smrj */
367712f080e7Smrj /*ARGSUSED*/
367812f080e7Smrj static void
rootnex_init_win(ddi_dma_impl_t * hp,rootnex_dma_t * dma,rootnex_window_t * window,ddi_dma_cookie_t * cookie,off_t cur_offset)367912f080e7Smrj rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
368012f080e7Smrj rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset)
36817c478bd9Sstevel@tonic-gate {
368212f080e7Smrj hp->dmai_nwin++;
368312f080e7Smrj window->wd_dosync = B_FALSE;
368412f080e7Smrj window->wd_offset = cur_offset;
368512f080e7Smrj window->wd_size = 0;
368612f080e7Smrj window->wd_first_cookie = cookie;
368712f080e7Smrj window->wd_cookie_cnt = 0;
368812f080e7Smrj window->wd_trim.tr_trim_first = B_FALSE;
368912f080e7Smrj window->wd_trim.tr_trim_last = B_FALSE;
369012f080e7Smrj window->wd_trim.tr_first_copybuf_win = B_FALSE;
369112f080e7Smrj window->wd_trim.tr_last_copybuf_win = B_FALSE;
369212f080e7Smrj }
36937c478bd9Sstevel@tonic-gate
369412f080e7Smrj
369512f080e7Smrj /*
369612f080e7Smrj * rootnex_setup_cookie()
369712f080e7Smrj * Called in the bind slow path when the sgl uses the copy buffer. If any of
369812f080e7Smrj * the sgl uses the copy buffer, we need to go through each cookie, figure
369912f080e7Smrj * out if it uses the copy buffer, and if it does, save away everything we'll
370012f080e7Smrj * need during sync.
370112f080e7Smrj */
370212f080e7Smrj static void
rootnex_setup_cookie(ddi_dma_obj_t * dmar_object,rootnex_dma_t * dma,ddi_dma_cookie_t * cookie,off_t cur_offset,size_t * copybuf_used,page_t ** cur_pp)370312f080e7Smrj rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma,
370412f080e7Smrj ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used,
370512f080e7Smrj page_t **cur_pp)
370612f080e7Smrj {
370712f080e7Smrj boolean_t copybuf_sz_power_2;
370812f080e7Smrj rootnex_sglinfo_t *sinfo;
3709843e1988Sjohnlev paddr_t paddr;
371012f080e7Smrj uint_t pidx;
371112f080e7Smrj uint_t pcnt;
371212f080e7Smrj off_t poff;
371312f080e7Smrj pfn_t pfn;
37147c478bd9Sstevel@tonic-gate
371550200e77SFrank Van Der Linden ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR);
371650200e77SFrank Van Der Linden
371712f080e7Smrj sinfo = &dma->dp_sglinfo;
37187c478bd9Sstevel@tonic-gate
371912f080e7Smrj /*
372012f080e7Smrj * Calculate the page index relative to the start of the buffer. The
372112f080e7Smrj * index to the current page for our buffer is the offset into the
372212f080e7Smrj * first page of the buffer plus our current offset into the buffer
372312f080e7Smrj * itself, shifted of course...
372412f080e7Smrj */
372512f080e7Smrj pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT;
372612f080e7Smrj ASSERT(pidx < sinfo->si_max_pages);
37277c478bd9Sstevel@tonic-gate
372812f080e7Smrj /* if this cookie uses the copy buffer */
372912f080e7Smrj if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
37307c478bd9Sstevel@tonic-gate /*
373112f080e7Smrj * NOTE: we know that since this cookie uses the copy buffer, it
373212f080e7Smrj * is <= MMU_PAGESIZE.
37337c478bd9Sstevel@tonic-gate */
37347c478bd9Sstevel@tonic-gate
37357c478bd9Sstevel@tonic-gate /*
373612f080e7Smrj * get the offset into the page. For the 64-bit kernel, get the
373712f080e7Smrj * pfn which we'll use with seg kpm.
37387c478bd9Sstevel@tonic-gate */
3739843e1988Sjohnlev poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
3740843e1988Sjohnlev /* mfn_to_pfn() is a NOP on i86pc */
3741843e1988Sjohnlev pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT);
37427c478bd9Sstevel@tonic-gate
374312f080e7Smrj /* figure out if the copybuf size is a power of 2 */
3744de710d24SJosef 'Jeff' Sipek if (!ISP2(dma->dp_copybuf_size)) {
374512f080e7Smrj copybuf_sz_power_2 = B_FALSE;
374612f080e7Smrj } else {
374712f080e7Smrj copybuf_sz_power_2 = B_TRUE;
37487c478bd9Sstevel@tonic-gate }
374912f080e7Smrj
375012f080e7Smrj /* This page uses the copy buffer */
375112f080e7Smrj dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE;
375212f080e7Smrj
375312f080e7Smrj /*
375412f080e7Smrj * save the copy buffer KVA that we'll use with this page.
375512f080e7Smrj * if we still fit within the copybuf, it's a simple add.
375612f080e7Smrj * otherwise, we need to wrap over using & or % accordingly.
375712f080e7Smrj */
375812f080e7Smrj if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) {
375912f080e7Smrj dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr +
376012f080e7Smrj *copybuf_used;
37617c478bd9Sstevel@tonic-gate } else {
376212f080e7Smrj if (copybuf_sz_power_2) {
376312f080e7Smrj dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
376412f080e7Smrj (uintptr_t)dma->dp_cbaddr +
376512f080e7Smrj (*copybuf_used &
376612f080e7Smrj (dma->dp_copybuf_size - 1)));
376712f080e7Smrj } else {
376812f080e7Smrj dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
376912f080e7Smrj (uintptr_t)dma->dp_cbaddr +
377012f080e7Smrj (*copybuf_used % dma->dp_copybuf_size));
37717c478bd9Sstevel@tonic-gate }
37727c478bd9Sstevel@tonic-gate }
37737c478bd9Sstevel@tonic-gate
377412f080e7Smrj /*
377512f080e7Smrj * over write the cookie physical address with the address of
377612f080e7Smrj * the physical address of the copy buffer page that we will
377712f080e7Smrj * use.
377812f080e7Smrj */
3779843e1988Sjohnlev paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
378012f080e7Smrj dma->dp_pgmap[pidx].pm_cbaddr)) + poff;
378112f080e7Smrj
378250200e77SFrank Van Der Linden cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
3783843e1988Sjohnlev
378412f080e7Smrj /* if we have a kernel VA, it's easy, just save that address */
378512f080e7Smrj if ((dmar_object->dmao_type != DMA_OTYP_PAGES) &&
378612f080e7Smrj (sinfo->si_asp == &kas)) {
37877c478bd9Sstevel@tonic-gate /*
378812f080e7Smrj * save away the page aligned virtual address of the
378912f080e7Smrj * driver buffer. Offsets are handled in the sync code.
37907c478bd9Sstevel@tonic-gate */
379112f080e7Smrj dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t)
379212f080e7Smrj dmar_object->dmao_obj.virt_obj.v_addr + cur_offset)
379312f080e7Smrj & MMU_PAGEMASK);
379412f080e7Smrj
379512f080e7Smrj /* we don't have a kernel VA. We need one for the bcopy. */
37967c478bd9Sstevel@tonic-gate } else {
379712f080e7Smrj /*
379812f080e7Smrj * for the 64-bit kernel, it's easy. We use seg kpm to
379912f080e7Smrj * get a Kernel VA for the corresponding pfn.
380012f080e7Smrj */
380112f080e7Smrj dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn);
380212f080e7Smrj /* go to the next page_t */
380312f080e7Smrj if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
380412f080e7Smrj *cur_pp = (*cur_pp)->p_next;
380512f080e7Smrj }
380612f080e7Smrj }
38077c478bd9Sstevel@tonic-gate
380812f080e7Smrj /* add to the copy buffer count */
380912f080e7Smrj *copybuf_used += MMU_PAGESIZE;
38107c478bd9Sstevel@tonic-gate
381112f080e7Smrj /*
381212f080e7Smrj * This cookie doesn't use the copy buffer. Walk through the pages this
381312f080e7Smrj * cookie occupies to reflect this.
381412f080e7Smrj */
381512f080e7Smrj } else {
381612f080e7Smrj /*
381712f080e7Smrj * figure out how many pages the cookie occupies. We need to
381812f080e7Smrj * use the original page offset of the buffer and the cookies
381912f080e7Smrj * offset in the buffer to do this.
382012f080e7Smrj */
382112f080e7Smrj poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET;
382212f080e7Smrj pcnt = mmu_btopr(cookie->dmac_size + poff);
38237c478bd9Sstevel@tonic-gate
382412f080e7Smrj while (pcnt > 0) {
382512f080e7Smrj dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE;
38267c478bd9Sstevel@tonic-gate
382712f080e7Smrj /*
382812f080e7Smrj * we need to update pidx and cur_pp or we'll loose
382912f080e7Smrj * track of where we are.
383012f080e7Smrj */
383112f080e7Smrj if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
383212f080e7Smrj *cur_pp = (*cur_pp)->p_next;
383312f080e7Smrj }
383412f080e7Smrj pidx++;
383512f080e7Smrj pcnt--;
383612f080e7Smrj }
38377c478bd9Sstevel@tonic-gate }
38387c478bd9Sstevel@tonic-gate }
38397c478bd9Sstevel@tonic-gate
384012f080e7Smrj
38417c478bd9Sstevel@tonic-gate /*
384212f080e7Smrj * rootnex_sgllen_window_boundary()
384312f080e7Smrj * Called in the bind slow path when the next cookie causes us to exceed (in
384412f080e7Smrj * this case == since we start at 0 and sgllen starts at 1) the maximum sgl
384512f080e7Smrj * length supported by the DMA H/W.
38467c478bd9Sstevel@tonic-gate */
38477c478bd9Sstevel@tonic-gate static int
rootnex_sgllen_window_boundary(ddi_dma_impl_t * hp,rootnex_dma_t * dma,rootnex_window_t ** windowp,ddi_dma_cookie_t * cookie,ddi_dma_attr_t * attr,off_t cur_offset)384812f080e7Smrj rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
384912f080e7Smrj rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr,
385012f080e7Smrj off_t cur_offset)
38517c478bd9Sstevel@tonic-gate {
385212f080e7Smrj off_t new_offset;
385312f080e7Smrj size_t trim_sz;
385412f080e7Smrj off_t coffset;
38557c478bd9Sstevel@tonic-gate
38567c478bd9Sstevel@tonic-gate
385712f080e7Smrj /*
385812f080e7Smrj * if we know we'll never have to trim, it's pretty easy. Just move to
385912f080e7Smrj * the next window and init it. We're done.
386012f080e7Smrj */
386112f080e7Smrj if (!dma->dp_trim_required) {
386212f080e7Smrj (*windowp)++;
386312f080e7Smrj rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
386412f080e7Smrj (*windowp)->wd_cookie_cnt++;
386512f080e7Smrj (*windowp)->wd_size = cookie->dmac_size;
386612f080e7Smrj return (DDI_SUCCESS);
38677c478bd9Sstevel@tonic-gate }
38687c478bd9Sstevel@tonic-gate
386912f080e7Smrj /* figure out how much we need to trim from the window */
387012f080e7Smrj ASSERT(attr->dma_attr_granular != 0);
387112f080e7Smrj if (dma->dp_granularity_power_2) {
387212f080e7Smrj trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1);
387312f080e7Smrj } else {
387412f080e7Smrj trim_sz = (*windowp)->wd_size % attr->dma_attr_granular;
38757c478bd9Sstevel@tonic-gate }
387612f080e7Smrj
387712f080e7Smrj /* The window's a whole multiple of granularity. We're done */
387812f080e7Smrj if (trim_sz == 0) {
387912f080e7Smrj (*windowp)++;
388012f080e7Smrj rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
388112f080e7Smrj (*windowp)->wd_cookie_cnt++;
388212f080e7Smrj (*windowp)->wd_size = cookie->dmac_size;
388312f080e7Smrj return (DDI_SUCCESS);
38847c478bd9Sstevel@tonic-gate }
38857c478bd9Sstevel@tonic-gate
388612f080e7Smrj /*
388712f080e7Smrj * The window's not a whole multiple of granularity, since we know this
388812f080e7Smrj * is due to the sgllen, we need to go back to the last cookie and trim
388912f080e7Smrj * that one, add the left over part of the old cookie into the new
389012f080e7Smrj * window, and then add in the new cookie into the new window.
389112f080e7Smrj */
38927c478bd9Sstevel@tonic-gate
389312f080e7Smrj /*
389412f080e7Smrj * make sure the driver isn't making us do something bad... Trimming and
389512f080e7Smrj * sgllen == 1 don't go together.
389612f080e7Smrj */
389712f080e7Smrj if (attr->dma_attr_sgllen == 1) {
389812f080e7Smrj return (DDI_DMA_NOMAPPING);
389912f080e7Smrj }
39007c478bd9Sstevel@tonic-gate
390112f080e7Smrj /*
390212f080e7Smrj * first, setup the current window to account for the trim. Need to go
390312f080e7Smrj * back to the last cookie for this.
390412f080e7Smrj */
390512f080e7Smrj cookie--;
390612f080e7Smrj (*windowp)->wd_trim.tr_trim_last = B_TRUE;
390712f080e7Smrj (*windowp)->wd_trim.tr_last_cookie = cookie;
3908843e1988Sjohnlev (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
390912f080e7Smrj ASSERT(cookie->dmac_size > trim_sz);
391012f080e7Smrj (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
391112f080e7Smrj (*windowp)->wd_size -= trim_sz;
391212f080e7Smrj
391312f080e7Smrj /* save the buffer offsets for the next window */
391412f080e7Smrj coffset = cookie->dmac_size - trim_sz;
391512f080e7Smrj new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
39167c478bd9Sstevel@tonic-gate
39177c478bd9Sstevel@tonic-gate /*
391812f080e7Smrj * set this now in case this is the first window. all other cases are
391912f080e7Smrj * set in dma_win()
392012f080e7Smrj */
392112f080e7Smrj cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
392212f080e7Smrj
392312f080e7Smrj /*
392412f080e7Smrj * initialize the next window using what's left over in the previous
392512f080e7Smrj * cookie.
392612f080e7Smrj */
392712f080e7Smrj (*windowp)++;
392812f080e7Smrj rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
392912f080e7Smrj (*windowp)->wd_cookie_cnt++;
393012f080e7Smrj (*windowp)->wd_trim.tr_trim_first = B_TRUE;
3931843e1988Sjohnlev (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
393212f080e7Smrj (*windowp)->wd_trim.tr_first_size = trim_sz;
393312f080e7Smrj if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
393412f080e7Smrj (*windowp)->wd_dosync = B_TRUE;
393512f080e7Smrj }
393612f080e7Smrj
393712f080e7Smrj /*
393812f080e7Smrj * now go back to the current cookie and add it to the new window. set
393912f080e7Smrj * the new window size to the what was left over from the previous
394012f080e7Smrj * cookie and what's in the current cookie.
39417c478bd9Sstevel@tonic-gate */
394212f080e7Smrj cookie++;
394312f080e7Smrj (*windowp)->wd_cookie_cnt++;
394412f080e7Smrj (*windowp)->wd_size = trim_sz + cookie->dmac_size;
39457c478bd9Sstevel@tonic-gate
394612f080e7Smrj /*
394712f080e7Smrj * trim plus the next cookie could put us over maxxfer (a cookie can be
394812f080e7Smrj * a max size of maxxfer). Handle that case.
394912f080e7Smrj */
395012f080e7Smrj if ((*windowp)->wd_size > dma->dp_maxxfer) {
39517c478bd9Sstevel@tonic-gate /*
395212f080e7Smrj * maxxfer is already a whole multiple of granularity, and this
395312f080e7Smrj * trim will be <= the previous trim (since a cookie can't be
395412f080e7Smrj * larger than maxxfer). Make things simple here.
39557c478bd9Sstevel@tonic-gate */
395612f080e7Smrj trim_sz = (*windowp)->wd_size - dma->dp_maxxfer;
395712f080e7Smrj (*windowp)->wd_trim.tr_trim_last = B_TRUE;
395812f080e7Smrj (*windowp)->wd_trim.tr_last_cookie = cookie;
3959843e1988Sjohnlev (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
396012f080e7Smrj (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
396112f080e7Smrj (*windowp)->wd_size -= trim_sz;
396212f080e7Smrj ASSERT((*windowp)->wd_size == dma->dp_maxxfer);
396312f080e7Smrj
396412f080e7Smrj /* save the buffer offsets for the next window */
396512f080e7Smrj coffset = cookie->dmac_size - trim_sz;
396612f080e7Smrj new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
396712f080e7Smrj
396812f080e7Smrj /* setup the next window */
396912f080e7Smrj (*windowp)++;
397012f080e7Smrj rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
397112f080e7Smrj (*windowp)->wd_cookie_cnt++;
397212f080e7Smrj (*windowp)->wd_trim.tr_trim_first = B_TRUE;
3973843e1988Sjohnlev (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
397412f080e7Smrj coffset;
397512f080e7Smrj (*windowp)->wd_trim.tr_first_size = trim_sz;
397612f080e7Smrj }
39777c478bd9Sstevel@tonic-gate
397812f080e7Smrj return (DDI_SUCCESS);
397912f080e7Smrj }
39807c478bd9Sstevel@tonic-gate
39817c478bd9Sstevel@tonic-gate
398212f080e7Smrj /*
398312f080e7Smrj * rootnex_copybuf_window_boundary()
398412f080e7Smrj * Called in bind slowpath when we get to a window boundary because we used
398512f080e7Smrj * up all the copy buffer that we have.
398612f080e7Smrj */
398712f080e7Smrj static int
rootnex_copybuf_window_boundary(ddi_dma_impl_t * hp,rootnex_dma_t * dma,rootnex_window_t ** windowp,ddi_dma_cookie_t * cookie,off_t cur_offset,size_t * copybuf_used)398812f080e7Smrj rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
398912f080e7Smrj rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset,
399012f080e7Smrj size_t *copybuf_used)
399112f080e7Smrj {
399212f080e7Smrj rootnex_sglinfo_t *sinfo;
399312f080e7Smrj off_t new_offset;
399412f080e7Smrj size_t trim_sz;
3995843e1988Sjohnlev paddr_t paddr;
399612f080e7Smrj off_t coffset;
399712f080e7Smrj uint_t pidx;
399812f080e7Smrj off_t poff;
39997c478bd9Sstevel@tonic-gate
4000584b574aSToomas Soome pidx = 0;
400112f080e7Smrj sinfo = &dma->dp_sglinfo;
40027c478bd9Sstevel@tonic-gate
400312f080e7Smrj /*
400412f080e7Smrj * the copy buffer should be a whole multiple of page size. We know that
400512f080e7Smrj * this cookie is <= MMU_PAGESIZE.
400612f080e7Smrj */
400712f080e7Smrj ASSERT(cookie->dmac_size <= MMU_PAGESIZE);
40087c478bd9Sstevel@tonic-gate
400912f080e7Smrj /* reset copybuf used */
401012f080e7Smrj *copybuf_used = 0;
40117c478bd9Sstevel@tonic-gate
401212f080e7Smrj /*
401312f080e7Smrj * if we don't have to trim (since granularity is set to 1), go to the
401412f080e7Smrj * next window and add the current cookie to it. We know the current
401512f080e7Smrj * cookie uses the copy buffer since we're in this code path.
401612f080e7Smrj */
401712f080e7Smrj if (!dma->dp_trim_required) {
401812f080e7Smrj (*windowp)++;
401912f080e7Smrj rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
402012f080e7Smrj
402112f080e7Smrj /* Add this cookie to the new window */
402212f080e7Smrj (*windowp)->wd_cookie_cnt++;
402312f080e7Smrj (*windowp)->wd_size += cookie->dmac_size;
402412f080e7Smrj *copybuf_used += MMU_PAGESIZE;
402512f080e7Smrj return (DDI_SUCCESS);
402612f080e7Smrj }
40277c478bd9Sstevel@tonic-gate
402812f080e7Smrj /*
402912f080e7Smrj * *** may need to trim, figure it out.
403012f080e7Smrj */
40317c478bd9Sstevel@tonic-gate
403212f080e7Smrj /* figure out how much we need to trim from the window */
403312f080e7Smrj if (dma->dp_granularity_power_2) {
403412f080e7Smrj trim_sz = (*windowp)->wd_size &
403512f080e7Smrj (hp->dmai_attr.dma_attr_granular - 1);
403612f080e7Smrj } else {
403712f080e7Smrj trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular;
403812f080e7Smrj }
403912f080e7Smrj
404012f080e7Smrj /*
404112f080e7Smrj * if the window's a whole multiple of granularity, go to the next
404212f080e7Smrj * window, init it, then add in the current cookie. We know the current
404312f080e7Smrj * cookie uses the copy buffer since we're in this code path.
404412f080e7Smrj */
404512f080e7Smrj if (trim_sz == 0) {
404612f080e7Smrj (*windowp)++;
404712f080e7Smrj rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
404812f080e7Smrj
404912f080e7Smrj /* Add this cookie to the new window */
405012f080e7Smrj (*windowp)->wd_cookie_cnt++;
405112f080e7Smrj (*windowp)->wd_size += cookie->dmac_size;
405212f080e7Smrj *copybuf_used += MMU_PAGESIZE;
40537c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
40547c478bd9Sstevel@tonic-gate }
40557c478bd9Sstevel@tonic-gate
405612f080e7Smrj /*
405712f080e7Smrj * *** We figured it out, we definitly need to trim
405812f080e7Smrj */
40597c478bd9Sstevel@tonic-gate
406012f080e7Smrj /*
406112f080e7Smrj * make sure the driver isn't making us do something bad...
406212f080e7Smrj * Trimming and sgllen == 1 don't go together.
406312f080e7Smrj */
406412f080e7Smrj if (hp->dmai_attr.dma_attr_sgllen == 1) {
406512f080e7Smrj return (DDI_DMA_NOMAPPING);
406612f080e7Smrj }
40677c478bd9Sstevel@tonic-gate
406812f080e7Smrj /*
406912f080e7Smrj * first, setup the current window to account for the trim. Need to go
407012f080e7Smrj * back to the last cookie for this. Some of the last cookie will be in
407112f080e7Smrj * the current window, and some of the last cookie will be in the new
407212f080e7Smrj * window. All of the current cookie will be in the new window.
407312f080e7Smrj */
407412f080e7Smrj cookie--;
407512f080e7Smrj (*windowp)->wd_trim.tr_trim_last = B_TRUE;
407612f080e7Smrj (*windowp)->wd_trim.tr_last_cookie = cookie;
4077843e1988Sjohnlev (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
407812f080e7Smrj ASSERT(cookie->dmac_size > trim_sz);
407912f080e7Smrj (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
408012f080e7Smrj (*windowp)->wd_size -= trim_sz;
40817c478bd9Sstevel@tonic-gate
408212f080e7Smrj /*
408312f080e7Smrj * we're trimming the last cookie (not the current cookie). So that
408412f080e7Smrj * last cookie may have or may not have been using the copy buffer (
408512f080e7Smrj * we know the cookie passed in uses the copy buffer since we're in
408612f080e7Smrj * this code path).
408712f080e7Smrj *
408812f080e7Smrj * If the last cookie doesn't use the copy buffer, nothing special to
408912f080e7Smrj * do. However, if it does uses the copy buffer, it will be both the
409012f080e7Smrj * last page in the current window and the first page in the next
409112f080e7Smrj * window. Since we are reusing the copy buffer (and KVA space on the
409212f080e7Smrj * 32-bit kernel), this page will use the end of the copy buffer in the
409312f080e7Smrj * current window, and the start of the copy buffer in the next window.
409412f080e7Smrj * Track that info... The cookie physical address was already set to
409512f080e7Smrj * the copy buffer physical address in setup_cookie..
409612f080e7Smrj */
409712f080e7Smrj if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
409812f080e7Smrj pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset +
409912f080e7Smrj (*windowp)->wd_size) >> MMU_PAGESHIFT;
410012f080e7Smrj (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE;
410112f080e7Smrj (*windowp)->wd_trim.tr_last_pidx = pidx;
410212f080e7Smrj (*windowp)->wd_trim.tr_last_cbaddr =
410312f080e7Smrj dma->dp_pgmap[pidx].pm_cbaddr;
41047c478bd9Sstevel@tonic-gate }
41057c478bd9Sstevel@tonic-gate
410612f080e7Smrj /* save the buffer offsets for the next window */
410712f080e7Smrj coffset = cookie->dmac_size - trim_sz;
410812f080e7Smrj new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
41097c478bd9Sstevel@tonic-gate
411012f080e7Smrj /*
411112f080e7Smrj * set this now in case this is the first window. all other cases are
411212f080e7Smrj * set in dma_win()
411312f080e7Smrj */
411412f080e7Smrj cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
41157c478bd9Sstevel@tonic-gate
411612f080e7Smrj /*
411712f080e7Smrj * initialize the next window using what's left over in the previous
411812f080e7Smrj * cookie.
411912f080e7Smrj */
412012f080e7Smrj (*windowp)++;
412112f080e7Smrj rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
412212f080e7Smrj (*windowp)->wd_cookie_cnt++;
412312f080e7Smrj (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4124843e1988Sjohnlev (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
412512f080e7Smrj (*windowp)->wd_trim.tr_first_size = trim_sz;
41267c478bd9Sstevel@tonic-gate
412712f080e7Smrj /*
412812f080e7Smrj * again, we're tracking if the last cookie uses the copy buffer.
412912f080e7Smrj * read the comment above for more info on why we need to track
413012f080e7Smrj * additional state.
413112f080e7Smrj *
413212f080e7Smrj * For the first cookie in the new window, we need reset the physical
413312f080e7Smrj * address to DMA into to the start of the copy buffer plus any
413412f080e7Smrj * initial page offset which may be present.
413512f080e7Smrj */
413612f080e7Smrj if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
413712f080e7Smrj (*windowp)->wd_dosync = B_TRUE;
413812f080e7Smrj (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE;
413912f080e7Smrj (*windowp)->wd_trim.tr_first_pidx = pidx;
414012f080e7Smrj (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr;
414112f080e7Smrj poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET;
4142843e1988Sjohnlev
4143843e1988Sjohnlev paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) +
4144843e1988Sjohnlev poff;
4145843e1988Sjohnlev (*windowp)->wd_trim.tr_first_paddr =
414650200e77SFrank Van Der Linden ROOTNEX_PADDR_TO_RBASE(paddr);
4147843e1988Sjohnlev
414812f080e7Smrj /* account for the cookie copybuf usage in the new window */
414912f080e7Smrj *copybuf_used += MMU_PAGESIZE;
41507c478bd9Sstevel@tonic-gate
415112f080e7Smrj /*
415212f080e7Smrj * every piece of code has to have a hack, and here is this
415312f080e7Smrj * ones :-)
415412f080e7Smrj *
415512f080e7Smrj * There is a complex interaction between setup_cookie and the
415612f080e7Smrj * copybuf window boundary. The complexity had to be in either
415712f080e7Smrj * the maxxfer window, or the copybuf window, and I chose the
415812f080e7Smrj * copybuf code.
415912f080e7Smrj *
416012f080e7Smrj * So in this code path, we have taken the last cookie,
416112f080e7Smrj * virtually broken it in half due to the trim, and it happens
416212f080e7Smrj * to use the copybuf which further complicates life. At the
416312f080e7Smrj * same time, we have already setup the current cookie, which
416412f080e7Smrj * is now wrong. More background info: the current cookie uses
416512f080e7Smrj * the copybuf, so it is only a page long max. So we need to
416612f080e7Smrj * fix the current cookies copy buffer address, physical
416712f080e7Smrj * address, and kva for the 32-bit kernel. We due this by
416812f080e7Smrj * bumping them by page size (of course, we can't due this on
416912f080e7Smrj * the physical address since the copy buffer may not be
417012f080e7Smrj * physically contiguous).
417112f080e7Smrj */
417212f080e7Smrj cookie++;
417312f080e7Smrj dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE;
4174843e1988Sjohnlev poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
4175843e1988Sjohnlev
4176843e1988Sjohnlev paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
417712f080e7Smrj dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff;
417850200e77SFrank Van Der Linden cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
4179843e1988Sjohnlev
418012f080e7Smrj } else {
418112f080e7Smrj /* go back to the current cookie */
418212f080e7Smrj cookie++;
418312f080e7Smrj }
41847c478bd9Sstevel@tonic-gate
418512f080e7Smrj /*
418612f080e7Smrj * add the current cookie to the new window. set the new window size to
418712f080e7Smrj * the what was left over from the previous cookie and what's in the
418812f080e7Smrj * current cookie.
418912f080e7Smrj */
419012f080e7Smrj (*windowp)->wd_cookie_cnt++;
419112f080e7Smrj (*windowp)->wd_size = trim_sz + cookie->dmac_size;
419212f080e7Smrj ASSERT((*windowp)->wd_size < dma->dp_maxxfer);
41937c478bd9Sstevel@tonic-gate
419412f080e7Smrj /*
419512f080e7Smrj * we know that the cookie passed in always uses the copy buffer. We
419612f080e7Smrj * wouldn't be here if it didn't.
419712f080e7Smrj */
419812f080e7Smrj *copybuf_used += MMU_PAGESIZE;
41997c478bd9Sstevel@tonic-gate
420012f080e7Smrj return (DDI_SUCCESS);
420112f080e7Smrj }
42027c478bd9Sstevel@tonic-gate
42037c478bd9Sstevel@tonic-gate
420412f080e7Smrj /*
420512f080e7Smrj * rootnex_maxxfer_window_boundary()
420612f080e7Smrj * Called in bind slowpath when we get to a window boundary because we will
420712f080e7Smrj * go over maxxfer.
420812f080e7Smrj */
420912f080e7Smrj static int
rootnex_maxxfer_window_boundary(ddi_dma_impl_t * hp,rootnex_dma_t * dma,rootnex_window_t ** windowp,ddi_dma_cookie_t * cookie)421012f080e7Smrj rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
421112f080e7Smrj rootnex_window_t **windowp, ddi_dma_cookie_t *cookie)
421212f080e7Smrj {
421312f080e7Smrj size_t dmac_size;
421412f080e7Smrj off_t new_offset;
421512f080e7Smrj size_t trim_sz;
421612f080e7Smrj off_t coffset;
42177c478bd9Sstevel@tonic-gate
42187c478bd9Sstevel@tonic-gate
421912f080e7Smrj /*
422012f080e7Smrj * calculate how much we have to trim off of the current cookie to equal
422112f080e7Smrj * maxxfer. We don't have to account for granularity here since our
422212f080e7Smrj * maxxfer already takes that into account.
422312f080e7Smrj */
422412f080e7Smrj trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer;
422512f080e7Smrj ASSERT(trim_sz <= cookie->dmac_size);
422612f080e7Smrj ASSERT(trim_sz <= dma->dp_maxxfer);
422712f080e7Smrj
422812f080e7Smrj /* save cookie size since we need it later and we might change it */
422912f080e7Smrj dmac_size = cookie->dmac_size;
423012f080e7Smrj
423112f080e7Smrj /*
423212f080e7Smrj * if we're not trimming the entire cookie, setup the current window to
423312f080e7Smrj * account for the trim.
423412f080e7Smrj */
423512f080e7Smrj if (trim_sz < cookie->dmac_size) {
423612f080e7Smrj (*windowp)->wd_cookie_cnt++;
423712f080e7Smrj (*windowp)->wd_trim.tr_trim_last = B_TRUE;
423812f080e7Smrj (*windowp)->wd_trim.tr_last_cookie = cookie;
4239843e1988Sjohnlev (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
424012f080e7Smrj (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
424112f080e7Smrj (*windowp)->wd_size = dma->dp_maxxfer;
424212f080e7Smrj
424312f080e7Smrj /*
424412f080e7Smrj * set the adjusted cookie size now in case this is the first
424512f080e7Smrj * window. All other windows are taken care of in get win
424612f080e7Smrj */
424712f080e7Smrj cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
424812f080e7Smrj }
424912f080e7Smrj
425012f080e7Smrj /*
425112f080e7Smrj * coffset is the current offset within the cookie, new_offset is the
425212f080e7Smrj * current offset with the entire buffer.
425312f080e7Smrj */
425412f080e7Smrj coffset = dmac_size - trim_sz;
425512f080e7Smrj new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
425612f080e7Smrj
425712f080e7Smrj /* initialize the next window */
425812f080e7Smrj (*windowp)++;
425912f080e7Smrj rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
426012f080e7Smrj (*windowp)->wd_cookie_cnt++;
426112f080e7Smrj (*windowp)->wd_size = trim_sz;
426212f080e7Smrj if (trim_sz < dmac_size) {
426312f080e7Smrj (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4264843e1988Sjohnlev (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
426512f080e7Smrj coffset;
426612f080e7Smrj (*windowp)->wd_trim.tr_first_size = trim_sz;
426712f080e7Smrj }
426812f080e7Smrj
426912f080e7Smrj return (DDI_SUCCESS);
42707c478bd9Sstevel@tonic-gate }
42717c478bd9Sstevel@tonic-gate
42727c478bd9Sstevel@tonic-gate
427312f080e7Smrj /*ARGSUSED*/
42747c478bd9Sstevel@tonic-gate static int
rootnex_coredma_sync(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle,off_t off,size_t len,uint_t cache_flags)427520906b23SVikram Hegde rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
427612f080e7Smrj off_t off, size_t len, uint_t cache_flags)
42777c478bd9Sstevel@tonic-gate {
427812f080e7Smrj rootnex_sglinfo_t *sinfo;
427912f080e7Smrj rootnex_pgmap_t *cbpage;
428012f080e7Smrj rootnex_window_t *win;
428112f080e7Smrj ddi_dma_impl_t *hp;
428212f080e7Smrj rootnex_dma_t *dma;
428312f080e7Smrj caddr_t fromaddr;
428412f080e7Smrj caddr_t toaddr;
428512f080e7Smrj uint_t psize;
428612f080e7Smrj off_t offset;
428712f080e7Smrj uint_t pidx;
428812f080e7Smrj size_t size;
428912f080e7Smrj off_t poff;
429012f080e7Smrj int e;
42917c478bd9Sstevel@tonic-gate
42927c478bd9Sstevel@tonic-gate
429312f080e7Smrj hp = (ddi_dma_impl_t *)handle;
429412f080e7Smrj dma = (rootnex_dma_t *)hp->dmai_private;
429512f080e7Smrj sinfo = &dma->dp_sglinfo;
429612f080e7Smrj
429712f080e7Smrj /*
429812f080e7Smrj * if we don't have any windows, we don't need to sync. A copybuf
429912f080e7Smrj * will cause us to have at least one window.
430012f080e7Smrj */
430112f080e7Smrj if (dma->dp_window == NULL) {
430212f080e7Smrj return (DDI_SUCCESS);
430312f080e7Smrj }
430412f080e7Smrj
430512f080e7Smrj /* This window may not need to be sync'd */
430612f080e7Smrj win = &dma->dp_window[dma->dp_current_win];
430712f080e7Smrj if (!win->wd_dosync) {
430812f080e7Smrj return (DDI_SUCCESS);
430912f080e7Smrj }
431012f080e7Smrj
431112f080e7Smrj /* handle off and len special cases */
431212f080e7Smrj if ((off == 0) || (rootnex_sync_ignore_params)) {
431312f080e7Smrj offset = win->wd_offset;
431412f080e7Smrj } else {
431512f080e7Smrj offset = off;
431612f080e7Smrj }
431712f080e7Smrj if ((len == 0) || (rootnex_sync_ignore_params)) {
431812f080e7Smrj size = win->wd_size;
431912f080e7Smrj } else {
432012f080e7Smrj size = len;
432112f080e7Smrj }
432212f080e7Smrj
432312f080e7Smrj /* check the sync args to make sure they make a little sense */
432412f080e7Smrj if (rootnex_sync_check_parms) {
432512f080e7Smrj e = rootnex_valid_sync_parms(hp, win, offset, size,
432612f080e7Smrj cache_flags);
432712f080e7Smrj if (e != DDI_SUCCESS) {
4328ef4ab52fSFrank Van Der Linden ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]);
432912f080e7Smrj return (DDI_FAILURE);
433012f080e7Smrj }
433112f080e7Smrj }
433212f080e7Smrj
433312f080e7Smrj /*
433412f080e7Smrj * special case the first page to handle the offset into the page. The
433512f080e7Smrj * offset to the current page for our buffer is the offset into the
433612f080e7Smrj * first page of the buffer plus our current offset into the buffer
433712f080e7Smrj * itself, masked of course.
433812f080e7Smrj */
433912f080e7Smrj poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET;
434012f080e7Smrj psize = MIN((MMU_PAGESIZE - poff), size);
434112f080e7Smrj
434212f080e7Smrj /* go through all the pages that we want to sync */
434312f080e7Smrj while (size > 0) {
434412f080e7Smrj /*
434512f080e7Smrj * Calculate the page index relative to the start of the buffer.
434612f080e7Smrj * The index to the current page for our buffer is the offset
434712f080e7Smrj * into the first page of the buffer plus our current offset
434812f080e7Smrj * into the buffer itself, shifted of course...
434912f080e7Smrj */
435012f080e7Smrj pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT;
435112f080e7Smrj ASSERT(pidx < sinfo->si_max_pages);
43527c478bd9Sstevel@tonic-gate
435312f080e7Smrj /*
435412f080e7Smrj * if this page uses the copy buffer, we need to sync it,
435512f080e7Smrj * otherwise, go on to the next page.
435612f080e7Smrj */
435712f080e7Smrj cbpage = &dma->dp_pgmap[pidx];
435812f080e7Smrj ASSERT((cbpage->pm_uses_copybuf == B_TRUE) ||
435912f080e7Smrj (cbpage->pm_uses_copybuf == B_FALSE));
436012f080e7Smrj if (cbpage->pm_uses_copybuf) {
436112f080e7Smrj /* cbaddr and kaddr should be page aligned */
436212f080e7Smrj ASSERT(((uintptr_t)cbpage->pm_cbaddr &
436312f080e7Smrj MMU_PAGEOFFSET) == 0);
436412f080e7Smrj ASSERT(((uintptr_t)cbpage->pm_kaddr &
436512f080e7Smrj MMU_PAGEOFFSET) == 0);
43667c478bd9Sstevel@tonic-gate
436712f080e7Smrj /*
436812f080e7Smrj * if we're copying for the device, we are going to
436912f080e7Smrj * copy from the drivers buffer and to the rootnex
437012f080e7Smrj * allocated copy buffer.
437112f080e7Smrj */
437212f080e7Smrj if (cache_flags == DDI_DMA_SYNC_FORDEV) {
437312f080e7Smrj fromaddr = cbpage->pm_kaddr + poff;
437412f080e7Smrj toaddr = cbpage->pm_cbaddr + poff;
4375ef4ab52fSFrank Van Der Linden ROOTNEX_DPROBE2(rootnex__sync__dev,
437612f080e7Smrj dev_info_t *, dma->dp_dip, size_t, psize);
43777c478bd9Sstevel@tonic-gate
437812f080e7Smrj /*
437912f080e7Smrj * if we're copying for the cpu/kernel, we are going to
438012f080e7Smrj * copy from the rootnex allocated copy buffer to the
438112f080e7Smrj * drivers buffer.
438212f080e7Smrj */
438312f080e7Smrj } else {
438412f080e7Smrj fromaddr = cbpage->pm_cbaddr + poff;
438512f080e7Smrj toaddr = cbpage->pm_kaddr + poff;
4386ef4ab52fSFrank Van Der Linden ROOTNEX_DPROBE2(rootnex__sync__cpu,
438712f080e7Smrj dev_info_t *, dma->dp_dip, size_t, psize);
438812f080e7Smrj }
43897c478bd9Sstevel@tonic-gate
439012f080e7Smrj bcopy(fromaddr, toaddr, psize);
43917c478bd9Sstevel@tonic-gate }
43927c478bd9Sstevel@tonic-gate
439312f080e7Smrj /*
439412f080e7Smrj * decrement size until we're done, update our offset into the
439512f080e7Smrj * buffer, and get the next page size.
439612f080e7Smrj */
439712f080e7Smrj size -= psize;
439812f080e7Smrj offset += psize;
439912f080e7Smrj psize = MIN(MMU_PAGESIZE, size);
440012f080e7Smrj
440112f080e7Smrj /* page offset is zero for the rest of this loop */
440212f080e7Smrj poff = 0;
440312f080e7Smrj }
440412f080e7Smrj
440512f080e7Smrj return (DDI_SUCCESS);
44067c478bd9Sstevel@tonic-gate }
44077c478bd9Sstevel@tonic-gate
440820906b23SVikram Hegde /*
440920906b23SVikram Hegde * rootnex_dma_sync()
441020906b23SVikram Hegde * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags.
441120906b23SVikram Hegde * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC
441220906b23SVikram Hegde * is set, ddi_dma_sync() returns immediately passing back success.
441320906b23SVikram Hegde */
441420906b23SVikram Hegde /*ARGSUSED*/
441520906b23SVikram Hegde static int
rootnex_dma_sync(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle,off_t off,size_t len,uint_t cache_flags)441620906b23SVikram Hegde rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
441720906b23SVikram Hegde off_t off, size_t len, uint_t cache_flags)
441820906b23SVikram Hegde {
441986ef0a63SRichard Lowe #if !defined(__xpv)
4420b51bbbf5SVikram Hegde if (IOMMU_USED(rdip)) {
442120906b23SVikram Hegde return (iommulib_nexdma_sync(dip, rdip, handle, off, len,
442220906b23SVikram Hegde cache_flags));
442320906b23SVikram Hegde }
442420906b23SVikram Hegde #endif
442520906b23SVikram Hegde return (rootnex_coredma_sync(dip, rdip, handle, off, len,
442620906b23SVikram Hegde cache_flags));
442720906b23SVikram Hegde }
442812f080e7Smrj
442912f080e7Smrj /*
443012f080e7Smrj * rootnex_valid_sync_parms()
443112f080e7Smrj * checks the parameters passed to sync to verify they are correct.
443212f080e7Smrj */
44337c478bd9Sstevel@tonic-gate static int
rootnex_valid_sync_parms(ddi_dma_impl_t * hp,rootnex_window_t * win,off_t offset,size_t size,uint_t cache_flags)443412f080e7Smrj rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
443512f080e7Smrj off_t offset, size_t size, uint_t cache_flags)
44367c478bd9Sstevel@tonic-gate {
443712f080e7Smrj off_t woffset;
44387c478bd9Sstevel@tonic-gate
44397c478bd9Sstevel@tonic-gate
444012f080e7Smrj /*
444112f080e7Smrj * the first part of the test to make sure the offset passed in is
444212f080e7Smrj * within the window.
444312f080e7Smrj */
444412f080e7Smrj if (offset < win->wd_offset) {
444512f080e7Smrj return (DDI_FAILURE);
444612f080e7Smrj }
44477c478bd9Sstevel@tonic-gate
444812f080e7Smrj /*
444912f080e7Smrj * second and last part of the test to make sure the offset and length
445012f080e7Smrj * passed in is within the window.
445112f080e7Smrj */
445212f080e7Smrj woffset = offset - win->wd_offset;
445312f080e7Smrj if ((woffset + size) > win->wd_size) {
445412f080e7Smrj return (DDI_FAILURE);
445512f080e7Smrj }
44567c478bd9Sstevel@tonic-gate
445712f080e7Smrj /*
445812f080e7Smrj * if we are sync'ing for the device, the DDI_DMA_WRITE flag should
445912f080e7Smrj * be set too.
446012f080e7Smrj */
446112f080e7Smrj if ((cache_flags == DDI_DMA_SYNC_FORDEV) &&
446212f080e7Smrj (hp->dmai_rflags & DDI_DMA_WRITE)) {
44637c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
446412f080e7Smrj }
44657c478bd9Sstevel@tonic-gate
446612f080e7Smrj /*
446712f080e7Smrj * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL
446812f080e7Smrj * should be set. Also DDI_DMA_READ should be set in the flags.
446912f080e7Smrj */
447012f080e7Smrj if (((cache_flags == DDI_DMA_SYNC_FORCPU) ||
447112f080e7Smrj (cache_flags == DDI_DMA_SYNC_FORKERNEL)) &&
447212f080e7Smrj (hp->dmai_rflags & DDI_DMA_READ)) {
44737c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
447412f080e7Smrj }
44757c478bd9Sstevel@tonic-gate
447612f080e7Smrj return (DDI_FAILURE);
447712f080e7Smrj }
44787c478bd9Sstevel@tonic-gate
44797c478bd9Sstevel@tonic-gate
448012f080e7Smrj /*ARGSUSED*/
448112f080e7Smrj static int
rootnex_coredma_win(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle,uint_t win,off_t * offp,size_t * lenp,ddi_dma_cookie_t * cookiep,uint_t * ccountp)448220906b23SVikram Hegde rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
448312f080e7Smrj uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
448412f080e7Smrj uint_t *ccountp)
448512f080e7Smrj {
448612f080e7Smrj rootnex_window_t *window;
448712f080e7Smrj rootnex_trim_t *trim;
448812f080e7Smrj ddi_dma_impl_t *hp;
448912f080e7Smrj rootnex_dma_t *dma;
449050200e77SFrank Van Der Linden ddi_dma_obj_t *dmao;
44917c478bd9Sstevel@tonic-gate
44927c478bd9Sstevel@tonic-gate
449312f080e7Smrj hp = (ddi_dma_impl_t *)handle;
449412f080e7Smrj dma = (rootnex_dma_t *)hp->dmai_private;
44957c478bd9Sstevel@tonic-gate
449612f080e7Smrj /* If we try and get a window which doesn't exist, return failure */
449712f080e7Smrj if (win >= hp->dmai_nwin) {
4498ef4ab52fSFrank Van Der Linden ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
44997c478bd9Sstevel@tonic-gate return (DDI_FAILURE);
450012f080e7Smrj }
45017c478bd9Sstevel@tonic-gate
4502bd155b87SGarrett D'Amore dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma;
450350200e77SFrank Van Der Linden
450412f080e7Smrj /*
450512f080e7Smrj * if we don't have any windows, and they're asking for the first
450612f080e7Smrj * window, setup the cookie pointer to the first cookie in the bind.
450712f080e7Smrj * setup our return values, then increment the cookie since we return
450812f080e7Smrj * the first cookie on the stack.
450912f080e7Smrj */
451012f080e7Smrj if (dma->dp_window == NULL) {
451112f080e7Smrj if (win != 0) {
4512ef4ab52fSFrank Van Der Linden ROOTNEX_DPROF_INC(
4513ef4ab52fSFrank Van Der Linden &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
451412f080e7Smrj return (DDI_FAILURE);
451512f080e7Smrj }
451612f080e7Smrj hp->dmai_cookie = dma->dp_cookies;
451712f080e7Smrj *offp = 0;
451850200e77SFrank Van Der Linden *lenp = dmao->dmao_size;
451912f080e7Smrj *ccountp = dma->dp_sglinfo.si_sgl_size;
452012f080e7Smrj *cookiep = hp->dmai_cookie[0];
452112f080e7Smrj hp->dmai_cookie++;
45229a63ec27SRobert Mustacchi hp->dmai_ncookies = *ccountp;
45239a63ec27SRobert Mustacchi hp->dmai_curcookie = 1;
45247c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
452512f080e7Smrj }
45267c478bd9Sstevel@tonic-gate
452712f080e7Smrj /* sync the old window before moving on to the new one */
452812f080e7Smrj window = &dma->dp_window[dma->dp_current_win];
452912f080e7Smrj if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) {
453094f1124eSVikram Hegde (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
453112f080e7Smrj DDI_DMA_SYNC_FORCPU);
453212f080e7Smrj }
45337c478bd9Sstevel@tonic-gate
45347c478bd9Sstevel@tonic-gate
453512f080e7Smrj /*
453612f080e7Smrj * Move to the new window.
453712f080e7Smrj * NOTE: current_win must be set for sync to work right
453812f080e7Smrj */
453912f080e7Smrj dma->dp_current_win = win;
454012f080e7Smrj window = &dma->dp_window[win];
454112f080e7Smrj
454212f080e7Smrj /* if needed, adjust the first and/or last cookies for trim */
454312f080e7Smrj trim = &window->wd_trim;
454412f080e7Smrj if (trim->tr_trim_first) {
4545843e1988Sjohnlev window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr;
454612f080e7Smrj window->wd_first_cookie->dmac_size = trim->tr_first_size;
454712f080e7Smrj if (trim->tr_first_copybuf_win) {
454812f080e7Smrj dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr =
454912f080e7Smrj trim->tr_first_cbaddr;
455012f080e7Smrj }
455112f080e7Smrj }
455212f080e7Smrj if (trim->tr_trim_last) {
4553843e1988Sjohnlev trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr;
455412f080e7Smrj trim->tr_last_cookie->dmac_size = trim->tr_last_size;
455512f080e7Smrj if (trim->tr_last_copybuf_win) {
455612f080e7Smrj dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr =
455712f080e7Smrj trim->tr_last_cbaddr;
455812f080e7Smrj }
45597c478bd9Sstevel@tonic-gate }
456012f080e7Smrj
45617c478bd9Sstevel@tonic-gate /*
456212f080e7Smrj * setup the cookie pointer to the first cookie in the window. setup
456312f080e7Smrj * our return values, then increment the cookie since we return the
456412f080e7Smrj * first cookie on the stack.
45657c478bd9Sstevel@tonic-gate */
456612f080e7Smrj hp->dmai_cookie = window->wd_first_cookie;
456712f080e7Smrj *offp = window->wd_offset;
456812f080e7Smrj *lenp = window->wd_size;
456912f080e7Smrj *ccountp = window->wd_cookie_cnt;
457012f080e7Smrj *cookiep = hp->dmai_cookie[0];
45719a63ec27SRobert Mustacchi hp->dmai_ncookies = *ccountp;
45729a63ec27SRobert Mustacchi hp->dmai_curcookie = 1;
457312f080e7Smrj hp->dmai_cookie++;
45747c478bd9Sstevel@tonic-gate
457512f080e7Smrj
457612f080e7Smrj /* if the new window uses the copy buffer, sync it for the device */
457712f080e7Smrj if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) {
457894f1124eSVikram Hegde (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
457912f080e7Smrj DDI_DMA_SYNC_FORDEV);
45807c478bd9Sstevel@tonic-gate }
458112f080e7Smrj
45827c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
45837c478bd9Sstevel@tonic-gate }
45847c478bd9Sstevel@tonic-gate
458520906b23SVikram Hegde /*
458620906b23SVikram Hegde * rootnex_dma_win()
458720906b23SVikram Hegde * called from ddi_dma_getwin()
458820906b23SVikram Hegde */
458920906b23SVikram Hegde /*ARGSUSED*/
459020906b23SVikram Hegde static int
rootnex_dma_win(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle,uint_t win,off_t * offp,size_t * lenp,ddi_dma_cookie_t * cookiep,uint_t * ccountp)459120906b23SVikram Hegde rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
459220906b23SVikram Hegde uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
459320906b23SVikram Hegde uint_t *ccountp)
459420906b23SVikram Hegde {
459586ef0a63SRichard Lowe #if !defined(__xpv)
4596b51bbbf5SVikram Hegde if (IOMMU_USED(rdip)) {
459720906b23SVikram Hegde return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp,
459820906b23SVikram Hegde cookiep, ccountp));
459920906b23SVikram Hegde }
460020906b23SVikram Hegde #endif
460112f080e7Smrj
460220906b23SVikram Hegde return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp,
460320906b23SVikram Hegde cookiep, ccountp));
460420906b23SVikram Hegde }
460512f080e7Smrj
460686ef0a63SRichard Lowe #if !defined(__xpv)
460750200e77SFrank Van Der Linden /*ARGSUSED*/
460850200e77SFrank Van Der Linden static int
rootnex_coredma_hdl_setprivate(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle,void * v)460950200e77SFrank Van Der Linden rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
461050200e77SFrank Van Der Linden ddi_dma_handle_t handle, void *v)
461150200e77SFrank Van Der Linden {
461250200e77SFrank Van Der Linden ddi_dma_impl_t *hp;
461350200e77SFrank Van Der Linden rootnex_dma_t *dma;
461450200e77SFrank Van Der Linden
461550200e77SFrank Van Der Linden hp = (ddi_dma_impl_t *)handle;
461650200e77SFrank Van Der Linden dma = (rootnex_dma_t *)hp->dmai_private;
461750200e77SFrank Van Der Linden dma->dp_iommu_private = v;
461850200e77SFrank Van Der Linden
461950200e77SFrank Van Der Linden return (DDI_SUCCESS);
462050200e77SFrank Van Der Linden }
462150200e77SFrank Van Der Linden
462250200e77SFrank Van Der Linden /*ARGSUSED*/
462350200e77SFrank Van Der Linden static void *
rootnex_coredma_hdl_getprivate(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle)462450200e77SFrank Van Der Linden rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
462550200e77SFrank Van Der Linden ddi_dma_handle_t handle)
462650200e77SFrank Van Der Linden {
462750200e77SFrank Van Der Linden ddi_dma_impl_t *hp;
462850200e77SFrank Van Der Linden rootnex_dma_t *dma;
462950200e77SFrank Van Der Linden
463050200e77SFrank Van Der Linden hp = (ddi_dma_impl_t *)handle;
463150200e77SFrank Van Der Linden dma = (rootnex_dma_t *)hp->dmai_private;
463250200e77SFrank Van Der Linden
463350200e77SFrank Van Der Linden return (dma->dp_iommu_private);
463450200e77SFrank Van Der Linden }
4635ee7536e8SFrank Van Der Linden #endif
463650200e77SFrank Van Der Linden
46377c478bd9Sstevel@tonic-gate /*
463812f080e7Smrj * ************************
463912f080e7Smrj * obsoleted dma routines
464012f080e7Smrj * ************************
46417c478bd9Sstevel@tonic-gate */
464212f080e7Smrj
46437c478bd9Sstevel@tonic-gate /*
464412f080e7Smrj * rootnex_dma_mctl()
464512f080e7Smrj *
4646cd21e7c5SGarrett D'Amore * We don't support this legacy interface any more on x86.
46477c478bd9Sstevel@tonic-gate */
46487c478bd9Sstevel@tonic-gate /* ARGSUSED */
46497c478bd9Sstevel@tonic-gate static int
rootnex_dma_mctl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle,enum ddi_dma_ctlops request,off_t * offp,size_t * lenp,caddr_t * objpp,uint_t cache_flags)4650b51bbbf5SVikram Hegde rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
465112f080e7Smrj enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp,
465212f080e7Smrj uint_t cache_flags)
46537c478bd9Sstevel@tonic-gate {
465412f080e7Smrj /*
4655cd21e7c5SGarrett D'Amore * The only thing dma_mctl is usef for anymore is legacy SPARC
4656cd21e7c5SGarrett D'Amore * dvma and sbus-specific routines.
465712f080e7Smrj */
465812f080e7Smrj return (DDI_FAILURE);
46597c478bd9Sstevel@tonic-gate }
46607aec1d6eScindi
466100d0963fSdilpreet /*
466200d0963fSdilpreet * *********
466300d0963fSdilpreet * FMA Code
466400d0963fSdilpreet * *********
466500d0963fSdilpreet */
466600d0963fSdilpreet
466700d0963fSdilpreet /*
466800d0963fSdilpreet * rootnex_fm_init()
466900d0963fSdilpreet * FMA init busop
467000d0963fSdilpreet */
467100d0963fSdilpreet /* ARGSUSED */
46727aec1d6eScindi static int
rootnex_fm_init(dev_info_t * dip,dev_info_t * tdip,int tcap,ddi_iblock_cookie_t * ibc)467300d0963fSdilpreet rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
467400d0963fSdilpreet ddi_iblock_cookie_t *ibc)
46757aec1d6eScindi {
467600d0963fSdilpreet *ibc = rootnex_state->r_err_ibc;
467700d0963fSdilpreet
467800d0963fSdilpreet return (ddi_system_fmcap);
467900d0963fSdilpreet }
468000d0963fSdilpreet
468100d0963fSdilpreet /*
468200d0963fSdilpreet * rootnex_dma_check()
468300d0963fSdilpreet * Function called after a dma fault occurred to find out whether the
468400d0963fSdilpreet * fault address is associated with a driver that is able to handle faults
468500d0963fSdilpreet * and recover from faults.
468600d0963fSdilpreet */
468700d0963fSdilpreet /* ARGSUSED */
468800d0963fSdilpreet static int
rootnex_dma_check(dev_info_t * dip,const void * handle,const void * addr,const void * not_used)468900d0963fSdilpreet rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr,
469000d0963fSdilpreet const void *not_used)
469100d0963fSdilpreet {
469200d0963fSdilpreet rootnex_window_t *window;
469300d0963fSdilpreet uint64_t start_addr;
469400d0963fSdilpreet uint64_t fault_addr;
469500d0963fSdilpreet ddi_dma_impl_t *hp;
469600d0963fSdilpreet rootnex_dma_t *dma;
469700d0963fSdilpreet uint64_t end_addr;
469800d0963fSdilpreet size_t csize;
469900d0963fSdilpreet int i;
470000d0963fSdilpreet int j;
470100d0963fSdilpreet
470200d0963fSdilpreet
470300d0963fSdilpreet /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */
470400d0963fSdilpreet hp = (ddi_dma_impl_t *)handle;
470500d0963fSdilpreet ASSERT(hp);
470600d0963fSdilpreet
470700d0963fSdilpreet dma = (rootnex_dma_t *)hp->dmai_private;
470800d0963fSdilpreet
470900d0963fSdilpreet /* Get the address that we need to search for */
471000d0963fSdilpreet fault_addr = *(uint64_t *)addr;
471100d0963fSdilpreet
471200d0963fSdilpreet /*
471300d0963fSdilpreet * if we don't have any windows, we can just walk through all the
471400d0963fSdilpreet * cookies.
471500d0963fSdilpreet */
471600d0963fSdilpreet if (dma->dp_window == NULL) {
471700d0963fSdilpreet /* for each cookie */
471800d0963fSdilpreet for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) {
471900d0963fSdilpreet /*
472000d0963fSdilpreet * if the faulted address is within the physical address
472100d0963fSdilpreet * range of the cookie, return DDI_FM_NONFATAL.
472200d0963fSdilpreet */
472300d0963fSdilpreet if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) &&
472400d0963fSdilpreet (fault_addr <= (dma->dp_cookies[i].dmac_laddress +
472500d0963fSdilpreet dma->dp_cookies[i].dmac_size))) {
472600d0963fSdilpreet return (DDI_FM_NONFATAL);
472700d0963fSdilpreet }
472800d0963fSdilpreet }
472900d0963fSdilpreet
473000d0963fSdilpreet /* fault_addr not within this DMA handle */
473100d0963fSdilpreet return (DDI_FM_UNKNOWN);
473200d0963fSdilpreet }
473300d0963fSdilpreet
473400d0963fSdilpreet /* we have mutiple windows, walk through each window */
473500d0963fSdilpreet for (i = 0; i < hp->dmai_nwin; i++) {
473600d0963fSdilpreet window = &dma->dp_window[i];
473700d0963fSdilpreet
473800d0963fSdilpreet /* Go through all the cookies in the window */
473900d0963fSdilpreet for (j = 0; j < window->wd_cookie_cnt; j++) {
474000d0963fSdilpreet
474100d0963fSdilpreet start_addr = window->wd_first_cookie[j].dmac_laddress;
474200d0963fSdilpreet csize = window->wd_first_cookie[j].dmac_size;
474300d0963fSdilpreet
474400d0963fSdilpreet /*
474500d0963fSdilpreet * if we are trimming the first cookie in the window,
474600d0963fSdilpreet * and this is the first cookie, adjust the start
474700d0963fSdilpreet * address and size of the cookie to account for the
474800d0963fSdilpreet * trim.
474900d0963fSdilpreet */
475000d0963fSdilpreet if (window->wd_trim.tr_trim_first && (j == 0)) {
475100d0963fSdilpreet start_addr = window->wd_trim.tr_first_paddr;
475200d0963fSdilpreet csize = window->wd_trim.tr_first_size;
475300d0963fSdilpreet }
475400d0963fSdilpreet
475500d0963fSdilpreet /*
475600d0963fSdilpreet * if we are trimming the last cookie in the window,
475700d0963fSdilpreet * and this is the last cookie, adjust the start
475800d0963fSdilpreet * address and size of the cookie to account for the
475900d0963fSdilpreet * trim.
476000d0963fSdilpreet */
476100d0963fSdilpreet if (window->wd_trim.tr_trim_last &&
476200d0963fSdilpreet (j == (window->wd_cookie_cnt - 1))) {
476300d0963fSdilpreet start_addr = window->wd_trim.tr_last_paddr;
476400d0963fSdilpreet csize = window->wd_trim.tr_last_size;
476500d0963fSdilpreet }
476600d0963fSdilpreet
476700d0963fSdilpreet end_addr = start_addr + csize;
476800d0963fSdilpreet
476900d0963fSdilpreet /*
47703a634bfcSVikram Hegde * if the faulted address is within the physical
47713a634bfcSVikram Hegde * address of the cookie, return DDI_FM_NONFATAL.
477200d0963fSdilpreet */
477300d0963fSdilpreet if ((fault_addr >= start_addr) &&
477400d0963fSdilpreet (fault_addr <= end_addr)) {
477500d0963fSdilpreet return (DDI_FM_NONFATAL);
477600d0963fSdilpreet }
477700d0963fSdilpreet }
477800d0963fSdilpreet }
477900d0963fSdilpreet
478000d0963fSdilpreet /* fault_addr not within this DMA handle */
478100d0963fSdilpreet return (DDI_FM_UNKNOWN);
47827aec1d6eScindi }
47833a634bfcSVikram Hegde
47843a634bfcSVikram Hegde /*ARGSUSED*/
47853a634bfcSVikram Hegde static int
rootnex_quiesce(dev_info_t * dip)47863a634bfcSVikram Hegde rootnex_quiesce(dev_info_t *dip)
47873a634bfcSVikram Hegde {
478886ef0a63SRichard Lowe #if !defined(__xpv)
47893a634bfcSVikram Hegde return (immu_quiesce());
47903a634bfcSVikram Hegde #else
47913a634bfcSVikram Hegde return (DDI_SUCCESS);
47923a634bfcSVikram Hegde #endif
47933a634bfcSVikram Hegde }
47943a634bfcSVikram Hegde
47953a634bfcSVikram Hegde #if defined(__xpv)
47963a634bfcSVikram Hegde void
immu_init(void)47973a634bfcSVikram Hegde immu_init(void)
47983a634bfcSVikram Hegde {
47993a634bfcSVikram Hegde ;
48003a634bfcSVikram Hegde }
48013a634bfcSVikram Hegde
48023a634bfcSVikram Hegde void
immu_startup(void)48033a634bfcSVikram Hegde immu_startup(void)
48043a634bfcSVikram Hegde {
48053a634bfcSVikram Hegde ;
48063a634bfcSVikram Hegde }
48073a634bfcSVikram Hegde /*ARGSUSED*/
48083a634bfcSVikram Hegde void
immu_physmem_update(uint64_t addr,uint64_t size)48093a634bfcSVikram Hegde immu_physmem_update(uint64_t addr, uint64_t size)
48103a634bfcSVikram Hegde {
48113a634bfcSVikram Hegde ;
48123a634bfcSVikram Hegde }
48133a634bfcSVikram Hegde #endif
4814