xref: /illumos-gate/usr/src/uts/sun4u/sys/pci/pci_dma.h (revision d0662dbf)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #ifndef	_SYS_PCI_DMA_H
27 #define	_SYS_PCI_DMA_H
28 
29 #pragma ident	"%Z%%M%	%I%	%E% SMI"
30 
31 #ifdef	__cplusplus
32 extern "C" {
33 #endif
34 
35 typedef	pfn_t iopfn_t;
36 #define	MAKE_DMA_COOKIE(cp, address, size)	\
37 	{					\
38 		(cp)->dmac_notused = 0;		\
39 		(cp)->dmac_type = 0;		\
40 		(cp)->dmac_laddress = (address);	\
41 		(cp)->dmac_size = (size);	\
42 	}
43 
44 #define	HAS_REDZONE(mp)	(((mp)->dmai_rflags & DDI_DMA_REDZONE) ? 1 : 0)
45 
46 #define	PCI_DMA_HAT_NUM_CB_COOKIES	5
47 
48 typedef struct pci_dma_hdl {
49 	ddi_dma_impl_t	pdh_ddi_hdl;
50 	ddi_dma_attr_t	pdh_attr_dev;
51 	uint64_t	pdh_sync_buf_pa;
52 	void		*pdh_cbcookie[PCI_DMA_HAT_NUM_CB_COOKIES];
53 } pci_dma_hdl_t;
54 
55 struct pci_dma_impl { /* forthdebug only, keep in sync with ddi_dma_impl_t */
56 	ulong_t		dmai_mapping;
57 	uint_t		dmai_size;
58 	off_t		dmai_offset;
59 	uint_t		dmai_minxfer;
60 	uint_t		dmai_burstsizes;
61 	uint_t		dmai_ndvmapages;
62 	uint_t		dmai_roffset;
63 	uint_t		dmai_rflags;
64 	uint_t		dmai_flags;
65 	uint_t		dmai_nwin;
66 	uint_t		dmai_winsize;
67 	caddr_t		dmai_tte_fdvma;
68 	void		*dmai_pfnlst;
69 	uint_t		*dmai_pfn0;
70 	void		*dmai_winlst;
71 	dev_info_t	*dmai_rdip;
72 	ddi_dma_obj_t	dmai_object;
73 	ddi_dma_attr_t	dmai_attr_aug;
74 	ddi_dma_cookie_t *dmai_cookie;
75 
76 	int		(*dmai_fault_check)(struct ddi_dma_impl *handle);
77 	void		(*dmai_fault_notify)(struct ddi_dma_impl *handle);
78 	int		dmai_fault;
79 
80 	ddi_dma_attr_t	pdh_attr_dev;
81 	uint64_t	pdh_sync_buf_pa;
82 	void		*pdh_cbcookie[PCI_DMA_HAT_NUM_CB_COOKIES];
83 };
84 
85 /*
86  * flags for overloading dmai_inuse field of the dma request
87  * structure:
88  */
89 #define	dmai_flags		dmai_inuse
90 #define	dmai_tte		dmai_nexus_private
91 #define	dmai_fdvma		dmai_nexus_private
92 #define	dmai_pfnlst		dmai_iopte
93 #define	dmai_winlst		dmai_minfo
94 #define	dmai_pfn0		dmai_sbi
95 #define	dmai_roffset		dmai_pool
96 
97 #define	MP_PFN0(mp)		((iopfn_t)(mp)->dmai_pfn0)
98 #define	MP_HAT_CB_COOKIE(mp, i)	((i < PCI_DMA_HAT_NUM_CB_COOKIES)? \
99 	(((pci_dma_hdl_t *)(mp))->pdh_cbcookie[i]) : NULL)
100 #define	MP_HAT_CB_COOKIE_PTR(mp, i) \
101 	((i < PCI_DMA_HAT_NUM_CB_COOKIES)? \
102 	&(((pci_dma_hdl_t *)(mp))->pdh_cbcookie[i]) : NULL)
103 #define	WINLST(mp)		((pci_dma_win_t *)(mp)->dmai_winlst)
104 #define	DEV_ATTR(mp)		(&((pci_dma_hdl_t *)(mp))->pdh_attr_dev)
105 #define	SYNC_BUF_PA(mp)		(((pci_dma_hdl_t *)(mp))->pdh_sync_buf_pa)
106 #define	SET_DMAATTR(p, lo, hi, nocross, cntmax)	\
107 	(p)->dma_attr_addr_lo	= (lo); \
108 	(p)->dma_attr_addr_hi	= (hi); \
109 	(p)->dma_attr_seg	= (nocross); \
110 	(p)->dma_attr_count_max	= (cntmax);
111 
112 #define	SET_DMAALIGN(p, align) \
113 	(p)->dma_attr_align = (align);
114 
115 #define	DMAI_FLAGS_INUSE	0x1
116 #define	DMAI_FLAGS_BYPASSREQ	0x2
117 #define	DMAI_FLAGS_PEER_ONLY	0x4
118 #define	DMAI_FLAGS_NOCTX	0x8
119 #define	DMAI_FLAGS_DVMA		0x10
120 #define	DMAI_FLAGS_BYPASS	0x20
121 #define	DMAI_FLAGS_PEER_TO_PEER	0x40
122 #define	DMAI_FLAGS_DMA		(DMAI_FLAGS_BYPASS | DMAI_FLAGS_PEER_TO_PEER)
123 #define	DMAI_FLAGS_DMA_TYPE	(DMAI_FLAGS_DMA | DMAI_FLAGS_DVMA)
124 #define	DMAI_FLAGS_CONTEXT	0x100
125 #define	DMAI_FLAGS_FASTTRACK	0x200
126 #define	DMAI_FLAGS_VMEMCACHE	0x400
127 #define	DMAI_FLAGS_PGPFN	0x800
128 #define	DMAI_FLAGS_NOSYSLIMIT	0x1000
129 #define	DMAI_FLAGS_NOFASTLIMIT	0x2000
130 #define	DMAI_FLAGS_NOSYNC	0x4000
131 #define	DMAI_FLAGS_RELOC	0x8000
132 #define	DMAI_FLAGS_MAPPED	0x10000
133 #define	DMAI_FLAGS_PRESERVE	(DMAI_FLAGS_PEER_ONLY | DMAI_FLAGS_BYPASSREQ | \
134 	DMAI_FLAGS_NOSYSLIMIT | DMAI_FLAGS_NOFASTLIMIT | DMAI_FLAGS_NOCTX)
135 
136 #define	HAS_NOFASTLIMIT(mp)	((mp)->dmai_flags & DMAI_FLAGS_NOFASTLIMIT)
137 #define	HAS_NOSYSLIMIT(mp)	((mp)->dmai_flags & DMAI_FLAGS_NOSYSLIMIT)
138 #define	PCI_DMA_ISPEERONLY(mp)	((mp)->dmai_flags & DMAI_FLAGS_PEER_ONLY)
139 #define	PCI_DMA_ISPGPFN(mp)	((mp)->dmai_flags & DMAI_FLAGS_PGPFN)
140 #define	PCI_DMA_TYPE(mp)	((mp)->dmai_flags & DMAI_FLAGS_DMA_TYPE)
141 #define	PCI_DMA_ISDVMA(mp)	(PCI_DMA_TYPE(mp) == DMAI_FLAGS_DVMA)
142 #define	PCI_DMA_ISBYPASS(mp)	(PCI_DMA_TYPE(mp) == DMAI_FLAGS_BYPASS)
143 #define	PCI_DMA_ISPTP(mp)	(PCI_DMA_TYPE(mp) == DMAI_FLAGS_PEER_TO_PEER)
144 #define	PCI_DMA_CANFAST(mp)	(((mp)->dmai_ndvmapages + HAS_REDZONE(mp) \
145 		<= pci_dvma_page_cache_clustsz) && HAS_NOFASTLIMIT(mp))
146 #define	PCI_DMA_WINNPGS(mp)	IOMMU_BTOP((mp)->dmai_winsize)
147 #define	PCI_DMA_CANCACHE(mp)	(!HAS_REDZONE(mp) && \
148 		(PCI_DMA_WINNPGS(mp) == 1) && HAS_NOSYSLIMIT(mp))
149 #define	PCI_DMA_CANRELOC(mp)	((mp)->dmai_flags & DMAI_FLAGS_RELOC)
150 #define	PCI_DMA_ISMAPPED(mp)	((mp)->dmai_flags & DMAI_FLAGS_MAPPED)
151 
152 #define	PCI_SYNC_FLAG_SZSHIFT	6
153 #define	PCI_SYNC_FLAG_SIZE	(1 << PCI_SYNC_FLAG_SZSHIFT)
154 #define	PCI_SYNC_FLAG_FAILED	1
155 #define	PCI_SYNC_FLAG_LOCKED	2
156 
157 #define	PCI_DMA_SYNC_DDI_FLAGS	((1 << 16) - 1)	/* Look for only DDI flags  */
158 #define	PCI_DMA_SYNC_EXT	(1 << 30)	/* enable/disable extension */
159 #define	PCI_DMA_SYNC_UNBIND	(1 << 28)	/* internal: part of unbind */
160 #define	PCI_DMA_SYNC_BAR	(1 << 26)	/* wait for all posted sync  */
161 #define	PCI_DMA_SYNC_POST	(1 << 25)	/* post request and return   */
162 #define	PCI_DMA_SYNC_PRIVATE	(1 << 24)	/* alloc private sync buffer */
163 #define	PCI_DMA_SYNC_DURING	(1 << 22)	/* sync in-progress dma */
164 #define	PCI_DMA_SYNC_BEFORE	(1 << 21)	/* before read or write */
165 #define	PCI_DMA_SYNC_AFTER	(1 << 20)	/* after read or write  */
166 #define	PCI_DMA_SYNC_WRITE	(1 << 17)	/* data from device to mem */
167 #define	PCI_DMA_SYNC_READ	(1 << 16)	/* data from memory to dev */
168 
169 #define	PCI_FLOW_ID_TO_PA(flow_p, flow_id) \
170 	((flow_p)->flow_buf_pa + ((flow_id) << PCI_SYNC_FLAG_SZSHIFT))
171 
172 #define	DEV_NOFASTLIMIT(lo, hi, fastlo, fasthi, align_pg) \
173 	(((lo) <= (fastlo)) && ((hi) >= (fasthi)) && \
174 	((align_pg) <= pci_dvma_page_cache_clustsz))
175 
176 #define	DEV_NOSYSLIMIT(lo, hi, syslo, syshi, align_pg) \
177 	(((lo) <= (syslo)) && ((hi) >= (syshi)) && (align_pg == 1))
178 
179 #define	PCI_DMA_NOCTX(rdip) (!pci_use_contexts || (pci_ctx_no_active_flush && \
180 	ddi_prop_exists(DDI_DEV_T_ANY, rdip, \
181 		DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "active-dma-flush")))
182 #define	PCI_DMA_USECTX(mp)	(!(mp->dmai_flags & DMAI_FLAGS_NOCTX))
183 
184 #define	PCI_DMA_BYPASS_PREFIX(mp, pfn) \
185 	(PCI_DMA_ISBYPASS(mp) ? COMMON_IOMMU_BYPASS_BASE | \
186 	(pf_is_memory(pfn) ? 0 : COMMON_IOMMU_BYPASS_NONCACHE) : 0)
187 #define	PCI_DMA_BADPTP(pfn, attrp) \
188 	((IOMMU_PTOB(pfn) < attrp->dma_attr_addr_lo) || \
189 	(IOMMU_PTOB(pfn) > attrp->dma_attr_addr_hi))
190 #define	PCI_DMA_CURWIN(mp) \
191 	(((mp)->dmai_offset + (mp)->dmai_roffset) / (mp)->dmai_winsize)
192 
193 #ifdef PCI_DMA_PROF
194 
195 /* collect fast track failure statistics */
196 #define	PCI_DVMA_FASTTRAK_PROF(mp) { \
197 if ((mp->dmai_ndvmapages + HAS_REDZONE(mp)) > pci_dvma_page_cache_clustsz) \
198 	pci_dvmaft_npages++; \
199 else if (!HAS_NOFASTLIMIT(mp)) \
200 	pci_dvmaft_limit++; \
201 }
202 
203 #else /* !PCI_DMA_PROF */
204 
205 #define	PCI_DVMA_FASTTRAK_PROF(mp)
206 
207 #endif	/* PCI_DMA_PROF */
208 
209 typedef struct pci_dma_win {
210 	struct pci_dma_win *win_next;
211 	uint32_t win_ncookies;
212 	uint32_t win_curseg;
213 	uint64_t win_size;
214 	uint64_t win_offset;
215 	/* cookie table: sizeof (ddi_dma_cookie_t) * win_ncookies */
216 } pci_dma_win_t;
217 
218 /* dvma debug records */
219 struct dvma_rec {
220 	char *dvma_addr;
221 	uint_t len;
222 	ddi_dma_impl_t *mp;
223 	struct dvma_rec *next;
224 };
225 
226 typedef struct pbm pbm_t;
227 extern int pci_dma_sync(dev_info_t *dip, dev_info_t *rdip,
228 	ddi_dma_handle_t handle, off_t off, size_t len, uint32_t sync_flags);
229 
230 extern int pci_dma_win(dev_info_t *dip, dev_info_t *rdip,
231 	ddi_dma_handle_t handle, uint_t win, off_t *offp,
232 	size_t *lenp, ddi_dma_cookie_t *cookiep, uint_t *ccountp);
233 
234 extern ddi_dma_impl_t *pci_dma_allocmp(dev_info_t *dip, dev_info_t *rdip,
235 	int (*waitfp)(caddr_t), caddr_t arg);
236 extern void pci_dma_freemp(ddi_dma_impl_t *mp);
237 extern void pci_dma_freepfn(ddi_dma_impl_t *mp);
238 extern ddi_dma_impl_t *pci_dma_lmts2hdl(dev_info_t *dip, dev_info_t *rdip,
239 	iommu_t *iommu_p, ddi_dma_req_t *dmareq);
240 extern int pci_dma_attr2hdl(pci_t *pci_p, ddi_dma_impl_t *mp);
241 extern uint32_t pci_dma_consist_check(uint32_t req_flags, pbm_t *pbm_p);
242 extern int pci_dma_type(pci_t *pci_p, ddi_dma_req_t *req, ddi_dma_impl_t *mp);
243 extern int pci_dma_pfn(pci_t *pci_p, ddi_dma_req_t *req, ddi_dma_impl_t *mp);
244 extern int pci_dvma_win(pci_t *pci_p, ddi_dma_req_t *r, ddi_dma_impl_t *mp);
245 extern void pci_dma_freewin(ddi_dma_impl_t *mp);
246 extern int pci_dvma_map_fast(iommu_t *iommu_p, ddi_dma_impl_t *mp);
247 extern int pci_dvma_map(ddi_dma_impl_t *mp, ddi_dma_req_t *dmareq,
248 	iommu_t *iommu_p);
249 extern void pci_dvma_unmap(iommu_t *iommu_p, ddi_dma_impl_t *mp);
250 extern void pci_dma_sync_unmap(dev_info_t *dip, dev_info_t *rdip,
251 	ddi_dma_impl_t *mp);
252 extern int pci_dma_physwin(pci_t *pci_p, ddi_dma_req_t *dmareq,
253 	ddi_dma_impl_t *mp);
254 extern int pci_dvma_ctl(dev_info_t *dip, dev_info_t *rdip,
255 	ddi_dma_impl_t *mp, enum ddi_dma_ctlops cmd, off_t *offp,
256 	size_t *lenp, caddr_t *objp, uint_t cache_flags);
257 extern int pci_dma_ctl(dev_info_t *dip, dev_info_t *rdip,
258 	ddi_dma_impl_t *mp, enum ddi_dma_ctlops cmd, off_t *offp,
259 	size_t *lenp, caddr_t *objp, uint_t cache_flags);
260 extern void pci_vmem_do_free(iommu_t *iommu_p, void *base_addr, size_t npages,
261 	int vmemcache);
262 
263 #define	PCI_GET_MP_NCOOKIES(mp)		((mp)->dmai_ncookies)
264 #define	PCI_SET_MP_NCOOKIES(mp, nc)	((mp)->dmai_ncookies = (nc))
265 #define	PCI_GET_MP_PFN1_ADDR(mp)	(((iopfn_t *)(mp)->dmai_pfnlst) + 1)
266 
267 #define	PCI_GET_MP_TTE(tte) \
268 	(((uint64_t)(uintptr_t)(tte) >> 5) << (32 + 5) | \
269 	    ((uint32_t)(uintptr_t)(tte)) & 0x12)
270 #define	PCI_SAVE_MP_TTE(mp, tte)	\
271 	(mp)->dmai_tte = (caddr_t)(HI32(tte) | ((tte) & 0x12))
272 
273 #define	PCI_GET_MP_PFN1(mp, page_no) (((iopfn_t *)(mp)->dmai_pfnlst)[page_no])
274 #define	PCI_GET_MP_PFN(mp, page_no)	((mp)->dmai_ndvmapages == 1 ? \
275 	(iopfn_t)(mp)->dmai_pfnlst : PCI_GET_MP_PFN1(mp, page_no))
276 
277 #define	PCI_SET_MP_PFN(mp, page_no, pfn) { \
278 	if ((mp)->dmai_ndvmapages == 1) { \
279 		ASSERT(!((page_no) || (mp)->dmai_pfnlst)); \
280 		(mp)->dmai_pfnlst = (void *)(pfn); \
281 	} else \
282 		((iopfn_t *)(mp)->dmai_pfnlst)[page_no] = (iopfn_t)(pfn); \
283 }
284 #define	PCI_SET_MP_PFN1(mp, page_no, pfn) { \
285 	((iopfn_t *)(mp)->dmai_pfnlst)[page_no] = (pfn); \
286 }
287 
288 #define	GET_TTE_TEMPLATE(mp) MAKE_TTE_TEMPLATE(PCI_GET_MP_PFN((mp), 0), (mp))
289 
290 extern int pci_dma_freehdl(dev_info_t *dip, dev_info_t *rdip,
291 	ddi_dma_handle_t handle);
292 
293 int pci_dma_handle_clean(dev_info_t *rdip, ddi_dma_handle_t handle);
294 
295 #if defined(DEBUG)
296 extern void dump_dma_handle(uint64_t flag, dev_info_t *dip, ddi_dma_impl_t *hp);
297 #else
298 #define	dump_dma_handle(flag, dip, hp)
299 #endif
300 
301 #ifdef	__cplusplus
302 }
303 #endif
304 
305 #endif	/* _SYS_PCI_DMA_H */
306