1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26#include <sys/types.h>
27#include <sys/conf.h>
28#include <sys/ddi.h>
29#include <sys/sunddi.h>
30#include <sys/ddi_impldefs.h>
31#include <sys/cmn_err.h>
32#include <vm/hat_sfmmu.h>
33
34#include <sys/iommu.h>
35#include <sys/iocache.h>
36#include <sys/sysiosbus.h>
37
38#include <sys/nexusdebug.h>
39#include <sys/debug.h>
40
41#define	IOCACHE_REGISTERS_DEBUG		0x1
42#define	IOCACHE_SYNC_DEBUG		0x2
43#define	IOCACHE_DIAG_REG_DEBUG		0x4
44#define	IOCACHE_SYNC_FAIL_DEBUG		0x8
45
46#define	MAX_RETRY			10
47
48/* Flag which enables the streaming buffer */
49int stream_buf_on = 1;
50/*
51 * This is the number of pages that a mapping request needs before we force
52 * the streaming buffer sync code to use diagnostic registers.  This value
53 * was determined through a series of test runs measuring dma mapping
54 * setup performance.
55 */
56int stream_buf_sync_using_diag = 36;
57
58int
59stream_buf_init(struct sbus_soft_state *softsp, caddr_t address)
60{
61	uchar_t version;
62#ifdef DEBUG
63	debug_info = 1;
64	debug_print_level = 0;
65#endif
66	version = (uchar_t)(*softsp->sysio_ctrl_reg >> SYSIO_VER_SHIFT);
67	version &= 0xf;
68
69	if (stream_buf_on == 0 || version == 0) {
70		softsp->stream_buf_off = STREAM_BUF_OFF;
71		if (version == 0)
72			cmn_err(CE_CONT, "Disabling streaming buffer due to "
73			    "SYSIO Rev %d.\n", version);
74		return (DDI_SUCCESS);
75	}
76
77	/*
78	 * Simply add each registers offset to the base address
79	 * to calculate the already mapped virtual address of
80	 * the device register...
81	 *
82	 * define a macro for the pointer arithmetic; all registers
83	 * are 64 bits wide and are defined as uint64_t's.
84	 */
85
86#define	REG_ADDR(b, o)	(uint64_t *)((caddr_t)(b) + (o))
87
88	softsp->str_buf_ctrl_reg = REG_ADDR(address, OFF_STR_BUF_CTRL_REG);
89	softsp->str_buf_flush_reg = REG_ADDR(address, OFF_STR_BUF_FLUSH_REG);
90	softsp->str_buf_sync_reg = REG_ADDR(address, OFF_STR_BUF_SYNC_REG);
91	softsp->str_buf_pg_tag_diag = REG_ADDR(address, STR_BUF_PAGE_TAG_DIAG);
92
93#undef	REG_ADDR
94
95	DPRINTF(IOCACHE_REGISTERS_DEBUG, ("Streaming buffer control reg: 0x%p, "
96	    "Streaming buffer flush reg: 0x%p, Streaming buffer sync reg: 0x%p",
97	    (void *)softsp->str_buf_ctrl_reg, (void *)softsp->str_buf_flush_reg,
98	    (void *)softsp->str_buf_sync_reg));
99
100	/* Initialize stream buffer sync reg mutex */
101	mutex_init(&softsp->sync_reg_lock, NULL, MUTEX_DEFAULT, NULL);
102
103	/* Turn on per instance streaming buffer flag */
104	softsp->stream_buf_off = 0;
105
106	/* Set the hardware registers */
107	(void) stream_buf_resume_init(softsp);
108
109	return (DDI_SUCCESS);
110}
111
112int
113stream_buf_uninit(struct sbus_soft_state *softsp)
114{
115	/* Turn off per instance streaming buffer flag */
116	softsp->stream_buf_off = 1;
117
118	/* Turn off the streaming buffer */
119	*softsp->str_buf_ctrl_reg = STREAM_BUF_DISABLE;
120
121	return (DDI_SUCCESS);
122}
123/*
124 * Initialize stream buf hardware when the system is being resumed.
125 * (Subset of stream_buf_init())
126 */
127int
128stream_buf_resume_init(struct sbus_soft_state *softsp)
129{
130	uchar_t version;
131
132	version = (uchar_t)(*softsp->sysio_ctrl_reg >> SYSIO_VER_SHIFT);
133	version &= 0xf;
134
135	if (stream_buf_on == 0 || version == 0) {
136		softsp->stream_buf_off = STREAM_BUF_OFF;
137		return (DDI_SUCCESS);
138	}
139
140	/* Turn on the streaming buffer */
141	*softsp->str_buf_ctrl_reg = STREAM_BUF_ENABLE;
142
143	return (DDI_SUCCESS);
144}
145
146/*
147 * The SYSIO spec says that it will get back to us within 0.5 seconds,
148 * but loaded systems have seen response times over 1.5 seconds.  We
149 * err on the side of caution and set the timeout to be 10 seconds.
150 */
151#define	SCACHE_NSEC_WAIT	(10ull * NANOSEC)
152
153/*
154 * We want to avoid using gethrtime every time we check sync_flag,
155 * so we take SCACHE_SPIN laps before beginning to use gethrtime.
156 */
157#define	SCACHE_SPIN		10000000
158
159void
160sync_stream_buf(struct sbus_soft_state *softsp, ioaddr_t addr, uint_t npages,
161	int *sync_flag, uint64_t phys_sync_flag)
162{
163#ifndef lint
164	volatile uint64_t tmp;
165#endif
166
167	int cntr = 0;
168
169	if (softsp->stream_buf_off != 0)
170		return;
171
172	DPRINTF(IOCACHE_SYNC_DEBUG, ("sync_stream_buf: ioaddr 0x%x, page cnt "
173	    "0x%x, sync flag 0x%p, sync flag pf 0x%lx\n", addr, npages,
174	    (void *)sync_flag, phys_sync_flag));
175
176	ASSERT(npages > (uint_t)0);
177
178	/* Acquire the sync lock */
179	mutex_enter(&softsp->sync_reg_lock);
180
181	*sync_flag = 0;
182
183	if (npages > stream_buf_sync_using_diag) {
184		int i;
185		volatile uint64_t *reg_addr;
186		uint64_t reg;
187		uint_t ioaddr;
188		uint_t hiaddr = addr + (npages * IOMMU_PAGESIZE);
189		int do_sync = 0;
190
191		for (i = 0, reg_addr = softsp->str_buf_pg_tag_diag;
192		    i < STREAM_CACHE_LINES; i++, reg_addr++) {
193
194			/* Read the page tag diag reg */
195			reg = *reg_addr;
196#ifdef DEBUG
197			{
198				uint_t hi, lo;
199				hi = (uint_t)(reg >> 32);
200				lo = (uint_t)(reg & 0xffffffff);
201				DPRINTF(IOCACHE_DIAG_REG_DEBUG,
202				    ("IO cache line diag "
203				    "reg addr 0x%p, hi0x%x lo0x%x\n",
204				    (void *)reg_addr, hi, lo));
205			}
206#endif /* DEBUG */
207			/* Check for a valid line */
208			if (reg & STR_PG_VALID) {
209				ioaddr = (uint_t)reg << STR_PG_SHIFT;
210
211				DPRINTF(IOCACHE_DIAG_REG_DEBUG, ("ioaddr 0x%x, "
212				    "range base 0x%x, range extent 0x%x\n",
213				    ioaddr, addr,
214				    addr + (npages * IOMMU_PAGESIZE)));
215				if (ioaddr >= addr && ioaddr <= hiaddr) {
216					*softsp->str_buf_flush_reg = (uint64_t)
217					    ioaddr;
218					do_sync = 1;
219				}
220			}
221		}
222
223		if (!do_sync) {
224			mutex_exit(&softsp->sync_reg_lock);
225			return;
226		}
227	} else {
228		do {
229			*softsp->str_buf_flush_reg = (uint64_t)addr;
230			addr += IOMMU_PAGESIZE;
231			npages--;
232		} while (npages > (uint_t)0);
233	}
234
235	/* Ask the hardware to flag when the flush is complete */
236	*softsp->str_buf_sync_reg = phys_sync_flag;
237
238#ifndef lint
239	/*
240	 * Due to the sun4u memory models, this noncached load will sync the
241	 * order of all prior loads and stores regardless of cacheability.
242	 * No membar_stst() is needed after zeroing the flush sync flag.
243	 */
244	tmp = *softsp->sbus_ctrl_reg;
245#endif
246
247	/*
248	 * Begin spinning on the hardware sync register.  We'll spin for
249	 * a while (SCACHE_SPIN) before using gethrtime, but once that time
250	 * is up we'll drop into an inner loop where we use gethrtime on
251	 * every iteration.  Once SCACHE_NSEC_WAIT nanoseconds have
252	 * elapsed, we'll assume a Bad Thing has happened and toss.
253	 */
254	while (!*((volatile int *)sync_flag)) {
255		if (cntr++ == SCACHE_SPIN) {
256			/*
257			 * If we're here, then we've spun long enough
258			 * to justify use of gethrtime each iteration.
259			 */
260			hrtime_t nsec_start, nsectowait, nsec_current;
261			nsectowait = SCACHE_NSEC_WAIT;
262			nsec_current = nsec_start = gethrtime();
263
264			while (!*((volatile int *)sync_flag)) {
265				/*
266				 * Double check the sync flag again before
267				 * we panic in case we get preempted.
268				 * See bugid 4126896
269				 */
270				nsec_current = gethrtime();
271				if ((nsec_current - nsec_start) > nsectowait &&
272				    !*((volatile int *)sync_flag)) {
273					/*
274					 * Trouble.  The SYSIO chip has
275					 * seemingly gone AWOL.  Vomit.
276					 */
277					panic("streaming buffer timed out");
278				}
279			}
280		}
281	}
282
283	/* Finally, drop the sync lock */
284	mutex_exit(&softsp->sync_reg_lock);
285}
286