1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License").  You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22/*
23 * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27#include <sys/param.h>
28#include <sys/errno.h>
29#include <sys/asm_linkage.h>
30#include <sys/vtrace.h>
31#include <sys/machthread.h>
32#include <sys/clock.h>
33#include <sys/asi.h>
34#include <sys/fsr.h>
35#include <sys/privregs.h>
36#include <sys/fpras_impl.h>
37
38#include "assym.h"
39
40/*
41 * Pseudo-code to aid in understanding the control flow of the
42 * bcopy/copyin/copyout routines.
43 *
44 * On entry:
45 *
46 * 	! Determine whether to use the FP register version
47 * 	! or the leaf routine version depending on size
48 * 	! of copy and flags.  Set up error handling accordingly.
49 *	! The transition point depends on whether the src and
50 * 	! dst addresses can be aligned to long word, word,
51 * 	! half word, or byte boundaries.
52 *	!
53 *	! WARNING: <Register usage convention>
54 *	! For FP version, %l6 holds previous error handling and
55 *	! a flag: TRAMP_FLAG (low bits)
56 *	! for leaf routine version, %o4 holds those values.
57 *	! So either %l6 or %o4 is reserved and not available for
58 *	! any other use.
59 *
60 * 	if (length <= VIS_COPY_THRESHOLD) 	! start with a quick test
61 * 		go to small_copy;		! to speed short copies
62 *
63 * 	! src, dst long word alignable
64 * 		if (hw_copy_limit_8 == 0) 	! hw_copy disabled
65 * 			go to small_copy;
66 *		if (length <= hw_copy_limit_8)
67 * 			go to small_copy;
68 * 		go to FPBLK_copy;
69 * 	}
70 * 	if (src,dst not alignable) {
71 * 		if (hw_copy_limit_1 == 0) 	! hw_copy disabled
72 * 			go to small_copy;
73 *		if (length <= hw_copy_limit_1)
74 * 			go to small_copy;
75 * 		go to FPBLK_copy;
76 * 	}
77 * 	if (src,dst halfword alignable) {
78 * 		if (hw_copy_limit_2 == 0) 	! hw_copy disabled
79 * 			go to small_copy;
80 *		if (length <= hw_copy_limit_2)
81 * 			go to small_copy;
82 * 		go to FPBLK_copy;
83 * 	}
84 * 	if (src,dst word alignable) {
85 * 		if (hw_copy_limit_4 == 0) 	! hw_copy disabled
86 * 			go to small_copy;
87 *		if (length <= hw_copy_limit_4)
88 * 			go to small_copy;
89 * 		go to FPBLK_copy;
90 * 	}
91 *
92 * small_copy:
93 *	Setup_leaf_rtn_error_handler; 		! diffs for each entry point
94 *
95 *	if (count <= 3)				! fast path for tiny copies
96 *		go to sm_left;			! special finish up code
97 *	else
98 *		if (count > CHKSIZE)		! medium sized copies
99 *			go to sm_med		! tuned by alignment
100 *		if(src&dst not both word aligned) {
101 *	sm_movebytes:
102 *			move byte by byte in 4-way unrolled loop
103 *			fall into sm_left;
104 *	sm_left:
105 *			move 0-3 bytes byte at a time as needed.
106 *			restore error handler and exit.
107 *
108 * 		} else {	! src&dst are word aligned
109 *			check for at least 8 bytes left,
110 *			move word at a time, unrolled by 2
111 *			when fewer than 8 bytes left,
112 *	sm_half:	move half word at a time while 2 or more bytes left
113 *	sm_byte:	move final byte if necessary
114 *	sm_exit:
115 *			restore error handler and exit.
116 *		}
117 *
118 * ! Medium length cases with at least CHKSIZE bytes available
119 * ! method: line up src and dst as best possible, then
120 * ! move data in 4-way unrolled loops.
121 *
122 * sm_med:
123 *	if(src&dst unalignable)
124 * 		go to sm_movebytes
125 *	if(src&dst halfword alignable)
126 *		go to sm_movehalf
127 *	if(src&dst word alignable)
128 *		go to sm_moveword
129 * ! fall into long word movement
130 *	move bytes until src is word aligned
131 *	if not long word aligned, move a word
132 *	move long words in 4-way unrolled loop until < 32 bytes left
133 *      move long words in 1-way unrolled loop until < 8 bytes left
134 *	if zero bytes left, goto sm_exit
135 *	if one byte left, go to sm_byte
136 *	else go to sm_half
137 *
138 * sm_moveword:
139 *	move bytes until src is word aligned
140 *	move words in 4-way unrolled loop until < 16 bytes left
141 *      move words in 1-way unrolled loop until < 4 bytes left
142 *	if zero bytes left, goto sm_exit
143 *	if one byte left, go to sm_byte
144 *	else go to sm_half
145 *
146 * sm_movehalf:
147 *	move a byte if needed to align src on halfword
148 *	move halfwords in 4-way unrolled loop until < 8 bytes left
149 *	if zero bytes left, goto sm_exit
150 *	if one byte left, go to sm_byte
151 *	else go to sm_half
152 *
153 *
154 * FPBLK_copy:
155 * 	%l6 = curthread->t_lofault;
156 * 	if (%l6 != NULL) {
157 * 		membar #Sync
158 * 		curthread->t_lofault = .copyerr;
159 * 		caller_error_handler = TRUE             ! %l6 |= 2
160 * 	}
161 *
162 *	! for FPU testing we must not migrate cpus
163 * 	if (curthread->t_lwp == NULL) {
164 *		! Kernel threads do not have pcb's in which to store
165 *		! the floating point state, so disallow preemption during
166 *		! the copy.  This also prevents cpu migration.
167 * 		kpreempt_disable(curthread);
168 *	} else {
169 *		thread_nomigrate();
170 *	}
171 *
172 * 	old_fprs = %fprs;
173 * 	old_gsr = %gsr;
174 * 	if (%fprs.fef) {
175 * 		%fprs.fef = 1;
176 * 		save current fpregs on stack using blockstore
177 * 	} else {
178 * 		%fprs.fef = 1;
179 * 	}
180 *
181 *
182 * 	do_blockcopy_here;
183 *
184 * In lofault handler:
185 *	curthread->t_lofault = .copyerr2;
186 *	Continue on with the normal exit handler
187 *
188 * On normal exit:
189 * 	%gsr = old_gsr;
190 * 	if (old_fprs & FPRS_FEF)
191 * 		restore fpregs from stack using blockload
192 *	else
193 *		zero fpregs
194 * 	%fprs = old_fprs;
195 * 	membar #Sync
196 * 	curthread->t_lofault = (%l6 & ~3);
197 *	! following test omitted from copyin/copyout as they
198 *	! will always have a current thread
199 * 	if (curthread->t_lwp == NULL)
200 *		kpreempt_enable(curthread);
201 *	else
202 *		thread_allowmigrate();
203 * 	return (0)
204 *
205 * In second lofault handler (.copyerr2):
206 *	We've tried to restore fp state from the stack and failed.  To
207 *	prevent from returning with a corrupted fp state, we will panic.
208 */
209
210/*
211 * Comments about optimization choices
212 *
213 * The initial optimization decision in this code is to determine
214 * whether to use the FP registers for a copy or not.  If we don't
215 * use the FP registers, we can execute the copy as a leaf routine,
216 * saving a register save and restore.  Also, less elaborate setup
217 * is required, allowing short copies to be completed more quickly.
218 * For longer copies, especially unaligned ones (where the src and
219 * dst do not align to allow simple ldx,stx operation), the FP
220 * registers allow much faster copy operations.
221 *
222 * The estimated extra cost of the FP path will vary depending on
223 * src/dst alignment, dst offset from the next 64 byte FPblock store
224 * boundary, remaining src data after the last full dst cache line is
225 * moved whether the FP registers need to be saved, and some other
226 * minor issues.  The average additional overhead is estimated to be
227 * 400 clocks.  Since each non-repeated/predicted tst and branch costs
228 * around 10 clocks, elaborate calculation would slow down to all
229 * longer copies and only benefit a small portion of medium sized
230 * copies.  Rather than incur such cost, we chose fixed transition
231 * points for each of the alignment choices.
232 *
233 * For the inner loop, here is a comparison of the per cache line
234 * costs for each alignment when src&dst are in cache:
235 *
236 * byte aligned:  108 clocks slower for non-FPBLK
237 * half aligned:   44 clocks slower for non-FPBLK
238 * word aligned:   12 clocks slower for non-FPBLK
239 * long aligned:    4 clocks >>faster<< for non-FPBLK
240 *
241 * The long aligned loop runs faster because it does no prefetching.
242 * That wins if the data is not in cache or there is too little
243 * data to gain much benefit from prefetching.  But when there
244 * is more data and that data is not in cache, failing to prefetch
245 * can run much slower.  In addition, there is a 2 Kbyte store queue
246 * which will cause the non-FPBLK inner loop to slow for larger copies.
247 * The exact tradeoff is strongly load and application dependent, with
248 * increasing risk of a customer visible performance regression if the
249 * non-FPBLK code is used for larger copies. Studies of synthetic in-cache
250 * vs out-of-cache copy tests in user space suggest 1024 bytes as a safe
251 * upper limit for the non-FPBLK code.  To minimize performance regression
252 * risk while still gaining the primary benefits of the improvements to
253 * the non-FPBLK code, we set an upper bound of 1024 bytes for the various
254 * hw_copy_limit_*.  Later experimental studies using different values
255 * of hw_copy_limit_* can be used to make further adjustments if
256 * appropriate.
257 *
258 * hw_copy_limit_1 = src and dst are byte aligned but not halfword aligned
259 * hw_copy_limit_2 = src and dst are halfword aligned but not word aligned
260 * hw_copy_limit_4 = src and dst are word aligned but not longword aligned
261 * hw_copy_limit_8 = src and dst are longword aligned
262 *
263 * To say that src and dst are word aligned means that after
264 * some initial alignment activity of moving 0 to 3 bytes,
265 * both the src and dst will be on word boundaries so that
266 * word loads and stores may be used.
267 *
268 * Recommended initial values as of Mar 2004, includes testing
269 * on Cheetah+ (900MHz), Cheetah++ (1200MHz), and Jaguar(1050MHz):
270 * hw_copy_limit_1 =  256
271 * hw_copy_limit_2 =  512
272 * hw_copy_limit_4 = 1024
273 * hw_copy_limit_8 = 1024 (or 1536 on some systems)
274 *
275 *
276 * If hw_copy_limit_? is set to zero, then use of FPBLK copy is
277 * disabled for that alignment choice.
278 * If hw_copy_limit_? is set to a value between 1 and VIS_COPY_THRESHOLD (256)
279 * the value of VIS_COPY_THRESHOLD is used.
280 * It is not envisioned that hw_copy_limit_? will be changed in the field
281 * It is provided to allow for disabling FPBLK copies and to allow
282 * easy testing of alternate values on future HW implementations
283 * that might have different cache sizes, clock rates or instruction
284 * timing rules.
285 *
286 * Our first test for FPBLK copies vs non-FPBLK copies checks a minimum
287 * threshold to speedup all shorter copies (less than 256).  That
288 * saves an alignment test, memory reference, and enabling test
289 * for all short copies, or an estimated 24 clocks.
290 *
291 * The order in which these limits are checked does matter since each
292 * non-predicted tst and branch costs around 10 clocks.
293 * If src and dst are randomly selected addresses,
294 * 4 of 8 will not be alignable.
295 * 2 of 8 will be half word alignable.
296 * 1 of 8 will be word alignable.
297 * 1 of 8 will be long word alignable.
298 * But, tests on running kernels show that src and dst to copy code
299 * are typically not on random alignments.  Structure copies and
300 * copies of larger data sizes are often on long word boundaries.
301 * So we test the long word alignment case first, then
302 * the byte alignment, then halfword, then word alignment.
303 *
304 * Several times, tests for length are made to split the code
305 * into subcases.  These tests often allow later tests to be
306 * avoided.  For example, within the non-FPBLK copy, we first
307 * check for tiny copies of 3 bytes or less.  That allows us
308 * to use a 4-way unrolled loop for the general byte copy case
309 * without a test on loop entry.
310 * We subdivide the non-FPBLK case further into CHKSIZE bytes and less
311 * vs longer cases.  For the really short case, we don't attempt
312 * align src and dst.  We try to minimize special case tests in
313 * the shortest loops as each test adds a significant percentage
314 * to the total time.
315 *
316 * For the medium sized cases, we allow ourselves to adjust the
317 * src and dst alignment and provide special cases for each of
318 * the four adjusted alignment cases. The CHKSIZE that was used
319 * to decide between short and medium size was chosen to be 39
320 * as that allows for the worst case of 7 bytes of alignment
321 * shift and 4 times 8 bytes for the first long word unrolling.
322 * That knowledge saves an initial test for length on entry into
323 * the medium cases.  If the general loop unrolling factor were
324 * to be increases, this number would also need to be adjusted.
325 *
326 * For all cases in the non-FPBLK code where it is known that at
327 * least 4 chunks of data are available for movement, the
328 * loop is unrolled by four.  This 4-way loop runs in 8 clocks
329 * or 2 clocks per data element.  Due to limitations of the
330 * branch instruction on Cheetah, Jaguar, and Panther, the
331 * minimum time for a small, tight loop is 3 clocks.  So
332 * the 4-way loop runs 50% faster than the fastest non-unrolled
333 * loop.
334 *
335 * Instruction alignment is forced by used of .align 16 directives
336 * and nops which are not executed in the code.  This
337 * combination of operations shifts the alignment of following
338 * loops to insure that loops are aligned so that their instructions
339 * fall within the minimum number of 4 instruction fetch groups.
340 * If instructions are inserted or removed between the .align
341 * instruction and the unrolled loops, then the alignment needs
342 * to be readjusted.  Misaligned loops can add a clock per loop
343 * iteration to the loop timing.
344 *
345 * In a few cases, code is duplicated to avoid a branch.  Since
346 * a non-predicted tst and branch takes 10 clocks, this savings
347 * is judged an appropriate time-space tradeoff.
348 *
349 * Within the FPBLK-code, the prefetch method in the inner
350 * loop needs to be explained as it is not standard.  Two
351 * prefetches are issued for each cache line instead of one.
352 * The primary one is at the maximum reach of 8 cache lines.
353 * Most of the time, that maximum prefetch reach gives the
354 * cache line more time to reach the processor for systems with
355 * higher processor clocks.  But, sometimes memory interference
356 * can cause that prefetch to be dropped.  Putting a second
357 * prefetch at a reach of 5 cache lines catches the drops
358 * three iterations later and shows a measured improvement
359 * in performance over any similar loop with a single prefetch.
360 * The prefetches are placed in the loop so they overlap with
361 * non-memory instructions, so that there is no extra cost
362 * when the data is already in-cache.
363 *
364 */
365
366/*
367 * Notes on preserving existing fp state and on membars.
368 *
369 * When a copyOP decides to use fp we may have to preserve existing
370 * floating point state.  It is not the caller's state that we need to
371 * preserve - the rest of the kernel does not use fp and, anyway, fp
372 * registers are volatile across a call.  Some examples:
373 *
374 *	- userland has fp state and is interrupted (device interrupt
375 *	  or trap) and within the interrupt/trap handling we use
376 *	  bcopy()
377 *	- another (higher level) interrupt or trap handler uses bcopy
378 *	  while a bcopy from an earlier interrupt is still active
379 *	- an asynchronous error trap occurs while fp state exists (in
380 *	  userland or in kernel copy) and the tl0 component of the handling
381 *	  uses bcopy
382 *	- a user process with fp state incurs a copy-on-write fault and
383 *	  hwblkpagecopy always uses fp
384 *
385 * We therefore need a per-call place in which to preserve fp state -
386 * using our stack is ideal (and since fp copy cannot be leaf optimized
387 * because of calls it makes, this is no hardship).
388 *
389 * The following membar BLD/BST discussion is Cheetah pipeline specific.
390 * In Cheetah BLD is blocking, #LoadLoad/#LoadStore/#StoreStore are
391 * nops (those semantics always apply) and #StoreLoad is implemented
392 * as a membar #Sync.
393 *
394 * It is possible that the owner of the fp state has a block load or
395 * block store still "in flight" at the time we come to preserve that
396 * state.  Block loads are blocking in Cheetah pipelines so we do not
397 * need to sync with them.  In preserving fp regs we will use block stores
398 * (which are not blocking in Cheetah pipelines) so we require a membar #Sync
399 * after storing state (so that our subsequent use of those registers
400 * does not modify them before the block stores complete);  this membar
401 * also serves to sync with block stores the owner of the fp state has
402 * initiated.
403 *
404 * When we have finished fp copy (with it's repeated block stores)
405 * we must membar #Sync so that our block stores may complete before
406 * we either restore the original fp state into the fp registers or
407 * return to a caller which may initiate other fp operations that could
408 * modify the fp regs we used before the block stores complete.
409 *
410 * Synchronous faults (eg, unresolvable DMMU miss) that occur while
411 * t_lofault is not NULL will not panic but will instead trampoline
412 * to the registered lofault handler.  There is no need for any
413 * membars for these - eg, our store to t_lofault will always be visible to
414 * ourselves and it is our cpu which will take any trap.
415 *
416 * Asynchronous faults (eg, uncorrectable ECC error from memory) that occur
417 * while t_lofault is not NULL will also not panic.  Since we're copying
418 * to or from userland the extent of the damage is known - the destination
419 * buffer is incomplete.  So trap handlers will trampoline to the lofault
420 * handler in this case which should take some form of error action to
421 * avoid using the incomplete buffer.  The trap handler also flags the
422 * fault so that later return-from-trap handling (for the trap that brought
423 * this thread into the kernel in the first place) can notify the process
424 * and reboot the system (or restart the service with Greenline/Contracts).
425 *
426 * Asynchronous faults (eg, uncorrectable ECC error from memory) can
427 * result in deferred error traps - the trap is taken sometime after
428 * the event and the trap PC may not be the PC of the faulting access.
429 * Delivery of such pending traps can be forced by a membar #Sync, acting
430 * as an "error barrier" in this role.  To accurately apply the user/kernel
431 * separation described in the preceding paragraph we must force delivery
432 * of deferred traps affecting kernel state before we install a lofault
433 * handler (if we interpose a new lofault handler on an existing one there
434 * is no need to repeat this), and we must force delivery of deferred
435 * errors affecting the lofault-protected region before we clear t_lofault.
436 * Failure to do so results in lost kernel state being interpreted as
437 * affecting a copyin/copyout only, or of an error that really only
438 * affects copy data being interpreted as losing kernel state.
439 *
440 * Since the copy operations may preserve and later restore floating
441 * point state that does not belong to the caller (see examples above),
442 * we must be careful in how we do this in order to prevent corruption
443 * of another program.
444 *
445 * To make sure that floating point state is always saved and restored
446 * correctly, the following "big rules" must be followed when the floating
447 * point registers will be used:
448 *
449 * 1. %l6 always holds the caller's lofault handler.  Also in this register,
450 *    Bit 1 (FPUSED_FLAG) indicates that the floating point registers are in
451 *    use.  Bit 2 (TRAMP_FLAG) indicates that the call was to bcopy, and a
452 *    lofault handler was set coming in.
453 *
454 * 2. The FPUSED flag indicates that all FP state has been successfully stored
455 *    on the stack.  It should not be set until this save has been completed.
456 *
457 * 3. The FPUSED flag should not be cleared on exit until all FP state has
458 *    been restored from the stack.  If an error occurs while restoring
459 *    data from the stack, the error handler can check this flag to see if
460 *    a restore is necessary.
461 *
462 * 4. Code run under the new lofault handler must be kept to a minimum.  In
463 *    particular, any calls to FP_ALLOWMIGRATE, which could result in a call
464 *    to kpreempt(), should not be made until after the lofault handler has
465 *    been restored.
466 */
467
468/*
469 * VIS_COPY_THRESHOLD indicates the minimum number of bytes needed
470 * to "break even" using FP/VIS-accelerated memory operations.
471 * The FPBLK code assumes a minimum number of bytes are available
472 * to be moved on entry.  Check that code carefully before
473 * reducing VIS_COPY_THRESHOLD below 256.
474 */
475/*
476 * This shadows sys/machsystm.h which can't be included due to the lack of
477 * _ASM guards in include files it references. Change it here, change it there.
478 */
479#define VIS_COPY_THRESHOLD 256
480
481/*
482 * TEST for very short copies
483 * Be aware that the maximum unroll for the short unaligned case
484 * is SHORTCOPY+1
485 */
486#define SHORTCOPY 3
487#define CHKSIZE  39
488
489/*
490 * Indicates that we're to trampoline to the error handler.
491 * Entry points bcopy, copyin_noerr, and copyout_noerr use this flag.
492 * kcopy, copyout, xcopyout, copyin, and xcopyin do not set this flag.
493 */
494#define	FPUSED_FLAG	1
495#define	TRAMP_FLAG	2
496#define	MASK_FLAGS	3
497
498/*
499 * Number of outstanding prefetches.
500 * Testing with 1200 MHz Cheetah+ and Jaguar gives best results with
501 * two prefetches, one with a reach of 8*BLOCK_SIZE+8 and one with a
502 * reach of 5*BLOCK_SIZE.  The double prefetch gives an typical improvement
503 * of 5% for large copies as compared to a single prefetch.  The reason
504 * for the improvement is that with Cheetah and Jaguar, some prefetches
505 * are dropped due to the prefetch queue being full.  The second prefetch
506 * reduces the number of cache lines that are dropped.
507 * Do not remove the double prefetch or change either CHEETAH_PREFETCH
508 * or CHEETAH_2ND_PREFETCH without extensive performance tests to prove
509 * there is no loss of performance.
510 */
511#define	CHEETAH_PREFETCH	8
512#define	CHEETAH_2ND_PREFETCH	5
513
514#define	VIS_BLOCKSIZE		64
515
516/*
517 * Size of stack frame in order to accomodate a 64-byte aligned
518 * floating-point register save area and 2 64-bit temp locations.
519 * All copy functions use two quadrants of fp registers; to assure a
520 * block-aligned two block buffer in which to save we must reserve
521 * three blocks on stack.  Not all functions preserve %pfrs on stack
522 * or need to preserve %gsr but we use HWCOPYFRAMESIZE for all.
523 *
524 *    _______________________________________ <-- %fp + STACK_BIAS
525 *    | We may need to preserve 2 quadrants |
526 *    | of fp regs, but since we do so with |
527 *    | BST/BLD we need room in which to    |
528 *    | align to VIS_BLOCKSIZE bytes.  So   |
529 *    | this area is 3 * VIS_BLOCKSIZE.     | <--  - SAVED_FPREGS_OFFSET
530 *    |-------------------------------------|
531 *    | 8 bytes to save %fprs               | <--  - SAVED_FPRS_OFFSET
532 *    |-------------------------------------|
533 *    | 8 bytes to save %gsr                | <--  - SAVED_GSR_OFFSET
534 *    ---------------------------------------
535 */
536#define	HWCOPYFRAMESIZE		((VIS_BLOCKSIZE * (2 + 1)) + (2 * 8))
537#define SAVED_FPREGS_OFFSET	(VIS_BLOCKSIZE * 3)
538#define SAVED_FPREGS_ADJUST	((VIS_BLOCKSIZE * 2) - 1)
539#define	SAVED_FPRS_OFFSET	(SAVED_FPREGS_OFFSET + 8)
540#define	SAVED_GSR_OFFSET	(SAVED_FPRS_OFFSET + 8)
541
542/*
543 * Common macros used by the various versions of the block copy
544 * routines in this file.
545 */
546
547/*
548 * In FP copies if we do not have preserved data to restore over
549 * the fp regs we used then we must zero those regs to avoid
550 * exposing portions of the data to later threads (data security).
551 *
552 * Copy functions use either quadrants 1 and 3 or 2 and 4.
553 *
554 * FZEROQ1Q3: Zero quadrants 1 and 3, ie %f0 - %f15 and %f32 - %f47
555 * FZEROQ2Q4: Zero quadrants 2 and 4, ie %f16 - %f31 and %f48 - %f63
556 *
557 * The instructions below are quicker than repeated fzero instructions
558 * since they can dispatch down two fp pipelines.
559 */
560#define	FZEROQ1Q3			\
561	fzero	%f0			;\
562	fzero	%f2			;\
563	faddd	%f0, %f2, %f4		;\
564	fmuld	%f0, %f2, %f6		;\
565	faddd	%f0, %f2, %f8		;\
566	fmuld	%f0, %f2, %f10		;\
567	faddd	%f0, %f2, %f12		;\
568	fmuld	%f0, %f2, %f14		;\
569	faddd	%f0, %f2, %f32		;\
570	fmuld	%f0, %f2, %f34		;\
571	faddd	%f0, %f2, %f36		;\
572	fmuld	%f0, %f2, %f38		;\
573	faddd	%f0, %f2, %f40		;\
574	fmuld	%f0, %f2, %f42		;\
575	faddd	%f0, %f2, %f44		;\
576	fmuld	%f0, %f2, %f46
577
578#define	FZEROQ2Q4			\
579	fzero	%f16			;\
580	fzero	%f18			;\
581	faddd	%f16, %f18, %f20	;\
582	fmuld	%f16, %f18, %f22	;\
583	faddd	%f16, %f18, %f24	;\
584	fmuld	%f16, %f18, %f26	;\
585	faddd	%f16, %f18, %f28	;\
586	fmuld	%f16, %f18, %f30	;\
587	faddd	%f16, %f18, %f48	;\
588	fmuld	%f16, %f18, %f50	;\
589	faddd	%f16, %f18, %f52	;\
590	fmuld	%f16, %f18, %f54	;\
591	faddd	%f16, %f18, %f56	;\
592	fmuld	%f16, %f18, %f58	;\
593	faddd	%f16, %f18, %f60	;\
594	fmuld	%f16, %f18, %f62
595
596/*
597 * Macros to save and restore quadrants 1 and 3 or 2 and 4 to/from the stack.
598 * Used to save and restore in-use fp registers when we want to use FP
599 * and find fp already in use and copy size still large enough to justify
600 * the additional overhead of this save and restore.
601 *
602 * A membar #Sync is needed before save to sync fp ops initiated before
603 * the call to the copy function (by whoever has fp in use); for example
604 * an earlier block load to the quadrant we are about to save may still be
605 * "in flight".  A membar #Sync is required at the end of the save to
606 * sync our block store (the copy code is about to begin ldd's to the
607 * first quadrant).  Note, however, that since Cheetah pipeline block load
608 * is blocking we can omit the initial membar before saving fp state (they're
609 * commented below in case of future porting to a chip that does not block
610 * on block load).
611 *
612 * Similarly: a membar #Sync before restore allows the block stores of
613 * the copy operation to complete before we fill the quadrants with their
614 * original data, and a membar #Sync after restore lets the block loads
615 * of the restore complete before we return to whoever has the fp regs
616 * in use.  To avoid repeated membar #Sync we make it the responsibility
617 * of the copy code to membar #Sync immediately after copy is complete
618 * and before using the BLD_*_FROMSTACK macro.
619 */
620#define BST_FPQ1Q3_TOSTACK(tmp1)				\
621	/* membar #Sync	*/					;\
622	add	%fp, STACK_BIAS - SAVED_FPREGS_ADJUST, tmp1	;\
623	and	tmp1, -VIS_BLOCKSIZE, tmp1 /* block align */	;\
624	stda	%f0, [tmp1]ASI_BLK_P				;\
625	add	tmp1, VIS_BLOCKSIZE, tmp1			;\
626	stda	%f32, [tmp1]ASI_BLK_P				;\
627	membar	#Sync
628
629#define	BLD_FPQ1Q3_FROMSTACK(tmp1)				\
630	/* membar #Sync - provided at copy completion */	;\
631	add	%fp, STACK_BIAS - SAVED_FPREGS_ADJUST, tmp1	;\
632	and	tmp1, -VIS_BLOCKSIZE, tmp1 /* block align */	;\
633	ldda	[tmp1]ASI_BLK_P, %f0				;\
634	add	tmp1, VIS_BLOCKSIZE, tmp1			;\
635	ldda	[tmp1]ASI_BLK_P, %f32				;\
636	membar	#Sync
637
638#define BST_FPQ2Q4_TOSTACK(tmp1)				\
639	/* membar #Sync */					;\
640	add	%fp, STACK_BIAS - SAVED_FPREGS_ADJUST, tmp1	;\
641	and	tmp1, -VIS_BLOCKSIZE, tmp1 /* block align */	;\
642	stda	%f16, [tmp1]ASI_BLK_P				;\
643	add	tmp1, VIS_BLOCKSIZE, tmp1			;\
644	stda	%f48, [tmp1]ASI_BLK_P				;\
645	membar	#Sync
646
647#define	BLD_FPQ2Q4_FROMSTACK(tmp1)				\
648	/* membar #Sync - provided at copy completion */	;\
649	add	%fp, STACK_BIAS - SAVED_FPREGS_ADJUST, tmp1	;\
650	and	tmp1, -VIS_BLOCKSIZE, tmp1 /* block align */	;\
651	ldda	[tmp1]ASI_BLK_P, %f16				;\
652	add	tmp1, VIS_BLOCKSIZE, tmp1			;\
653	ldda	[tmp1]ASI_BLK_P, %f48				;\
654	membar	#Sync
655
656/*
657 * FP_NOMIGRATE and FP_ALLOWMIGRATE.  Prevent migration (or, stronger,
658 * prevent preemption if there is no t_lwp to save FP state to on context
659 * switch) before commencing a FP copy, and reallow it on completion or
660 * in error trampoline paths when we were using FP copy.
661 *
662 * Both macros may call other functions, so be aware that all outputs are
663 * forfeit after using these macros.  For this reason we do not pass registers
664 * to use - we just use any outputs we want.
665 *
666 * For fpRAS we need to perform the fpRAS mechanism test on the same
667 * CPU as we use for the copy operation, both so that we validate the
668 * CPU we perform the copy on and so that we know which CPU failed
669 * if a failure is detected.  Hence we need to be bound to "our" CPU.
670 * This could be achieved through disabling preemption (and we have do it that
671 * way for threads with no t_lwp) but for larger copies this may hold
672 * higher priority threads off of cpu for too long (eg, realtime).  So we
673 * make use of the lightweight t_nomigrate mechanism where we can (ie, when
674 * we have a t_lwp).
675 *
676 * Pseudo code:
677 *
678 * FP_NOMIGRATE:
679 *
680 * if (curthread->t_lwp) {
681 *	thread_nomigrate();
682 * } else {
683 *	kpreempt_disable();
684 * }
685 *
686 * FP_ALLOWMIGRATE:
687 *
688 * if (curthread->t_lwp) {
689 *	thread_allowmigrate();
690 * } else {
691 *	kpreempt_enable();
692 * }
693 */
694
695#define	FP_NOMIGRATE(label1, label2)				\
696	ldn	[THREAD_REG + T_LWP], %o0			;\
697	brz,a,pn %o0, label1/**/f				;\
698	  ldsb	[THREAD_REG + T_PREEMPT], %o1			;\
699	call	thread_nomigrate				;\
700	  nop							;\
701	ba	label2/**/f					;\
702	  nop							;\
703label1:								;\
704	inc	%o1						;\
705	stb	%o1, [THREAD_REG + T_PREEMPT]			;\
706label2:
707
708#define	FP_ALLOWMIGRATE(label1, label2)			\
709	ldn	[THREAD_REG + T_LWP], %o0			;\
710	brz,a,pn %o0, label1/**/f				;\
711	  ldsb	[THREAD_REG + T_PREEMPT], %o1			;\
712	call thread_allowmigrate				;\
713	  nop							;\
714	ba	label2/**/f					;\
715	  nop							;\
716label1:								;\
717	dec	%o1						;\
718	brnz,pn	%o1, label2/**/f				;\
719	  stb	%o1, [THREAD_REG + T_PREEMPT]			;\
720	ldn	[THREAD_REG + T_CPU], %o0			;\
721	ldub	[%o0 + CPU_KPRUNRUN], %o0			;\
722	brz,pt	%o0, label2/**/f				;\
723	  nop							;\
724	call	kpreempt					;\
725	  rdpr	%pil, %o0					;\
726label2:
727
728/*
729 * Copy a block of storage, returning an error code if `from' or
730 * `to' takes a kernel pagefault which cannot be resolved.
731 * Returns errno value on pagefault error, 0 if all ok
732 */
733
734	.seg	".text"
735	.align	4
736
737	ENTRY(kcopy)
738
739	cmp	%o2, VIS_COPY_THRESHOLD		! check for leaf rtn case
740	bleu,pt	%ncc, .kcopy_small		! go to larger cases
741	  xor	%o0, %o1, %o3			! are src, dst alignable?
742	btst	7, %o3				!
743	bz,pt	%ncc, .kcopy_8			! check for longword alignment
744	  nop
745	btst	1, %o3				!
746	bz,pt	%ncc, .kcopy_2			! check for half-word
747	  nop
748	sethi	%hi(hw_copy_limit_1), %o3	! Check copy limit
749	ld	[%o3 + %lo(hw_copy_limit_1)], %o3
750	tst	%o3
751	bz,pn	%icc, .kcopy_small		! if zero, disable HW copy
752	  cmp	%o2, %o3			! if length <= limit
753	bleu,pt	%ncc, .kcopy_small		! go to small copy
754	  nop
755	ba,pt	%ncc, .kcopy_more		! otherwise go to large copy
756	  nop
757.kcopy_2:
758	btst	3, %o3				!
759	bz,pt	%ncc, .kcopy_4			! check for word alignment
760	  nop
761	sethi	%hi(hw_copy_limit_2), %o3	! Check copy limit
762	ld	[%o3 + %lo(hw_copy_limit_2)], %o3
763	tst	%o3
764	bz,pn	%icc, .kcopy_small		! if zero, disable HW copy
765	  cmp	%o2, %o3			! if length <= limit
766	bleu,pt	%ncc, .kcopy_small		! go to small copy
767	  nop
768	ba,pt	%ncc, .kcopy_more		! otherwise go to large copy
769	  nop
770.kcopy_4:
771	! already checked longword, must be word aligned
772	sethi	%hi(hw_copy_limit_4), %o3	! Check copy limit
773	ld	[%o3 + %lo(hw_copy_limit_4)], %o3
774	tst	%o3
775	bz,pn	%icc, .kcopy_small		! if zero, disable HW copy
776	  cmp	%o2, %o3			! if length <= limit
777	bleu,pt	%ncc, .kcopy_small		! go to small copy
778	  nop
779	ba,pt	%ncc, .kcopy_more		! otherwise go to large copy
780	  nop
781.kcopy_8:
782	sethi	%hi(hw_copy_limit_8), %o3	! Check copy limit
783	ld	[%o3 + %lo(hw_copy_limit_8)], %o3
784	tst	%o3
785	bz,pn	%icc, .kcopy_small		! if zero, disable HW copy
786	  cmp	%o2, %o3			! if length <= limit
787	bleu,pt	%ncc, .kcopy_small		! go to small copy
788	  nop
789	ba,pt	%ncc, .kcopy_more		! otherwise go to large copy
790	  nop
791
792.kcopy_small:
793	sethi	%hi(.sm_copyerr), %o5		! sm_copyerr is lofault value
794	or	%o5, %lo(.sm_copyerr), %o5
795	ldn	[THREAD_REG + T_LOFAULT], %o4	! save existing handler
796	membar	#Sync				! sync error barrier
797	ba,pt	%ncc, .sm_do_copy		! common code
798	 stn	%o5, [THREAD_REG + T_LOFAULT]	! set t_lofault
799
800.kcopy_more:
801	save	%sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
802	sethi	%hi(.copyerr), %l7		! copyerr is lofault value
803	or	%l7, %lo(.copyerr), %l7
804	ldn	[THREAD_REG + T_LOFAULT], %l6	! save existing handler
805	membar	#Sync				! sync error barrier
806	ba,pt	%ncc, .do_copy			! common code
807	  stn	%l7, [THREAD_REG + T_LOFAULT]	! set t_lofault
808
809
810/*
811 * We got here because of a fault during bcopy_more, called from kcopy or bcopy.
812 * Errno value is in %g1.  bcopy_more uses fp quadrants 1 and 3.
813 */
814.copyerr:
815	set	.copyerr2, %l0
816	membar	#Sync				! sync error barrier
817	stn	%l0, [THREAD_REG + T_LOFAULT]	! set t_lofault
818	btst	FPUSED_FLAG, %l6
819	bz	%ncc, 1f
820	  and	%l6, TRAMP_FLAG, %l0		! copy trampoline flag to %l0
821
822	ldx	[%fp + STACK_BIAS - SAVED_GSR_OFFSET], %o2	! restore gsr
823	wr	%o2, 0, %gsr
824
825	ld	[%fp + STACK_BIAS - SAVED_FPRS_OFFSET], %o3
826	btst	FPRS_FEF, %o3
827	bz,pt	%icc, 4f
828	  nop
829
830	BLD_FPQ1Q3_FROMSTACK(%o2)
831
832	ba,pt	%ncc, 1f
833	  wr	%o3, 0, %fprs		! restore fprs
834
8354:
836	FZEROQ1Q3
837	wr	%o3, 0, %fprs		! restore fprs
838
839	!
840	! Need to cater for the different expectations of kcopy
841	! and bcopy. kcopy will *always* set a t_lofault handler
842	! If it fires, we're expected to just return the error code
843	! and *not* to invoke any existing error handler. As far as
844	! bcopy is concerned, we only set t_lofault if there was an
845	! existing lofault handler. In that case we're expected to
846	! invoke the previously existing handler after resetting the
847	! t_lofault value.
848	!
8491:
850	andn	%l6, MASK_FLAGS, %l6		! turn trampoline flag off
851	membar	#Sync				! sync error barrier
852	stn	%l6, [THREAD_REG + T_LOFAULT]	! restore old t_lofault
853	FP_ALLOWMIGRATE(5, 6)
854
855	btst	TRAMP_FLAG, %l0
856	bnz,pn	%ncc, 3f
857	  nop
858	ret
859	  restore	%g1, 0, %o0
860
8613:
862	!
863	! We're here via bcopy. There *must* have been an error handler
864	! in place otherwise we would have died a nasty death already.
865	!
866	jmp	%l6				! goto real handler
867	  restore	%g0, 0, %o0		! dispose of copy window
868
869/*
870 * We got here because of a fault in .copyerr.  We can't safely restore fp
871 * state, so we panic.
872 */
873fp_panic_msg:
874	.asciz	"Unable to restore fp state after copy operation"
875
876	.align	4
877.copyerr2:
878	set	fp_panic_msg, %o0
879	call	panic
880	  nop
881
882/*
883 * We got here because of a fault during a small kcopy or bcopy.
884 * No floating point registers are used by the small copies.
885 * Errno value is in %g1.
886 */
887.sm_copyerr:
8881:
889	btst	TRAMP_FLAG, %o4
890	membar	#Sync
891	andn	%o4, TRAMP_FLAG, %o4
892	bnz,pn	%ncc, 3f
893	  stn	%o4, [THREAD_REG + T_LOFAULT]	! restore old t_lofault
894	retl
895	  mov	%g1, %o0
8963:
897	jmp	%o4				! goto real handler
898	  mov	%g0, %o0			!
899
900	SET_SIZE(kcopy)
901
902
903/*
904 * Copy a block of storage - must not overlap (from + len <= to).
905 * Registers: l6 - saved t_lofault
906 * (for short copies, o4 - saved t_lofault)
907 *
908 * Copy a page of memory.
909 * Assumes double word alignment and a count >= 256.
910 */
911
912	ENTRY(bcopy)
913
914	cmp	%o2, VIS_COPY_THRESHOLD		! check for leaf rtn case
915	bleu,pt	%ncc, .bcopy_small		! go to larger cases
916	  xor	%o0, %o1, %o3			! are src, dst alignable?
917	btst	7, %o3				!
918	bz,pt	%ncc, .bcopy_8			! check for longword alignment
919	  nop
920	btst	1, %o3				!
921	bz,pt	%ncc, .bcopy_2			! check for half-word
922	  nop
923	sethi	%hi(hw_copy_limit_1), %o3	! Check copy limit
924	ld	[%o3 + %lo(hw_copy_limit_1)], %o3
925	tst	%o3
926	bz,pn	%icc, .bcopy_small		! if zero, disable HW copy
927	  cmp	%o2, %o3			! if length <= limit
928	bleu,pt	%ncc, .bcopy_small		! go to small copy
929	  nop
930	ba,pt	%ncc, .bcopy_more		! otherwise go to large copy
931	  nop
932.bcopy_2:
933	btst	3, %o3				!
934	bz,pt	%ncc, .bcopy_4			! check for word alignment
935	  nop
936	sethi	%hi(hw_copy_limit_2), %o3	! Check copy limit
937	ld	[%o3 + %lo(hw_copy_limit_2)], %o3
938	tst	%o3
939	bz,pn	%icc, .bcopy_small		! if zero, disable HW copy
940	  cmp	%o2, %o3			! if length <= limit
941	bleu,pt	%ncc, .bcopy_small		! go to small copy
942	  nop
943	ba,pt	%ncc, .bcopy_more		! otherwise go to large copy
944	  nop
945.bcopy_4:
946	! already checked longword, must be word aligned
947	sethi	%hi(hw_copy_limit_4), %o3	! Check copy limit
948	ld	[%o3 + %lo(hw_copy_limit_4)], %o3
949	tst	%o3
950	bz,pn	%icc, .bcopy_small		! if zero, disable HW copy
951	  cmp	%o2, %o3			! if length <= limit
952	bleu,pt	%ncc, .bcopy_small		! go to small copy
953	  nop
954	ba,pt	%ncc, .bcopy_more		! otherwise go to large copy
955	  nop
956.bcopy_8:
957	sethi	%hi(hw_copy_limit_8), %o3	! Check copy limit
958	ld	[%o3 + %lo(hw_copy_limit_8)], %o3
959	tst	%o3
960	bz,pn	%icc, .bcopy_small		! if zero, disable HW copy
961	  cmp	%o2, %o3			! if length <= limit
962	bleu,pt	%ncc, .bcopy_small		! go to small copy
963	  nop
964	ba,pt	%ncc, .bcopy_more		! otherwise go to large copy
965	  nop
966
967	.align	16
968.bcopy_small:
969	ldn	[THREAD_REG + T_LOFAULT], %o4	! save t_lofault
970	tst	%o4
971	bz,pt	%icc, .sm_do_copy
972	  nop
973	sethi	%hi(.sm_copyerr), %o5
974	or	%o5, %lo(.sm_copyerr), %o5
975	membar	#Sync				! sync error barrier
976	stn	%o5, [THREAD_REG + T_LOFAULT]	! install new vector
977	or	%o4, TRAMP_FLAG, %o4		! error should trampoline
978.sm_do_copy:
979	cmp	%o2, SHORTCOPY		! check for really short case
980	bleu,pt	%ncc, .bc_sm_left	!
981	  cmp	%o2, CHKSIZE		! check for medium length cases
982	bgu,pn	%ncc, .bc_med		!
983	  or	%o0, %o1, %o3		! prepare alignment check
984	andcc	%o3, 0x3, %g0		! test for alignment
985	bz,pt	%ncc, .bc_sm_word	! branch to word aligned case
986.bc_sm_movebytes:
987	  sub	%o2, 3, %o2		! adjust count to allow cc zero test
988.bc_sm_notalign4:
989	ldub	[%o0], %o3		! read byte
990	stb	%o3, [%o1]		! write byte
991	subcc	%o2, 4, %o2		! reduce count by 4
992	ldub	[%o0 + 1], %o3		! repeat for a total of 4 bytes
993	add	%o0, 4, %o0		! advance SRC by 4
994	stb	%o3, [%o1 + 1]
995	ldub	[%o0 - 2], %o3
996	add	%o1, 4, %o1		! advance DST by 4
997	stb	%o3, [%o1 - 2]
998	ldub	[%o0 - 1], %o3
999	bgt,pt	%ncc, .bc_sm_notalign4	! loop til 3 or fewer bytes remain
1000	  stb	%o3, [%o1 - 1]
1001	add	%o2, 3, %o2		! restore count
1002.bc_sm_left:
1003	tst	%o2
1004	bz,pt	%ncc, .bc_sm_exit	! check for zero length
1005	  deccc	%o2			! reduce count for cc test
1006	ldub	[%o0], %o3		! move one byte
1007	bz,pt	%ncc, .bc_sm_exit
1008	  stb	%o3, [%o1]
1009	ldub	[%o0 + 1], %o3		! move another byte
1010	deccc	%o2			! check for more
1011	bz,pt	%ncc, .bc_sm_exit
1012	  stb	%o3, [%o1 + 1]
1013	ldub	[%o0 + 2], %o3		! move final byte
1014	stb	%o3, [%o1 + 2]
1015	membar	#Sync				! sync error barrier
1016	andn	%o4, TRAMP_FLAG, %o4
1017	stn	%o4, [THREAD_REG + T_LOFAULT]	! restore old t_lofault
1018	retl
1019	  mov	%g0, %o0		! return 0
1020	.align	16
1021	nop				! instruction alignment
1022					! see discussion at start of file
1023.bc_sm_words:
1024	lduw	[%o0], %o3		! read word
1025.bc_sm_wordx:
1026	subcc	%o2, 8, %o2		! update count
1027	stw	%o3, [%o1]		! write word
1028	add	%o0, 8, %o0		! update SRC
1029	lduw	[%o0 - 4], %o3		! read word
1030	add	%o1, 8, %o1		! update DST
1031	bgt,pt	%ncc, .bc_sm_words	! loop til done
1032	  stw	%o3, [%o1 - 4]		! write word
1033	addcc	%o2, 7, %o2		! restore count
1034	bz,pt	%ncc, .bc_sm_exit
1035	  deccc	%o2
1036	bz,pt	%ncc, .bc_sm_byte
1037.bc_sm_half:
1038	  subcc	%o2, 2, %o2		! reduce count by 2
1039	add	%o0, 2, %o0		! advance SRC by 2
1040	lduh	[%o0 - 2], %o3		! read half word
1041	add	%o1, 2, %o1		! advance DST by 2
1042	bgt,pt	%ncc, .bc_sm_half	! loop til done
1043	  sth	%o3, [%o1 - 2]		! write half word
1044	addcc	%o2, 1, %o2		! restore count
1045	bz,pt	%ncc, .bc_sm_exit
1046	  nop
1047.bc_sm_byte:
1048	ldub	[%o0], %o3
1049	stb	%o3, [%o1]
1050	membar	#Sync				! sync error barrier
1051	andn	%o4, TRAMP_FLAG, %o4
1052	stn	%o4, [THREAD_REG + T_LOFAULT]	! restore old t_lofault
1053	retl
1054	  mov	%g0, %o0		! return 0
1055
1056.bc_sm_word:
1057	subcc	%o2, 4, %o2		! update count
1058	bgt,pt	%ncc, .bc_sm_wordx
1059	  lduw	[%o0], %o3		! read word
1060	addcc	%o2, 3, %o2		! restore count
1061	bz,pt	%ncc, .bc_sm_exit
1062	  stw	%o3, [%o1]		! write word
1063	deccc	%o2			! reduce count for cc test
1064	ldub	[%o0 + 4], %o3		! load one byte
1065	bz,pt	%ncc, .bc_sm_exit
1066	  stb	%o3, [%o1 + 4]		! store one byte
1067	ldub	[%o0 + 5], %o3		! load second byte
1068	deccc	%o2
1069	bz,pt	%ncc, .bc_sm_exit
1070	  stb	%o3, [%o1 + 5]		! store second byte
1071	ldub	[%o0 + 6], %o3		! load third byte
1072	stb	%o3, [%o1 + 6]		! store third byte
1073.bc_sm_exit:
1074	membar	#Sync				! sync error barrier
1075	andn	%o4, TRAMP_FLAG, %o4
1076	stn	%o4, [THREAD_REG + T_LOFAULT]	! restore old t_lofault
1077	retl
1078	  mov	%g0, %o0		! return 0
1079
1080	.align 16
1081.bc_med:
1082	xor	%o0, %o1, %o3		! setup alignment check
1083	btst	1, %o3
1084	bnz,pt	%ncc, .bc_sm_movebytes	! unaligned
1085	  nop
1086	btst	3, %o3
1087	bnz,pt	%ncc, .bc_med_half	! halfword aligned
1088	  nop
1089	btst	7, %o3
1090	bnz,pt	%ncc, .bc_med_word	! word aligned
1091	  nop
1092.bc_med_long:
1093	btst	3, %o0			! check for
1094	bz,pt	%ncc, .bc_med_long1	! word alignment
1095	  nop
1096.bc_med_long0:
1097	ldub	[%o0], %o3		! load one byte
1098	inc	%o0
1099	stb	%o3,[%o1]		! store byte
1100	inc	%o1
1101	btst	3, %o0
1102	bnz,pt	%ncc, .bc_med_long0
1103	  dec	%o2
1104.bc_med_long1:			! word aligned
1105	btst	7, %o0			! check for long word
1106	bz,pt	%ncc, .bc_med_long2
1107	  nop
1108	lduw	[%o0], %o3		! load word
1109	add	%o0, 4, %o0		! advance SRC by 4
1110	stw	%o3, [%o1]		! store word
1111	add	%o1, 4, %o1		! advance DST by 4
1112	sub	%o2, 4, %o2		! reduce count by 4
1113!
1114!  Now long word aligned and have at least 32 bytes to move
1115!
1116.bc_med_long2:
1117	sub	%o2, 31, %o2		! adjust count to allow cc zero test
1118.bc_med_lmove:
1119	ldx	[%o0], %o3		! read long word
1120	stx	%o3, [%o1]		! write long word
1121	subcc	%o2, 32, %o2		! reduce count by 32
1122	ldx	[%o0 + 8], %o3		! repeat for a total for 4 long words
1123	add	%o0, 32, %o0		! advance SRC by 32
1124	stx	%o3, [%o1 + 8]
1125	ldx	[%o0 - 16], %o3
1126	add	%o1, 32, %o1		! advance DST by 32
1127	stx	%o3, [%o1 - 16]
1128	ldx	[%o0 - 8], %o3
1129	bgt,pt	%ncc, .bc_med_lmove	! loop til 31 or fewer bytes left
1130	  stx	%o3, [%o1 - 8]
1131	addcc	%o2, 24, %o2		! restore count to long word offset
1132	ble,pt	%ncc, .bc_med_lextra	! check for more long words to move
1133	  nop
1134.bc_med_lword:
1135	ldx	[%o0], %o3		! read long word
1136	subcc	%o2, 8, %o2		! reduce count by 8
1137	stx	%o3, [%o1]		! write long word
1138	add	%o0, 8, %o0		! advance SRC by 8
1139	bgt,pt	%ncc, .bc_med_lword	! loop til 7 or fewer bytes left
1140	  add	%o1, 8, %o1		! advance DST by 8
1141.bc_med_lextra:
1142	addcc	%o2, 7, %o2		! restore rest of count
1143	bz,pt	%ncc, .bc_sm_exit	! if zero, then done
1144	  deccc	%o2
1145	bz,pt	%ncc, .bc_sm_byte
1146	  nop
1147	ba,pt	%ncc, .bc_sm_half
1148	  nop
1149
1150	.align 16
1151.bc_med_word:
1152	btst	3, %o0			! check for
1153	bz,pt	%ncc, .bc_med_word1	! word alignment
1154	  nop
1155.bc_med_word0:
1156	ldub	[%o0], %o3		! load one byte
1157	inc	%o0
1158	stb	%o3,[%o1]		! store byte
1159	inc	%o1
1160	btst	3, %o0
1161	bnz,pt	%ncc, .bc_med_word0
1162	  dec	%o2
1163!
1164!  Now word aligned and have at least 36 bytes to move
1165!
1166.bc_med_word1:
1167	sub	%o2, 15, %o2		! adjust count to allow cc zero test
1168.bc_med_wmove:
1169	lduw	[%o0], %o3		! read word
1170	stw	%o3, [%o1]		! write word
1171	subcc	%o2, 16, %o2		! reduce count by 16
1172	lduw	[%o0 + 4], %o3		! repeat for a total for 4 words
1173	add	%o0, 16, %o0		! advance SRC by 16
1174	stw	%o3, [%o1 + 4]
1175	lduw	[%o0 - 8], %o3
1176	add	%o1, 16, %o1		! advance DST by 16
1177	stw	%o3, [%o1 - 8]
1178	lduw	[%o0 - 4], %o3
1179	bgt,pt	%ncc, .bc_med_wmove	! loop til 15 or fewer bytes left
1180	  stw	%o3, [%o1 - 4]
1181	addcc	%o2, 12, %o2		! restore count to word offset
1182	ble,pt	%ncc, .bc_med_wextra	! check for more words to move
1183	  nop
1184.bc_med_word2:
1185	lduw	[%o0], %o3		! read word
1186	subcc	%o2, 4, %o2		! reduce count by 4
1187	stw	%o3, [%o1]		! write word
1188	add	%o0, 4, %o0		! advance SRC by 4
1189	bgt,pt	%ncc, .bc_med_word2	! loop til 3 or fewer bytes left
1190	  add	%o1, 4, %o1		! advance DST by 4
1191.bc_med_wextra:
1192	addcc	%o2, 3, %o2		! restore rest of count
1193	bz,pt	%ncc, .bc_sm_exit	! if zero, then done
1194	  deccc	%o2
1195	bz,pt	%ncc, .bc_sm_byte
1196	  nop
1197	ba,pt	%ncc, .bc_sm_half
1198	  nop
1199
1200	.align 16
1201.bc_med_half:
1202	btst	1, %o0			! check for
1203	bz,pt	%ncc, .bc_med_half1	! half word alignment
1204	  nop
1205	ldub	[%o0], %o3		! load one byte
1206	inc	%o0
1207	stb	%o3,[%o1]		! store byte
1208	inc	%o1
1209	dec	%o2
1210!
1211!  Now half word aligned and have at least 38 bytes to move
1212!
1213.bc_med_half1:
1214	sub	%o2, 7, %o2		! adjust count to allow cc zero test
1215.bc_med_hmove:
1216	lduh	[%o0], %o3		! read half word
1217	sth	%o3, [%o1]		! write half word
1218	subcc	%o2, 8, %o2		! reduce count by 8
1219	lduh	[%o0 + 2], %o3		! repeat for a total for 4 halfwords
1220	add	%o0, 8, %o0		! advance SRC by 8
1221	sth	%o3, [%o1 + 2]
1222	lduh	[%o0 - 4], %o3
1223	add	%o1, 8, %o1		! advance DST by 8
1224	sth	%o3, [%o1 - 4]
1225	lduh	[%o0 - 2], %o3
1226	bgt,pt	%ncc, .bc_med_hmove	! loop til 7 or fewer bytes left
1227	  sth	%o3, [%o1 - 2]
1228	addcc	%o2, 7, %o2		! restore count
1229	bz,pt	%ncc, .bc_sm_exit
1230	  deccc	%o2
1231	bz,pt	%ncc, .bc_sm_byte
1232	  nop
1233	ba,pt	%ncc, .bc_sm_half
1234	  nop
1235
1236	SET_SIZE(bcopy)
1237
1238/*
1239 * The _more entry points are not intended to be used directly by
1240 * any caller from outside this file.  They are provided to allow
1241 * profiling and dtrace of the portions of the copy code that uses
1242 * the floating point registers.
1243 * This entry is particularly important as DTRACE (at least as of
1244 * 4/2004) does not support leaf functions.
1245 */
1246
1247	ENTRY(bcopy_more)
1248.bcopy_more:
1249	save	%sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
1250	ldn	[THREAD_REG + T_LOFAULT], %l6	! save t_lofault
1251	tst	%l6
1252	bz,pt	%ncc, .do_copy
1253	  nop
1254	sethi	%hi(.copyerr), %o2
1255	or	%o2, %lo(.copyerr), %o2
1256	membar	#Sync				! sync error barrier
1257	stn	%o2, [THREAD_REG + T_LOFAULT]	! install new vector
1258	!
1259	! We've already captured whether t_lofault was zero on entry.
1260	! We need to mark ourselves as being from bcopy since both
1261	! kcopy and bcopy use the same code path. If TRAMP_FLAG is set
1262	! and the saved lofault was zero, we won't reset lofault on
1263	! returning.
1264	!
1265	or	%l6, TRAMP_FLAG, %l6
1266
1267/*
1268 * Copies that reach here are larger than VIS_COPY_THRESHOLD bytes
1269 * Also, use of FP registers has been tested to be enabled
1270 */
1271.do_copy:
1272	FP_NOMIGRATE(6, 7)
1273
1274	rd	%fprs, %o2		! check for unused fp
1275	st	%o2, [%fp + STACK_BIAS - SAVED_FPRS_OFFSET] ! save orig %fprs
1276	btst	FPRS_FEF, %o2
1277	bz,a,pt	%icc, .do_blockcopy
1278	  wr	%g0, FPRS_FEF, %fprs
1279
1280	BST_FPQ1Q3_TOSTACK(%o2)
1281
1282.do_blockcopy:
1283	rd	%gsr, %o2
1284	stx	%o2, [%fp + STACK_BIAS - SAVED_GSR_OFFSET]	! save gsr
1285	or	%l6, FPUSED_FLAG, %l6
1286
1287#define	REALSRC	%i0
1288#define	DST	%i1
1289#define	CNT	%i2
1290#define	SRC	%i3
1291#define	TMP	%i5
1292
1293	andcc	DST, VIS_BLOCKSIZE - 1, TMP
1294	bz,pt	%ncc, 2f
1295	  neg	TMP
1296	add	TMP, VIS_BLOCKSIZE, TMP
1297
1298	! TMP = bytes required to align DST on FP_BLOCK boundary
1299	! Using SRC as a tmp here
1300	cmp	TMP, 3
1301	bleu,pt	%ncc, 1f
1302	  sub	CNT,TMP,CNT		! adjust main count
1303	sub	TMP, 3, TMP		! adjust for end of loop test
1304.bc_blkalign:
1305	ldub	[REALSRC], SRC		! move 4 bytes per loop iteration
1306	stb	SRC, [DST]
1307	subcc	TMP, 4, TMP
1308	ldub	[REALSRC + 1], SRC
1309	add	REALSRC, 4, REALSRC
1310	stb	SRC, [DST + 1]
1311	ldub	[REALSRC - 2], SRC
1312	add	DST, 4, DST
1313	stb	SRC, [DST - 2]
1314	ldub	[REALSRC - 1], SRC
1315	bgu,pt	%ncc, .bc_blkalign
1316	  stb	SRC, [DST - 1]
1317
1318	addcc	TMP, 3, TMP		! restore count adjustment
1319	bz,pt	%ncc, 2f		! no bytes left?
1320	  nop
13211:	ldub	[REALSRC], SRC
1322	inc	REALSRC
1323	inc	DST
1324	deccc	TMP
1325	bgu	%ncc, 1b
1326	  stb	SRC, [DST - 1]
1327
13282:
1329	andn	REALSRC, 0x7, SRC
1330	alignaddr REALSRC, %g0, %g0
1331
1332	! SRC - 8-byte aligned
1333	! DST - 64-byte aligned
1334	prefetch [SRC], #one_read
1335	prefetch [SRC + (1 * VIS_BLOCKSIZE)], #one_read
1336	prefetch [SRC + (2 * VIS_BLOCKSIZE)], #one_read
1337	prefetch [SRC + (3 * VIS_BLOCKSIZE)], #one_read
1338	ldd	[SRC], %f0
1339#if CHEETAH_PREFETCH > 4
1340	prefetch [SRC + (4 * VIS_BLOCKSIZE)], #one_read
1341#endif
1342	ldd	[SRC + 0x08], %f2
1343#if CHEETAH_PREFETCH > 5
1344	prefetch [SRC + (5 * VIS_BLOCKSIZE)], #one_read
1345#endif
1346	ldd	[SRC + 0x10], %f4
1347#if CHEETAH_PREFETCH > 6
1348	prefetch [SRC + (6 * VIS_BLOCKSIZE)], #one_read
1349#endif
1350	faligndata %f0, %f2, %f32
1351	ldd	[SRC + 0x18], %f6
1352#if CHEETAH_PREFETCH > 7
1353	prefetch [SRC + (7 * VIS_BLOCKSIZE)], #one_read
1354#endif
1355	faligndata %f2, %f4, %f34
1356	ldd	[SRC + 0x20], %f8
1357	faligndata %f4, %f6, %f36
1358	ldd	[SRC + 0x28], %f10
1359	faligndata %f6, %f8, %f38
1360	ldd	[SRC + 0x30], %f12
1361	faligndata %f8, %f10, %f40
1362	ldd	[SRC + 0x38], %f14
1363	faligndata %f10, %f12, %f42
1364	ldd	[SRC + VIS_BLOCKSIZE], %f0
1365	sub	CNT, VIS_BLOCKSIZE, CNT
1366	add	SRC, VIS_BLOCKSIZE, SRC
1367	add	REALSRC, VIS_BLOCKSIZE, REALSRC
1368	ba,a,pt	%ncc, 1f
1369	  nop
1370	.align	16
13711:
1372	ldd	[SRC + 0x08], %f2
1373	faligndata %f12, %f14, %f44
1374	ldd	[SRC + 0x10], %f4
1375	faligndata %f14, %f0, %f46
1376	stda	%f32, [DST]ASI_BLK_P
1377	ldd	[SRC + 0x18], %f6
1378	faligndata %f0, %f2, %f32
1379	ldd	[SRC + 0x20], %f8
1380	faligndata %f2, %f4, %f34
1381	ldd	[SRC + 0x28], %f10
1382	faligndata %f4, %f6, %f36
1383	ldd	[SRC + 0x30], %f12
1384	faligndata %f6, %f8, %f38
1385	ldd	[SRC + 0x38], %f14
1386	faligndata %f8, %f10, %f40
1387	sub	CNT, VIS_BLOCKSIZE, CNT
1388	ldd	[SRC + VIS_BLOCKSIZE], %f0
1389	faligndata %f10, %f12, %f42
1390	prefetch [SRC + ((CHEETAH_PREFETCH) * VIS_BLOCKSIZE) + 8], #one_read
1391	add	DST, VIS_BLOCKSIZE, DST
1392	prefetch [SRC + ((CHEETAH_2ND_PREFETCH) * VIS_BLOCKSIZE)], #one_read
1393	add	REALSRC, VIS_BLOCKSIZE, REALSRC
1394	cmp	CNT, VIS_BLOCKSIZE + 8
1395	bgu,pt	%ncc, 1b
1396	  add	SRC, VIS_BLOCKSIZE, SRC
1397
1398	! only if REALSRC & 0x7 is 0
1399	cmp	CNT, VIS_BLOCKSIZE
1400	bne	%ncc, 3f
1401	  andcc	REALSRC, 0x7, %g0
1402	bz,pt	%ncc, 2f
1403	  nop
14043:
1405	faligndata %f12, %f14, %f44
1406	faligndata %f14, %f0, %f46
1407	stda	%f32, [DST]ASI_BLK_P
1408	add	DST, VIS_BLOCKSIZE, DST
1409	ba,pt	%ncc, 3f
1410	  nop
14112:
1412	ldd	[SRC + 0x08], %f2
1413	fsrc1	%f12, %f44
1414	ldd	[SRC + 0x10], %f4
1415	fsrc1	%f14, %f46
1416	stda	%f32, [DST]ASI_BLK_P
1417	ldd	[SRC + 0x18], %f6
1418	fsrc1	%f0, %f32
1419	ldd	[SRC + 0x20], %f8
1420	fsrc1	%f2, %f34
1421	ldd	[SRC + 0x28], %f10
1422	fsrc1	%f4, %f36
1423	ldd	[SRC + 0x30], %f12
1424	fsrc1	%f6, %f38
1425	ldd	[SRC + 0x38], %f14
1426	fsrc1	%f8, %f40
1427	sub	CNT, VIS_BLOCKSIZE, CNT
1428	add	DST, VIS_BLOCKSIZE, DST
1429	add	SRC, VIS_BLOCKSIZE, SRC
1430	add	REALSRC, VIS_BLOCKSIZE, REALSRC
1431	fsrc1	%f10, %f42
1432	fsrc1	%f12, %f44
1433	fsrc1	%f14, %f46
1434	stda	%f32, [DST]ASI_BLK_P
1435	add	DST, VIS_BLOCKSIZE, DST
1436	ba,a,pt	%ncc, .bcb_exit
1437	  nop
1438
14393:	tst	CNT
1440	bz,a,pt	%ncc, .bcb_exit
1441	  nop
1442
14435:	ldub	[REALSRC], TMP
1444	inc	REALSRC
1445	inc	DST
1446	deccc	CNT
1447	bgu	%ncc, 5b
1448	  stb	TMP, [DST - 1]
1449.bcb_exit:
1450	membar	#Sync
1451
1452	FPRAS_INTERVAL(FPRAS_BCOPY, 0, %l5, %o2, %o3, %o4, %o5, 8)
1453	FPRAS_REWRITE_TYPE2Q1(0, %l5, %o2, %o3, 8, 9)
1454	FPRAS_CHECK(FPRAS_BCOPY, %l5, 9)	! outputs lost
1455
1456	ldx	[%fp + STACK_BIAS - SAVED_GSR_OFFSET], %o2	! restore gsr
1457	wr	%o2, 0, %gsr
1458
1459	ld	[%fp + STACK_BIAS - SAVED_FPRS_OFFSET], %o3
1460	btst	FPRS_FEF, %o3
1461	bz,pt	%icc, 4f
1462	  nop
1463
1464	BLD_FPQ1Q3_FROMSTACK(%o2)
1465
1466	ba,pt	%ncc, 2f
1467	  wr	%o3, 0, %fprs		! restore fprs
14684:
1469	FZEROQ1Q3
1470	wr	%o3, 0, %fprs		! restore fprs
14712:
1472	membar	#Sync				! sync error barrier
1473	andn	%l6, MASK_FLAGS, %l6
1474	stn	%l6, [THREAD_REG + T_LOFAULT]	! restore old t_lofault
1475	FP_ALLOWMIGRATE(5, 6)
1476	ret
1477	  restore	%g0, 0, %o0
1478
1479	SET_SIZE(bcopy_more)
1480
1481/*
1482 * Block copy with possibly overlapped operands.
1483 */
1484
1485	ENTRY(ovbcopy)
1486	tst	%o2			! check count
1487	bgu,a	%ncc, 1f		! nothing to do or bad arguments
1488	  subcc	%o0, %o1, %o3		! difference of from and to address
1489
1490	retl				! return
1491	  nop
14921:
1493	bneg,a	%ncc, 2f
1494	  neg	%o3			! if < 0, make it positive
14952:	cmp	%o2, %o3		! cmp size and abs(from - to)
1496	bleu	%ncc, bcopy		! if size <= abs(diff): use bcopy,
1497	  .empty				!   no overlap
1498	  cmp	%o0, %o1		! compare from and to addresses
1499	blu	%ncc, .ov_bkwd		! if from < to, copy backwards
1500	  nop
1501	!
1502	! Copy forwards.
1503	!
1504.ov_fwd:
1505	ldub	[%o0], %o3		! read from address
1506	inc	%o0			! inc from address
1507	stb	%o3, [%o1]		! write to address
1508	deccc	%o2			! dec count
1509	bgu	%ncc, .ov_fwd		! loop till done
1510	  inc	%o1			! inc to address
1511
1512	retl				! return
1513	  nop
1514	!
1515	! Copy backwards.
1516	!
1517.ov_bkwd:
1518	deccc	%o2			! dec count
1519	ldub	[%o0 + %o2], %o3	! get byte at end of src
1520	bgu	%ncc, .ov_bkwd		! loop till done
1521	  stb	%o3, [%o1 + %o2]	! delay slot, store at end of dst
1522
1523	retl				! return
1524	  nop
1525
1526	SET_SIZE(ovbcopy)
1527
1528
1529/*
1530 * hwblkpagecopy()
1531 *
1532 * Copies exactly one page.  This routine assumes the caller (ppcopy)
1533 * has already disabled kernel preemption and has checked
1534 * use_hw_bcopy.  Preventing preemption also prevents cpu migration.
1535 */
1536	ENTRY(hwblkpagecopy)
1537	! get another window w/space for three aligned blocks of saved fpregs
1538	save	%sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
1539
1540	! %i0 - source address (arg)
1541	! %i1 - destination address (arg)
1542	! %i2 - length of region (not arg)
1543	! %l0 - saved fprs
1544	! %l1 - pointer to saved fpregs
1545
1546	rd	%fprs, %l0		! check for unused fp
1547	btst	FPRS_FEF, %l0
1548	bz,a,pt	%icc, 1f
1549	  wr	%g0, FPRS_FEF, %fprs
1550
1551	BST_FPQ1Q3_TOSTACK(%l1)
1552
15531:	set	PAGESIZE, CNT
1554	mov	REALSRC, SRC
1555
1556	prefetch [SRC], #one_read
1557	prefetch [SRC + (1 * VIS_BLOCKSIZE)], #one_read
1558	prefetch [SRC + (2 * VIS_BLOCKSIZE)], #one_read
1559	prefetch [SRC + (3 * VIS_BLOCKSIZE)], #one_read
1560	ldd	[SRC], %f0
1561#if CHEETAH_PREFETCH > 4
1562	prefetch [SRC + (4 * VIS_BLOCKSIZE)], #one_read
1563#endif
1564	ldd	[SRC + 0x08], %f2
1565#if CHEETAH_PREFETCH > 5
1566	prefetch [SRC + (5 * VIS_BLOCKSIZE)], #one_read
1567#endif
1568	ldd	[SRC + 0x10], %f4
1569#if CHEETAH_PREFETCH > 6
1570	prefetch [SRC + (6 * VIS_BLOCKSIZE)], #one_read
1571#endif
1572	fsrc1	%f0, %f32
1573	ldd	[SRC + 0x18], %f6
1574#if CHEETAH_PREFETCH > 7
1575	prefetch [SRC + (7 * VIS_BLOCKSIZE)], #one_read
1576#endif
1577	fsrc1	%f2, %f34
1578	ldd	[SRC + 0x20], %f8
1579	fsrc1	%f4, %f36
1580	ldd	[SRC + 0x28], %f10
1581	fsrc1	%f6, %f38
1582	ldd	[SRC + 0x30], %f12
1583	fsrc1	%f8, %f40
1584	ldd	[SRC + 0x38], %f14
1585	fsrc1	%f10, %f42
1586	ldd	[SRC + VIS_BLOCKSIZE], %f0
1587	sub	CNT, VIS_BLOCKSIZE, CNT
1588	add	SRC, VIS_BLOCKSIZE, SRC
1589	ba,a,pt	%ncc, 2f
1590	  nop
1591	.align	16
15922:
1593	ldd	[SRC + 0x08], %f2
1594	fsrc1	%f12, %f44
1595	ldd	[SRC + 0x10], %f4
1596	fsrc1	%f14, %f46
1597	stda	%f32, [DST]ASI_BLK_P
1598	ldd	[SRC + 0x18], %f6
1599	fsrc1	%f0, %f32
1600	ldd	[SRC + 0x20], %f8
1601	fsrc1	%f2, %f34
1602	ldd	[SRC + 0x28], %f10
1603	fsrc1	%f4, %f36
1604	ldd	[SRC + 0x30], %f12
1605	fsrc1	%f6, %f38
1606	ldd	[SRC + 0x38], %f14
1607	fsrc1	%f8, %f40
1608	ldd	[SRC + VIS_BLOCKSIZE], %f0
1609	fsrc1	%f10, %f42
1610	prefetch [SRC + ((CHEETAH_PREFETCH) * VIS_BLOCKSIZE) + 8], #one_read
1611	sub	CNT, VIS_BLOCKSIZE, CNT
1612	add	DST, VIS_BLOCKSIZE, DST
1613	cmp	CNT, VIS_BLOCKSIZE + 8
1614	prefetch [SRC + ((CHEETAH_2ND_PREFETCH) * VIS_BLOCKSIZE)], #one_read
1615	bgu,pt	%ncc, 2b
1616	  add	SRC, VIS_BLOCKSIZE, SRC
1617
1618	! trailing block
1619	ldd	[SRC + 0x08], %f2
1620	fsrc1	%f12, %f44
1621	ldd	[SRC + 0x10], %f4
1622	fsrc1	%f14, %f46
1623	stda	%f32, [DST]ASI_BLK_P
1624	ldd	[SRC + 0x18], %f6
1625	fsrc1	%f0, %f32
1626	ldd	[SRC + 0x20], %f8
1627	fsrc1	%f2, %f34
1628	ldd	[SRC + 0x28], %f10
1629	fsrc1	%f4, %f36
1630	ldd	[SRC + 0x30], %f12
1631	fsrc1	%f6, %f38
1632	ldd	[SRC + 0x38], %f14
1633	fsrc1	%f8, %f40
1634	sub	CNT, VIS_BLOCKSIZE, CNT
1635	add	DST, VIS_BLOCKSIZE, DST
1636	add	SRC, VIS_BLOCKSIZE, SRC
1637	fsrc1	%f10, %f42
1638	fsrc1	%f12, %f44
1639	fsrc1	%f14, %f46
1640	stda	%f32, [DST]ASI_BLK_P
1641
1642	membar	#Sync
1643
1644	FPRAS_INTERVAL(FPRAS_PGCOPY, 1, %l5, %o2, %o3, %o4, %o5, 8)
1645	FPRAS_REWRITE_TYPE1(1, %l5, %f32, %o2, 9)
1646	FPRAS_CHECK(FPRAS_PGCOPY, %l5, 9)	! lose outputs
1647
1648	btst	FPRS_FEF, %l0
1649	bz,pt	%icc, 2f
1650	  nop
1651
1652	BLD_FPQ1Q3_FROMSTACK(%l3)
1653	ba	3f
1654	  nop
1655
16562:	FZEROQ1Q3
1657
16583:	wr	%l0, 0, %fprs		! restore fprs
1659	ret
1660	  restore	%g0, 0, %o0
1661
1662	SET_SIZE(hwblkpagecopy)
1663
1664
1665/*
1666 * Transfer data to and from user space -
1667 * Note that these routines can cause faults
1668 * It is assumed that the kernel has nothing at
1669 * less than KERNELBASE in the virtual address space.
1670 *
1671 * Note that copyin(9F) and copyout(9F) are part of the
1672 * DDI/DKI which specifies that they return '-1' on "errors."
1673 *
1674 * Sigh.
1675 *
1676 * So there's two extremely similar routines - xcopyin() and xcopyout()
1677 * which return the errno that we've faithfully computed.  This
1678 * allows other callers (e.g. uiomove(9F)) to work correctly.
1679 * Given that these are used pretty heavily, we expand the calling
1680 * sequences inline for all flavours (rather than making wrappers).
1681 *
1682 * There are also stub routines for xcopyout_little and xcopyin_little,
1683 * which currently are intended to handle requests of <= 16 bytes from
1684 * do_unaligned. Future enhancement to make them handle 8k pages efficiently
1685 * is left as an exercise...
1686 */
1687
1688/*
1689 * Copy user data to kernel space (copyOP/xcopyOP/copyOP_noerr)
1690 *
1691 * General theory of operation:
1692 *
1693 * The only difference between copy{in,out} and
1694 * xcopy{in,out} is in the error handling routine they invoke
1695 * when a memory access error occurs. xcopyOP returns the errno
1696 * while copyOP returns -1 (see above). copy{in,out}_noerr set
1697 * a special flag (by oring the TRAMP_FLAG into the fault handler address)
1698 * if they are called with a fault handler already in place. That flag
1699 * causes the default handlers to trampoline to the previous handler
1700 * upon an error.
1701 *
1702 * None of the copyops routines grab a window until it's decided that
1703 * we need to do a HW block copy operation. This saves a window
1704 * spill/fill when we're called during socket ops. The typical IO
1705 * path won't cause spill/fill traps.
1706 *
1707 * This code uses a set of 4 limits for the maximum size that will
1708 * be copied given a particular input/output address alignment.
1709 * If the value for a particular limit is zero, the copy will be performed
1710 * by the plain copy loops rather than FPBLK.
1711 *
1712 * See the description of bcopy above for more details of the
1713 * data copying algorithm and the default limits.
1714 *
1715 */
1716
1717/*
1718 * Copy kernel data to user space (copyout/xcopyout/xcopyout_little).
1719 */
1720
1721/*
1722 * We save the arguments in the following registers in case of a fault:
1723 *	kaddr - %l1
1724 *	uaddr - %l2
1725 *	count - %l3
1726 */
1727#define SAVE_SRC	%l1
1728#define SAVE_DST	%l2
1729#define SAVE_COUNT	%l3
1730
1731#define SM_SAVE_SRC		%g4
1732#define SM_SAVE_DST		%g5
1733#define SM_SAVE_COUNT		%o5
1734#define ERRNO		%l5
1735
1736
1737#define REAL_LOFAULT	%l4
1738/*
1739 * Generic copyio fault handler.  This is the first line of defense when a
1740 * fault occurs in (x)copyin/(x)copyout.  In order for this to function
1741 * properly, the value of the 'real' lofault handler should be in REAL_LOFAULT.
1742 * This allows us to share common code for all the flavors of the copy
1743 * operations, including the _noerr versions.
1744 *
1745 * Note that this function will restore the original input parameters before
1746 * calling REAL_LOFAULT.  So the real handler can vector to the appropriate
1747 * member of the t_copyop structure, if needed.
1748 */
1749	ENTRY(copyio_fault)
1750	membar	#Sync
1751	mov	%g1,ERRNO			! save errno in ERRNO
1752	btst	FPUSED_FLAG, %l6
1753	bz	%ncc, 1f
1754	  nop
1755
1756	ldx	[%fp + STACK_BIAS - SAVED_GSR_OFFSET], %o2
1757	wr	%o2, 0, %gsr    	! restore gsr
1758
1759	ld	[%fp + STACK_BIAS - SAVED_FPRS_OFFSET], %o3
1760	btst	FPRS_FEF, %o3
1761	bz,pt	%icc, 4f
1762	  nop
1763
1764	BLD_FPQ2Q4_FROMSTACK(%o2)
1765
1766	ba,pt	%ncc, 1f
1767	  wr	%o3, 0, %fprs   	! restore fprs
1768
17694:
1770	FZEROQ2Q4
1771	wr	%o3, 0, %fprs   	! restore fprs
1772
17731:
1774	andn	%l6, FPUSED_FLAG, %l6
1775	membar	#Sync
1776	stn	%l6, [THREAD_REG + T_LOFAULT]   ! restore old t_lofault
1777	FP_ALLOWMIGRATE(5, 6)
1778
1779	mov	SAVE_SRC, %i0
1780	mov	SAVE_DST, %i1
1781	jmp	REAL_LOFAULT
1782	  mov	SAVE_COUNT, %i2
1783
1784	SET_SIZE(copyio_fault)
1785
1786
1787	ENTRY(copyout)
1788
1789	cmp	%o2, VIS_COPY_THRESHOLD		! check for leaf rtn case
1790	bleu,pt	%ncc, .copyout_small		! go to larger cases
1791	  xor	%o0, %o1, %o3			! are src, dst alignable?
1792	btst	7, %o3				!
1793	bz,pt	%ncc, .copyout_8		! check for longword alignment
1794	  nop
1795	btst	1, %o3				!
1796	bz,pt	%ncc, .copyout_2		! check for half-word
1797	  nop
1798	sethi	%hi(hw_copy_limit_1), %o3	! Check copy limit
1799	ld	[%o3 + %lo(hw_copy_limit_1)], %o3
1800	tst	%o3
1801	bz,pn	%icc, .copyout_small		! if zero, disable HW copy
1802	  cmp	%o2, %o3			! if length <= limit
1803	bleu,pt	%ncc, .copyout_small		! go to small copy
1804	  nop
1805	ba,pt	%ncc, .copyout_more		! otherwise go to large copy
1806	  nop
1807.copyout_2:
1808	btst	3, %o3				!
1809	bz,pt	%ncc, .copyout_4		! check for word alignment
1810	  nop
1811	sethi	%hi(hw_copy_limit_2), %o3	! Check copy limit
1812	ld	[%o3 + %lo(hw_copy_limit_2)], %o3
1813	tst	%o3
1814	bz,pn	%icc, .copyout_small		! if zero, disable HW copy
1815	  cmp	%o2, %o3			! if length <= limit
1816	bleu,pt	%ncc, .copyout_small		! go to small copy
1817	  nop
1818	ba,pt	%ncc, .copyout_more		! otherwise go to large copy
1819	  nop
1820.copyout_4:
1821	! already checked longword, must be word aligned
1822	sethi	%hi(hw_copy_limit_4), %o3	! Check copy limit
1823	ld	[%o3 + %lo(hw_copy_limit_4)], %o3
1824	tst	%o3
1825	bz,pn	%icc, .copyout_small		! if zero, disable HW copy
1826	  cmp	%o2, %o3			! if length <= limit
1827	bleu,pt	%ncc, .copyout_small		! go to small copy
1828	  nop
1829	ba,pt	%ncc, .copyout_more		! otherwise go to large copy
1830	  nop
1831.copyout_8:
1832	sethi	%hi(hw_copy_limit_8), %o3	! Check copy limit
1833	ld	[%o3 + %lo(hw_copy_limit_8)], %o3
1834	tst	%o3
1835	bz,pn	%icc, .copyout_small		! if zero, disable HW copy
1836	  cmp	%o2, %o3			! if length <= limit
1837	bleu,pt	%ncc, .copyout_small		! go to small copy
1838	  nop
1839	ba,pt	%ncc, .copyout_more		! otherwise go to large copy
1840	  nop
1841
1842	.align	16
1843	nop				! instruction alignment
1844					! see discussion at start of file
1845.copyout_small:
1846	sethi	%hi(.sm_copyout_err), %o5	! .sm_copyout_err is lofault
1847	or	%o5, %lo(.sm_copyout_err), %o5
1848	ldn	[THREAD_REG + T_LOFAULT], %o4	! save existing handler
1849	membar	#Sync				! sync error barrier
1850	stn	%o5, [THREAD_REG + T_LOFAULT]	! set t_lofault
1851.sm_do_copyout:
1852	mov	%o0, SM_SAVE_SRC
1853	mov	%o1, SM_SAVE_DST
1854	cmp	%o2, SHORTCOPY		! check for really short case
1855	bleu,pt	%ncc, .co_sm_left	!
1856	  mov	%o2, SM_SAVE_COUNT
1857	cmp	%o2, CHKSIZE		! check for medium length cases
1858	bgu,pn	%ncc, .co_med		!
1859	  or	%o0, %o1, %o3		! prepare alignment check
1860	andcc	%o3, 0x3, %g0		! test for alignment
1861	bz,pt	%ncc, .co_sm_word	! branch to word aligned case
1862.co_sm_movebytes:
1863	  sub	%o2, 3, %o2		! adjust count to allow cc zero test
1864.co_sm_notalign4:
1865	ldub	[%o0], %o3		! read byte
1866	subcc	%o2, 4, %o2		! reduce count by 4
1867	stba	%o3, [%o1]ASI_USER	! write byte
1868	inc	%o1			! advance DST by 1
1869	ldub	[%o0 + 1], %o3		! repeat for a total of 4 bytes
1870	add	%o0, 4, %o0		! advance SRC by 4
1871	stba	%o3, [%o1]ASI_USER
1872	inc	%o1			! advance DST by 1
1873	ldub	[%o0 - 2], %o3
1874	stba	%o3, [%o1]ASI_USER
1875	inc	%o1			! advance DST by 1
1876	ldub	[%o0 - 1], %o3
1877	stba	%o3, [%o1]ASI_USER
1878	bgt,pt	%ncc, .co_sm_notalign4	! loop til 3 or fewer bytes remain
1879	  inc	%o1			! advance DST by 1
1880	add	%o2, 3, %o2		! restore count
1881.co_sm_left:
1882	tst	%o2
1883	bz,pt	%ncc, .co_sm_exit	! check for zero length
1884	  nop
1885	ldub	[%o0], %o3		! load one byte
1886	deccc	%o2			! reduce count for cc test
1887	bz,pt	%ncc, .co_sm_exit
1888	  stba	%o3,[%o1]ASI_USER	! store one byte
1889	ldub	[%o0 + 1], %o3		! load second byte
1890	deccc	%o2
1891	inc	%o1
1892	bz,pt	%ncc, .co_sm_exit
1893	  stba	%o3,[%o1]ASI_USER	! store second byte
1894	ldub	[%o0 + 2], %o3		! load third byte
1895	inc	%o1
1896	stba	%o3,[%o1]ASI_USER	! store third byte
1897	membar	#Sync				! sync error barrier
1898	stn	%o4, [THREAD_REG + T_LOFAULT]	! restore old t_lofault
1899	retl
1900	  mov	%g0, %o0		! return 0
1901	.align	16
1902.co_sm_words:
1903	lduw	[%o0], %o3		! read word
1904.co_sm_wordx:
1905	subcc	%o2, 8, %o2		! update count
1906	stwa	%o3, [%o1]ASI_USER	! write word
1907	add	%o0, 8, %o0		! update SRC
1908	lduw	[%o0 - 4], %o3		! read word
1909	add	%o1, 4, %o1		! update DST
1910	stwa	%o3, [%o1]ASI_USER	! write word
1911	bgt,pt	%ncc, .co_sm_words	! loop til done
1912	  add	%o1, 4, %o1		! update DST
1913	addcc	%o2, 7, %o2		! restore count
1914	bz,pt	%ncc, .co_sm_exit
1915	  nop
1916	deccc	%o2
1917	bz,pt	%ncc, .co_sm_byte
1918.co_sm_half:
1919	  subcc	%o2, 2, %o2		! reduce count by 2
1920	lduh	[%o0], %o3		! read half word
1921	add	%o0, 2, %o0		! advance SRC by 2
1922	stha	%o3, [%o1]ASI_USER	! write half word
1923	bgt,pt	%ncc, .co_sm_half	! loop til done
1924	  add	%o1, 2, %o1		! advance DST by 2
1925	addcc	%o2, 1, %o2		! restore count
1926	bz,pt	%ncc, .co_sm_exit
1927	  nop
1928.co_sm_byte:
1929	ldub	[%o0], %o3
1930	stba	%o3, [%o1]ASI_USER
1931	membar	#Sync				! sync error barrier
1932	stn	%o4, [THREAD_REG + T_LOFAULT]	! restore old t_lofault
1933	retl
1934	  mov	%g0, %o0		! return 0
1935	.align 16
1936.co_sm_word:
1937	subcc	%o2, 4, %o2		! update count
1938	bgt,pt	%ncc, .co_sm_wordx
1939	  lduw	[%o0], %o3		! read word
1940	addcc	%o2, 3, %o2		! restore count
1941	bz,pt	%ncc, .co_sm_exit
1942	  stwa	%o3, [%o1]ASI_USER	! write word
1943	deccc	%o2			! reduce count for cc test
1944	ldub	[%o0 + 4], %o3		! load one byte
1945	add	%o1, 4, %o1
1946	bz,pt	%ncc, .co_sm_exit
1947	  stba	%o3, [%o1]ASI_USER	! store one byte
1948	ldub	[%o0 + 5], %o3		! load second byte
1949	deccc	%o2
1950	inc	%o1
1951	bz,pt	%ncc, .co_sm_exit
1952	  stba	%o3, [%o1]ASI_USER	! store second byte
1953	ldub	[%o0 + 6], %o3		! load third byte
1954	inc	%o1
1955	stba	%o3, [%o1]ASI_USER	! store third byte
1956.co_sm_exit:
1957	  membar	#Sync				! sync error barrier
1958	stn	%o4, [THREAD_REG + T_LOFAULT]	! restore old t_lofault
1959	retl
1960	  mov	%g0, %o0		! return 0
1961
1962	.align 16
1963.co_med:
1964	xor	%o0, %o1, %o3		! setup alignment check
1965	btst	1, %o3
1966	bnz,pt	%ncc, .co_sm_movebytes	! unaligned
1967	  nop
1968	btst	3, %o3
1969	bnz,pt	%ncc, .co_med_half	! halfword aligned
1970	  nop
1971	btst	7, %o3
1972	bnz,pt	%ncc, .co_med_word	! word aligned
1973	  nop
1974.co_med_long:
1975	btst	3, %o0			! check for
1976	bz,pt	%ncc, .co_med_long1	! word alignment
1977	  nop
1978.co_med_long0:
1979	ldub	[%o0], %o3		! load one byte
1980	inc	%o0
1981	stba	%o3,[%o1]ASI_USER	! store byte
1982	inc	%o1
1983	btst	3, %o0
1984	bnz,pt	%ncc, .co_med_long0
1985	  dec	%o2
1986.co_med_long1:			! word aligned
1987	btst	7, %o0			! check for long word
1988	bz,pt	%ncc, .co_med_long2
1989	  nop
1990	lduw	[%o0], %o3		! load word
1991	add	%o0, 4, %o0		! advance SRC by 4
1992	stwa	%o3, [%o1]ASI_USER	! store word
1993	add	%o1, 4, %o1		! advance DST by 4
1994	sub	%o2, 4, %o2		! reduce count by 4
1995!
1996!  Now long word aligned and have at least 32 bytes to move
1997!
1998.co_med_long2:
1999	sub	%o2, 31, %o2		! adjust count to allow cc zero test
2000	sub	%o1, 8, %o1		! adjust pointer to allow store in
2001					! branch delay slot instead of add
2002.co_med_lmove:
2003	add	%o1, 8, %o1		! advance DST by 8
2004	ldx	[%o0], %o3		! read long word
2005	subcc	%o2, 32, %o2		! reduce count by 32
2006	stxa	%o3, [%o1]ASI_USER	! write long word
2007	add	%o1, 8, %o1		! advance DST by 8
2008	ldx	[%o0 + 8], %o3		! repeat for a total for 4 long words
2009	add	%o0, 32, %o0		! advance SRC by 32
2010	stxa	%o3, [%o1]ASI_USER
2011	ldx	[%o0 - 16], %o3
2012	add	%o1, 8, %o1		! advance DST by 8
2013	stxa	%o3, [%o1]ASI_USER
2014	ldx	[%o0 - 8], %o3
2015	add	%o1, 8, %o1		! advance DST by 8
2016	bgt,pt	%ncc, .co_med_lmove	! loop til 31 or fewer bytes left
2017	  stxa	%o3, [%o1]ASI_USER
2018	add	%o1, 8, %o1		! advance DST by 8
2019	addcc	%o2, 24, %o2		! restore count to long word offset
2020	ble,pt	%ncc, .co_med_lextra	! check for more long words to move
2021	  nop
2022.co_med_lword:
2023	ldx	[%o0], %o3		! read long word
2024	subcc	%o2, 8, %o2		! reduce count by 8
2025	stxa	%o3, [%o1]ASI_USER	! write long word
2026	add	%o0, 8, %o0		! advance SRC by 8
2027	bgt,pt	%ncc, .co_med_lword	! loop til 7 or fewer bytes left
2028	  add	%o1, 8, %o1		! advance DST by 8
2029.co_med_lextra:
2030	addcc	%o2, 7, %o2		! restore rest of count
2031	bz,pt	%ncc, .co_sm_exit	! if zero, then done
2032	  deccc	%o2
2033	bz,pt	%ncc, .co_sm_byte
2034	  nop
2035	ba,pt	%ncc, .co_sm_half
2036	  nop
2037
2038	.align 16
2039	nop				! instruction alignment
2040					! see discussion at start of file
2041.co_med_word:
2042	btst	3, %o0			! check for
2043	bz,pt	%ncc, .co_med_word1	! word alignment
2044	  nop
2045.co_med_word0:
2046	ldub	[%o0], %o3		! load one byte
2047	inc	%o0
2048	stba	%o3,[%o1]ASI_USER	! store byte
2049	inc	%o1
2050	btst	3, %o0
2051	bnz,pt	%ncc, .co_med_word0
2052	  dec	%o2
2053!
2054!  Now word aligned and have at least 36 bytes to move
2055!
2056.co_med_word1:
2057	sub	%o2, 15, %o2		! adjust count to allow cc zero test
2058.co_med_wmove:
2059	lduw	[%o0], %o3		! read word
2060	subcc	%o2, 16, %o2		! reduce count by 16
2061	stwa	%o3, [%o1]ASI_USER	! write word
2062	add	%o1, 4, %o1		! advance DST by 4
2063	lduw	[%o0 + 4], %o3		! repeat for a total for 4 words
2064	add	%o0, 16, %o0		! advance SRC by 16
2065	stwa	%o3, [%o1]ASI_USER
2066	add	%o1, 4, %o1		! advance DST by 4
2067	lduw	[%o0 - 8], %o3
2068	stwa	%o3, [%o1]ASI_USER
2069	add	%o1, 4, %o1		! advance DST by 4
2070	lduw	[%o0 - 4], %o3
2071	stwa	%o3, [%o1]ASI_USER
2072	bgt,pt	%ncc, .co_med_wmove	! loop til 15 or fewer bytes left
2073	  add	%o1, 4, %o1		! advance DST by 4
2074	addcc	%o2, 12, %o2		! restore count to word offset
2075	ble,pt	%ncc, .co_med_wextra	! check for more words to move
2076	  nop
2077.co_med_word2:
2078	lduw	[%o0], %o3		! read word
2079	subcc	%o2, 4, %o2		! reduce count by 4
2080	stwa	%o3, [%o1]ASI_USER	! write word
2081	add	%o0, 4, %o0		! advance SRC by 4
2082	bgt,pt	%ncc, .co_med_word2	! loop til 3 or fewer bytes left
2083	  add	%o1, 4, %o1		! advance DST by 4
2084.co_med_wextra:
2085	addcc	%o2, 3, %o2		! restore rest of count
2086	bz,pt	%ncc, .co_sm_exit	! if zero, then done
2087	  deccc	%o2
2088	bz,pt	%ncc, .co_sm_byte
2089	  nop
2090	ba,pt	%ncc, .co_sm_half
2091	  nop
2092
2093	.align 16
2094	nop				! instruction alignment
2095	nop				! see discussion at start of file
2096	nop
2097.co_med_half:
2098	btst	1, %o0			! check for
2099	bz,pt	%ncc, .co_med_half1	! half word alignment
2100	  nop
2101	ldub	[%o0], %o3		! load one byte
2102	inc	%o0
2103	stba	%o3,[%o1]ASI_USER	! store byte
2104	inc	%o1
2105	dec	%o2
2106!
2107!  Now half word aligned and have at least 38 bytes to move
2108!
2109.co_med_half1:
2110	sub	%o2, 7, %o2		! adjust count to allow cc zero test
2111.co_med_hmove:
2112	lduh	[%o0], %o3		! read half word
2113	subcc	%o2, 8, %o2		! reduce count by 8
2114	stha	%o3, [%o1]ASI_USER	! write half word
2115	add	%o1, 2, %o1		! advance DST by 2
2116	lduh	[%o0 + 2], %o3		! repeat for a total for 4 halfwords
2117	add	%o0, 8, %o0		! advance SRC by 8
2118	stha	%o3, [%o1]ASI_USER
2119	add	%o1, 2, %o1		! advance DST by 2
2120	lduh	[%o0 - 4], %o3
2121	stha	%o3, [%o1]ASI_USER
2122	add	%o1, 2, %o1		! advance DST by 2
2123	lduh	[%o0 - 2], %o3
2124	stha	%o3, [%o1]ASI_USER
2125	bgt,pt	%ncc, .co_med_hmove	! loop til 7 or fewer bytes left
2126	  add	%o1, 2, %o1		! advance DST by 2
2127	addcc	%o2, 7, %o2		! restore count
2128	bz,pt	%ncc, .co_sm_exit
2129	  deccc	%o2
2130	bz,pt	%ncc, .co_sm_byte
2131	  nop
2132	ba,pt	%ncc, .co_sm_half
2133	  nop
2134
2135/*
2136 * We got here because of a fault during short copyout.
2137 * Errno value is in ERRNO, but DDI/DKI says return -1 (sigh).
2138 */
2139.sm_copyout_err:
2140	membar	#Sync
2141	stn	%o4, [THREAD_REG + T_LOFAULT]   ! restore old t_lofault
2142	mov	SM_SAVE_SRC, %o0
2143	mov	SM_SAVE_DST, %o1
2144	mov	SM_SAVE_COUNT, %o2
2145	ldn	[THREAD_REG + T_COPYOPS], %o3	! check for copyop handler
2146	tst	%o3
2147	bz,pt	%ncc, 3f			! if not, return error
2148	  nop
2149	ldn	[%o3 + CP_COPYOUT], %o5		! if handler, invoke it with
2150	jmp	%o5				! original arguments
2151	  nop
21523:
2153	retl
2154	  or	%g0, -1, %o0		! return error value
2155
2156	SET_SIZE(copyout)
2157
2158/*
2159 * The _more entry points are not intended to be used directly by
2160 * any caller from outside this file.  They are provided to allow
2161 * profiling and dtrace of the portions of the copy code that uses
2162 * the floating point registers.
2163 * This entry is particularly important as DTRACE (at least as of
2164 * 4/2004) does not support leaf functions.
2165 */
2166
2167	ENTRY(copyout_more)
2168.copyout_more:
2169	save	%sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
2170	set	.copyout_err, REAL_LOFAULT
2171
2172/*
2173 * Copy outs that reach here are larger than VIS_COPY_THRESHOLD bytes
2174 */
2175.do_copyout:
2176        set     copyio_fault, %l7		! .copyio_fault is lofault val
2177
2178	ldn	[THREAD_REG + T_LOFAULT], %l6	! save existing handler
2179	membar	#Sync				! sync error barrier
2180	stn	%l7, [THREAD_REG + T_LOFAULT]	! set t_lofault
2181
2182	mov	%i0, SAVE_SRC
2183	mov	%i1, SAVE_DST
2184	mov	%i2, SAVE_COUNT
2185
2186	FP_NOMIGRATE(6, 7)
2187
2188	rd	%fprs, %o2		! check for unused fp
2189	st	%o2, [%fp + STACK_BIAS - SAVED_FPRS_OFFSET] ! save orig %fprs
2190	btst	FPRS_FEF, %o2
2191	bz,a,pt	%icc, .do_blockcopyout
2192	  wr	%g0, FPRS_FEF, %fprs
2193
2194	BST_FPQ2Q4_TOSTACK(%o2)
2195
2196.do_blockcopyout:
2197	rd	%gsr, %o2
2198	stx	%o2, [%fp + STACK_BIAS - SAVED_GSR_OFFSET]	! save gsr
2199	or	%l6, FPUSED_FLAG, %l6
2200
2201	andcc	DST, VIS_BLOCKSIZE - 1, TMP
2202	mov	ASI_USER, %asi
2203	bz,pt	%ncc, 2f
2204	  neg	TMP
2205	add	TMP, VIS_BLOCKSIZE, TMP
2206
2207	! TMP = bytes required to align DST on FP_BLOCK boundary
2208	! Using SRC as a tmp here
2209	cmp	TMP, 3
2210	bleu,pt	%ncc, 1f
2211	  sub	CNT,TMP,CNT		! adjust main count
2212	sub	TMP, 3, TMP		! adjust for end of loop test
2213.co_blkalign:
2214	ldub	[REALSRC], SRC		! move 4 bytes per loop iteration
2215	stba	SRC, [DST]%asi
2216	subcc	TMP, 4, TMP
2217	ldub	[REALSRC + 1], SRC
2218	add	REALSRC, 4, REALSRC
2219	stba	SRC, [DST + 1]%asi
2220	ldub	[REALSRC - 2], SRC
2221	add	DST, 4, DST
2222	stba	SRC, [DST - 2]%asi
2223	ldub	[REALSRC - 1], SRC
2224	bgu,pt	%ncc, .co_blkalign
2225	  stba	SRC, [DST - 1]%asi
2226
2227	addcc	TMP, 3, TMP		! restore count adjustment
2228	bz,pt	%ncc, 2f		! no bytes left?
2229	  nop
22301:	ldub	[REALSRC], SRC
2231	inc	REALSRC
2232	inc	DST
2233	deccc	TMP
2234	bgu	%ncc, 1b
2235	  stba	SRC, [DST - 1]%asi
2236
22372:
2238	andn	REALSRC, 0x7, SRC
2239	alignaddr REALSRC, %g0, %g0
2240
2241	! SRC - 8-byte aligned
2242	! DST - 64-byte aligned
2243	prefetch [SRC], #one_read
2244	prefetch [SRC + (1 * VIS_BLOCKSIZE)], #one_read
2245	prefetch [SRC + (2 * VIS_BLOCKSIZE)], #one_read
2246	prefetch [SRC + (3 * VIS_BLOCKSIZE)], #one_read
2247	ldd	[SRC], %f16
2248#if CHEETAH_PREFETCH > 4
2249	prefetch [SRC + (4 * VIS_BLOCKSIZE)], #one_read
2250#endif
2251	ldd	[SRC + 0x08], %f18
2252#if CHEETAH_PREFETCH > 5
2253	prefetch [SRC + (5 * VIS_BLOCKSIZE)], #one_read
2254#endif
2255	ldd	[SRC + 0x10], %f20
2256#if CHEETAH_PREFETCH > 6
2257	prefetch [SRC + (6 * VIS_BLOCKSIZE)], #one_read
2258#endif
2259	faligndata %f16, %f18, %f48
2260	ldd	[SRC + 0x18], %f22
2261#if CHEETAH_PREFETCH > 7
2262	prefetch [SRC + (7 * VIS_BLOCKSIZE)], #one_read
2263#endif
2264	faligndata %f18, %f20, %f50
2265	ldd	[SRC + 0x20], %f24
2266	faligndata %f20, %f22, %f52
2267	ldd	[SRC + 0x28], %f26
2268	faligndata %f22, %f24, %f54
2269	ldd	[SRC + 0x30], %f28
2270	faligndata %f24, %f26, %f56
2271	ldd	[SRC + 0x38], %f30
2272	faligndata %f26, %f28, %f58
2273	ldd	[SRC + VIS_BLOCKSIZE], %f16
2274	sub	CNT, VIS_BLOCKSIZE, CNT
2275	add	SRC, VIS_BLOCKSIZE, SRC
2276	add	REALSRC, VIS_BLOCKSIZE, REALSRC
2277	ba,a,pt	%ncc, 1f
2278	  nop
2279	.align	16
22801:
2281	ldd	[SRC + 0x08], %f18
2282	faligndata %f28, %f30, %f60
2283	ldd	[SRC + 0x10], %f20
2284	faligndata %f30, %f16, %f62
2285	stda	%f48, [DST]ASI_BLK_AIUS
2286	ldd	[SRC + 0x18], %f22
2287	faligndata %f16, %f18, %f48
2288	ldd	[SRC + 0x20], %f24
2289	faligndata %f18, %f20, %f50
2290	ldd	[SRC + 0x28], %f26
2291	faligndata %f20, %f22, %f52
2292	ldd	[SRC + 0x30], %f28
2293	faligndata %f22, %f24, %f54
2294	ldd	[SRC + 0x38], %f30
2295	faligndata %f24, %f26, %f56
2296	sub	CNT, VIS_BLOCKSIZE, CNT
2297	ldd	[SRC + VIS_BLOCKSIZE], %f16
2298	faligndata %f26, %f28, %f58
2299	prefetch [SRC + ((CHEETAH_PREFETCH) * VIS_BLOCKSIZE) + 8], #one_read
2300	add	DST, VIS_BLOCKSIZE, DST
2301	prefetch [SRC + ((CHEETAH_2ND_PREFETCH) * VIS_BLOCKSIZE)], #one_read
2302	add	REALSRC, VIS_BLOCKSIZE, REALSRC
2303	cmp	CNT, VIS_BLOCKSIZE + 8
2304	bgu,pt	%ncc, 1b
2305	  add	SRC, VIS_BLOCKSIZE, SRC
2306
2307	! only if REALSRC & 0x7 is 0
2308	cmp	CNT, VIS_BLOCKSIZE
2309	bne	%ncc, 3f
2310	  andcc	REALSRC, 0x7, %g0
2311	bz,pt	%ncc, 2f
2312	  nop
23133:
2314	faligndata %f28, %f30, %f60
2315	faligndata %f30, %f16, %f62
2316	stda	%f48, [DST]ASI_BLK_AIUS
2317	add	DST, VIS_BLOCKSIZE, DST
2318	ba,pt	%ncc, 3f
2319	  nop
23202:
2321	ldd	[SRC + 0x08], %f18
2322	fsrc1	%f28, %f60
2323	ldd	[SRC + 0x10], %f20
2324	fsrc1	%f30, %f62
2325	stda	%f48, [DST]ASI_BLK_AIUS
2326	ldd	[SRC + 0x18], %f22
2327	fsrc1	%f16, %f48
2328	ldd	[SRC + 0x20], %f24
2329	fsrc1	%f18, %f50
2330	ldd	[SRC + 0x28], %f26
2331	fsrc1	%f20, %f52
2332	ldd	[SRC + 0x30], %f28
2333	fsrc1	%f22, %f54
2334	ldd	[SRC + 0x38], %f30
2335	fsrc1	%f24, %f56
2336	sub	CNT, VIS_BLOCKSIZE, CNT
2337	add	DST, VIS_BLOCKSIZE, DST
2338	add	SRC, VIS_BLOCKSIZE, SRC
2339	add	REALSRC, VIS_BLOCKSIZE, REALSRC
2340	fsrc1	%f26, %f58
2341	fsrc1	%f28, %f60
2342	fsrc1	%f30, %f62
2343	stda	%f48, [DST]ASI_BLK_AIUS
2344	add	DST, VIS_BLOCKSIZE, DST
2345	ba,a,pt	%ncc, 4f
2346	  nop
2347
23483:	tst	CNT
2349	bz,a	%ncc, 4f
2350	  nop
2351
23525:	ldub	[REALSRC], TMP
2353	inc	REALSRC
2354	inc	DST
2355	deccc	CNT
2356	bgu	%ncc, 5b
2357	  stba	TMP, [DST - 1]%asi
23584:
2359
2360.copyout_exit:
2361	membar	#Sync
2362
2363	FPRAS_INTERVAL(FPRAS_COPYOUT, 0, %l5, %o2, %o3, %o4, %o5, 8)
2364	FPRAS_REWRITE_TYPE2Q2(0, %l5, %o2, %o3, 8, 9)
2365	FPRAS_CHECK(FPRAS_COPYOUT, %l5, 9)	! lose outputs
2366
2367	ldx	[%fp + STACK_BIAS - SAVED_GSR_OFFSET], %o2
2368	wr	%o2, 0, %gsr		! restore gsr
2369
2370	ld	[%fp + STACK_BIAS - SAVED_FPRS_OFFSET], %o3
2371	btst	FPRS_FEF, %o3
2372	bz,pt	%icc, 4f
2373	  nop
2374
2375	BLD_FPQ2Q4_FROMSTACK(%o2)
2376
2377	ba,pt	%ncc, 1f
2378	  wr	%o3, 0, %fprs		! restore fprs
2379
23804:
2381	FZEROQ2Q4
2382	wr	%o3, 0, %fprs		! restore fprs
2383
23841:
2385	membar	#Sync
2386	andn	%l6, FPUSED_FLAG, %l6
2387	stn	%l6, [THREAD_REG + T_LOFAULT]	! restore old t_lofault
2388	FP_ALLOWMIGRATE(5, 6)
2389	ret
2390	  restore	%g0, 0, %o0
2391
2392/*
2393 * We got here because of a fault during copyout.
2394 * Errno value is in ERRNO, but DDI/DKI says return -1 (sigh).
2395 */
2396.copyout_err:
2397	ldn	[THREAD_REG + T_COPYOPS], %o4	! check for copyop handler
2398	tst	%o4
2399	bz,pt	%ncc, 2f			! if not, return error
2400	  nop
2401	ldn	[%o4 + CP_COPYOUT], %g2		! if handler, invoke it with
2402	jmp	%g2				! original arguments
2403	  restore %g0, 0, %g0			! dispose of copy window
24042:
2405        ret
2406	  restore %g0, -1, %o0			! return error value
2407
2408
2409	SET_SIZE(copyout_more)
2410
2411
2412	ENTRY(xcopyout)
2413	cmp	%o2, VIS_COPY_THRESHOLD		! check for leaf rtn case
2414	bleu,pt	%ncc, .xcopyout_small		! go to larger cases
2415	  xor	%o0, %o1, %o3			! are src, dst alignable?
2416	btst	7, %o3				!
2417	bz,pt	%ncc, .xcopyout_8		!
2418	  nop
2419	btst	1, %o3				!
2420	bz,pt	%ncc, .xcopyout_2		! check for half-word
2421	  nop
2422	sethi	%hi(hw_copy_limit_1), %o3	! Check copy limit
2423	ld	[%o3 + %lo(hw_copy_limit_1)], %o3
2424	tst	%o3
2425	bz,pn	%icc, .xcopyout_small		! if zero, disable HW copy
2426	  cmp	%o2, %o3			! if length <= limit
2427	bleu,pt	%ncc, .xcopyout_small		! go to small copy
2428	  nop
2429	ba,pt	%ncc, .xcopyout_more		! otherwise go to large copy
2430	  nop
2431.xcopyout_2:
2432	btst	3, %o3				!
2433	bz,pt	%ncc, .xcopyout_4		! check for word alignment
2434	  nop
2435	sethi	%hi(hw_copy_limit_2), %o3	! Check copy limit
2436	ld	[%o3 + %lo(hw_copy_limit_2)], %o3
2437	tst	%o3
2438	bz,pn	%icc, .xcopyout_small		! if zero, disable HW copy
2439	  cmp	%o2, %o3			! if length <= limit
2440	bleu,pt	%ncc, .xcopyout_small		! go to small copy
2441	  nop
2442	ba,pt	%ncc, .xcopyout_more		! otherwise go to large copy
2443	  nop
2444.xcopyout_4:
2445	! already checked longword, must be word aligned
2446	sethi	%hi(hw_copy_limit_4), %o3	! Check copy limit
2447	ld	[%o3 + %lo(hw_copy_limit_4)], %o3
2448	tst	%o3
2449	bz,pn	%icc, .xcopyout_small		! if zero, disable HW copy
2450	  cmp	%o2, %o3			! if length <= limit
2451	bleu,pt	%ncc, .xcopyout_small		! go to small copy
2452	  nop
2453	ba,pt	%ncc, .xcopyout_more		! otherwise go to large copy
2454	  nop
2455.xcopyout_8:
2456	sethi	%hi(hw_copy_limit_8), %o3	! Check copy limit
2457	ld	[%o3 + %lo(hw_copy_limit_8)], %o3
2458	tst	%o3
2459	bz,pn	%icc, .xcopyout_small		! if zero, disable HW copy
2460	  cmp	%o2, %o3			! if length <= limit
2461	bleu,pt	%ncc, .xcopyout_small		! go to small copy
2462	  nop
2463	ba,pt	%ncc, .xcopyout_more		! otherwise go to large copy
2464	  nop
2465
2466.xcopyout_small:
2467	sethi	%hi(.sm_xcopyout_err), %o5	! .sm_xcopyout_err is lofault
2468	or	%o5, %lo(.sm_xcopyout_err), %o5
2469	ldn	[THREAD_REG + T_LOFAULT], %o4	! save existing handler
2470	membar	#Sync				! sync error barrier
2471	ba,pt	%ncc, .sm_do_copyout		! common code
2472	  stn	%o5, [THREAD_REG + T_LOFAULT]	! set t_lofault
2473
2474.xcopyout_more:
2475	save	%sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
2476	sethi	%hi(.xcopyout_err), REAL_LOFAULT
2477	ba,pt	%ncc, .do_copyout		! common code
2478	  or	REAL_LOFAULT, %lo(.xcopyout_err), REAL_LOFAULT
2479
2480/*
2481 * We got here because of fault during xcopyout
2482 * Errno value is in ERRNO
2483 */
2484.xcopyout_err:
2485	ldn	[THREAD_REG + T_COPYOPS], %o4	! check for copyop handler
2486	tst	%o4
2487	bz,pt	%ncc, 2f			! if not, return error
2488	  nop
2489	ldn	[%o4 + CP_XCOPYOUT], %g2	! if handler, invoke it with
2490	jmp	%g2				! original arguments
2491	  restore %g0, 0, %g0			! dispose of copy window
24922:
2493        ret
2494	  restore ERRNO, 0, %o0			! return errno value
2495
2496.sm_xcopyout_err:
2497
2498	membar	#Sync
2499	stn	%o4, [THREAD_REG + T_LOFAULT]   ! restore old t_lofault
2500	mov	SM_SAVE_SRC, %o0
2501	mov	SM_SAVE_DST, %o1
2502	mov	SM_SAVE_COUNT, %o2
2503	ldn	[THREAD_REG + T_COPYOPS], %o3	! check for copyop handler
2504	tst	%o3
2505	bz,pt	%ncc, 3f			! if not, return error
2506	  nop
2507	ldn	[%o3 + CP_XCOPYOUT], %o5	! if handler, invoke it with
2508	jmp	%o5				! original arguments
2509	  nop
25103:
2511	retl
2512	  or	%g1, 0, %o0		! return errno value
2513
2514	SET_SIZE(xcopyout)
2515
2516	ENTRY(xcopyout_little)
2517	sethi	%hi(.xcopyio_err), %o5
2518	or	%o5, %lo(.xcopyio_err), %o5
2519	ldn	[THREAD_REG + T_LOFAULT], %o4
2520	membar	#Sync				! sync error barrier
2521	stn	%o5, [THREAD_REG + T_LOFAULT]
2522	mov	%o4, %o5
2523
2524	subcc	%g0, %o2, %o3
2525	add	%o0, %o2, %o0
2526	bz,pn	%ncc, 2f		! check for zero bytes
2527	  sub	%o2, 1, %o4
2528	add	%o0, %o4, %o0		! start w/last byte
2529	add	%o1, %o2, %o1
2530	ldub	[%o0 + %o3], %o4
2531
25321:	stba	%o4, [%o1 + %o3]ASI_AIUSL
2533	inccc	%o3
2534	sub	%o0, 2, %o0		! get next byte
2535	bcc,a,pt %ncc, 1b
2536	  ldub	[%o0 + %o3], %o4
2537
25382:
2539	membar	#Sync				! sync error barrier
2540	stn	%o5, [THREAD_REG + T_LOFAULT]	! restore old t_lofault
2541	retl
2542	  mov	%g0, %o0		! return (0)
2543
2544	SET_SIZE(xcopyout_little)
2545
2546/*
2547 * Copy user data to kernel space (copyin/xcopyin/xcopyin_little)
2548 */
2549
2550	ENTRY(copyin)
2551	cmp	%o2, VIS_COPY_THRESHOLD		! check for leaf rtn case
2552	bleu,pt	%ncc, .copyin_small		! go to larger cases
2553	  xor	%o0, %o1, %o3			! are src, dst alignable?
2554	btst	7, %o3				!
2555	bz,pt	%ncc, .copyin_8			! check for longword alignment
2556	  nop
2557	btst	1, %o3				!
2558	bz,pt	%ncc, .copyin_2			! check for half-word
2559	  nop
2560	sethi	%hi(hw_copy_limit_1), %o3	! Check copy limit
2561	ld	[%o3 + %lo(hw_copy_limit_1)], %o3
2562	tst	%o3
2563	bz,pn	%icc, .copyin_small		! if zero, disable HW copy
2564	  cmp	%o2, %o3			! if length <= limit
2565	bleu,pt	%ncc, .copyin_small		! go to small copy
2566	  nop
2567	ba,pt	%ncc, .copyin_more		! otherwise go to large copy
2568	  nop
2569.copyin_2:
2570	btst	3, %o3				!
2571	bz,pt	%ncc, .copyin_4			! check for word alignment
2572	  nop
2573	sethi	%hi(hw_copy_limit_2), %o3	! Check copy limit
2574	ld	[%o3 + %lo(hw_copy_limit_2)], %o3
2575	tst	%o3
2576	bz,pn	%icc, .copyin_small		! if zero, disable HW copy
2577	  cmp	%o2, %o3			! if length <= limit
2578	bleu,pt	%ncc, .copyin_small		! go to small copy
2579	  nop
2580	ba,pt	%ncc, .copyin_more		! otherwise go to large copy
2581	  nop
2582.copyin_4:
2583	! already checked longword, must be word aligned
2584	sethi	%hi(hw_copy_limit_4), %o3	! Check copy limit
2585	ld	[%o3 + %lo(hw_copy_limit_4)], %o3
2586	tst	%o3
2587	bz,pn	%icc, .copyin_small		! if zero, disable HW copy
2588	  cmp	%o2, %o3			! if length <= limit
2589	bleu,pt	%ncc, .copyin_small		! go to small copy
2590	  nop
2591	ba,pt	%ncc, .copyin_more		! otherwise go to large copy
2592	  nop
2593.copyin_8:
2594	sethi	%hi(hw_copy_limit_8), %o3	! Check copy limit
2595	ld	[%o3 + %lo(hw_copy_limit_8)], %o3
2596	tst	%o3
2597	bz,pn	%icc, .copyin_small		! if zero, disable HW copy
2598	  cmp	%o2, %o3			! if length <= limit
2599	bleu,pt	%ncc, .copyin_small		! go to small copy
2600	  nop
2601	ba,pt	%ncc, .copyin_more		! otherwise go to large copy
2602	  nop
2603
2604	.align	16
2605	nop				! instruction alignment
2606					! see discussion at start of file
2607.copyin_small:
2608	sethi	%hi(.sm_copyin_err), %o5	! .sm_copyin_err is lofault
2609	or	%o5, %lo(.sm_copyin_err), %o5
2610	ldn	[THREAD_REG + T_LOFAULT], %o4	! set/save t_lofault, no tramp
2611	membar	#Sync				! sync error barrier
2612	stn	%o5, [THREAD_REG + T_LOFAULT]
2613.sm_do_copyin:
2614	mov	%o0, SM_SAVE_SRC
2615	mov	%o1, SM_SAVE_DST
2616	cmp	%o2, SHORTCOPY		! check for really short case
2617	bleu,pt	%ncc, .ci_sm_left	!
2618	  mov	%o2, SM_SAVE_COUNT
2619	cmp	%o2, CHKSIZE		! check for medium length cases
2620	bgu,pn	%ncc, .ci_med		!
2621	  or	%o0, %o1, %o3		! prepare alignment check
2622	andcc	%o3, 0x3, %g0		! test for alignment
2623	bz,pt	%ncc, .ci_sm_word	! branch to word aligned case
2624.ci_sm_movebytes:
2625	  sub	%o2, 3, %o2		! adjust count to allow cc zero test
2626.ci_sm_notalign4:
2627	lduba	[%o0]ASI_USER, %o3	! read byte
2628	subcc	%o2, 4, %o2		! reduce count by 4
2629	stb	%o3, [%o1]		! write byte
2630	add	%o0, 1, %o0		! advance SRC by 1
2631	lduba	[%o0]ASI_USER, %o3	! repeat for a total of 4 bytes
2632	add	%o0, 1, %o0		! advance SRC by 1
2633	stb	%o3, [%o1 + 1]
2634	add	%o1, 4, %o1		! advance DST by 4
2635	lduba	[%o0]ASI_USER, %o3
2636	add	%o0, 1, %o0		! advance SRC by 1
2637	stb	%o3, [%o1 - 2]
2638	lduba	[%o0]ASI_USER, %o3
2639	add	%o0, 1, %o0		! advance SRC by 1
2640	bgt,pt	%ncc, .ci_sm_notalign4	! loop til 3 or fewer bytes remain
2641	  stb	%o3, [%o1 - 1]
2642	add	%o2, 3, %o2		! restore count
2643.ci_sm_left:
2644	tst	%o2
2645	bz,pt	%ncc, .ci_sm_exit
2646	  nop
2647	lduba	[%o0]ASI_USER, %o3		! load one byte
2648	deccc	%o2			! reduce count for cc test
2649	bz,pt	%ncc, .ci_sm_exit
2650	  stb	%o3,[%o1]		! store one byte
2651	inc	%o0
2652	lduba	[%o0]ASI_USER, %o3	! load second byte
2653	deccc	%o2
2654	bz,pt	%ncc, .ci_sm_exit
2655	  stb	%o3,[%o1 + 1]		! store second byte
2656	inc	%o0
2657	lduba	[%o0]ASI_USER, %o3	! load third byte
2658	stb	%o3,[%o1 + 2]		! store third byte
2659	membar	#Sync				! sync error barrier
2660	stn	%o4, [THREAD_REG + T_LOFAULT]	! restore old t_lofault
2661	retl
2662	  mov	%g0, %o0		! return 0
2663	.align	16
2664.ci_sm_words:
2665	lduwa	[%o0]ASI_USER, %o3		! read word
2666.ci_sm_wordx:
2667	subcc	%o2, 8, %o2		! update count
2668	stw	%o3, [%o1]		! write word
2669	add	%o0, 4, %o0		! update SRC
2670	add	%o1, 8, %o1		! update DST
2671	lduwa	[%o0]ASI_USER, %o3	! read word
2672	add	%o0, 4, %o0		! update SRC
2673	bgt,pt	%ncc, .ci_sm_words	! loop til done
2674	  stw	%o3, [%o1 - 4]		! write word
2675	addcc	%o2, 7, %o2		! restore count
2676	bz,pt	%ncc, .ci_sm_exit
2677	  nop
2678	deccc	%o2
2679	bz,pt	%ncc, .ci_sm_byte
2680.ci_sm_half:
2681	  subcc	%o2, 2, %o2		! reduce count by 2
2682	lduha	[%o0]ASI_USER, %o3	! read half word
2683	add	%o0, 2, %o0		! advance SRC by 2
2684	add	%o1, 2, %o1		! advance DST by 2
2685	bgt,pt	%ncc, .ci_sm_half	! loop til done
2686	  sth	%o3, [%o1 - 2]		! write half word
2687	addcc	%o2, 1, %o2		! restore count
2688	bz,pt	%ncc, .ci_sm_exit
2689	  nop
2690.ci_sm_byte:
2691	lduba	[%o0]ASI_USER, %o3
2692	stb	%o3, [%o1]
2693	membar	#Sync				! sync error barrier
2694	stn	%o4, [THREAD_REG + T_LOFAULT]	! restore old t_lofault
2695	retl
2696	  mov	%g0, %o0		! return 0
2697	.align	16
2698.ci_sm_word:
2699	subcc	%o2, 4, %o2		! update count
2700	bgt,pt	%ncc, .ci_sm_wordx
2701	  lduwa	[%o0]ASI_USER, %o3		! read word
2702	addcc	%o2, 3, %o2		! restore count
2703	bz,pt	%ncc, .ci_sm_exit
2704	  stw	%o3, [%o1]		! write word
2705	deccc	%o2			! reduce count for cc test
2706	add	%o0, 4, %o0
2707	lduba	[%o0]ASI_USER, %o3	! load one byte
2708	bz,pt	%ncc, .ci_sm_exit
2709	  stb	%o3, [%o1 + 4]		! store one byte
2710	inc	%o0
2711	lduba	[%o0]ASI_USER, %o3	! load second byte
2712	deccc	%o2
2713	bz,pt	%ncc, .ci_sm_exit
2714	  stb	%o3, [%o1 + 5]		! store second byte
2715	inc	%o0
2716	lduba	[%o0]ASI_USER, %o3	! load third byte
2717	stb	%o3, [%o1 + 6]		! store third byte
2718.ci_sm_exit:
2719	membar	#Sync				! sync error barrier
2720	stn	%o4, [THREAD_REG + T_LOFAULT]	! restore old t_lofault
2721	retl
2722	  mov	%g0, %o0		! return 0
2723
2724	.align 16
2725.ci_med:
2726	xor	%o0, %o1, %o3		! setup alignment check
2727	btst	1, %o3
2728	bnz,pt	%ncc, .ci_sm_movebytes	! unaligned
2729	  nop
2730	btst	3, %o3
2731	bnz,pt	%ncc, .ci_med_half	! halfword aligned
2732	  nop
2733	btst	7, %o3
2734	bnz,pt	%ncc, .ci_med_word	! word aligned
2735	  nop
2736.ci_med_long:
2737	btst	3, %o0			! check for
2738	bz,pt	%ncc, .ci_med_long1	! word alignment
2739	  nop
2740.ci_med_long0:
2741	lduba	[%o0]ASI_USER, %o3		! load one byte
2742	inc	%o0
2743	stb	%o3,[%o1]		! store byte
2744	inc	%o1
2745	btst	3, %o0
2746	bnz,pt	%ncc, .ci_med_long0
2747	  dec	%o2
2748.ci_med_long1:			! word aligned
2749	btst	7, %o0			! check for long word
2750	bz,pt	%ncc, .ci_med_long2
2751	  nop
2752	lduwa	[%o0]ASI_USER, %o3	! load word
2753	add	%o0, 4, %o0		! advance SRC by 4
2754	stw	%o3, [%o1]		! store word
2755	add	%o1, 4, %o1		! advance DST by 4
2756	sub	%o2, 4, %o2		! reduce count by 4
2757!
2758!  Now long word aligned and have at least 32 bytes to move
2759!
2760.ci_med_long2:
2761	sub	%o2, 31, %o2		! adjust count to allow cc zero test
2762.ci_med_lmove:
2763	ldxa	[%o0]ASI_USER, %o3	! read long word
2764	subcc	%o2, 32, %o2		! reduce count by 32
2765	stx	%o3, [%o1]		! write long word
2766	add	%o0, 8, %o0		! advance SRC by 8
2767	ldxa	[%o0]ASI_USER, %o3	! repeat for a total for 4 long words
2768	add	%o0, 8, %o0		! advance SRC by 8
2769	stx	%o3, [%o1 + 8]
2770	add	%o1, 32, %o1		! advance DST by 32
2771	ldxa	[%o0]ASI_USER, %o3
2772	add	%o0, 8, %o0		! advance SRC by 8
2773	stx	%o3, [%o1 - 16]
2774	ldxa	[%o0]ASI_USER, %o3
2775	add	%o0, 8, %o0		! advance SRC by 8
2776	bgt,pt	%ncc, .ci_med_lmove	! loop til 31 or fewer bytes left
2777	  stx	%o3, [%o1 - 8]
2778	addcc	%o2, 24, %o2		! restore count to long word offset
2779	ble,pt	%ncc, .ci_med_lextra	! check for more long words to move
2780	  nop
2781.ci_med_lword:
2782	ldxa	[%o0]ASI_USER, %o3	! read long word
2783	subcc	%o2, 8, %o2		! reduce count by 8
2784	stx	%o3, [%o1]		! write long word
2785	add	%o0, 8, %o0		! advance SRC by 8
2786	bgt,pt	%ncc, .ci_med_lword	! loop til 7 or fewer bytes left
2787	  add	%o1, 8, %o1		! advance DST by 8
2788.ci_med_lextra:
2789	addcc	%o2, 7, %o2		! restore rest of count
2790	bz,pt	%ncc, .ci_sm_exit	! if zero, then done
2791	  deccc	%o2
2792	bz,pt	%ncc, .ci_sm_byte
2793	  nop
2794	ba,pt	%ncc, .ci_sm_half
2795	  nop
2796
2797	.align 16
2798	nop				! instruction alignment
2799					! see discussion at start of file
2800.ci_med_word:
2801	btst	3, %o0			! check for
2802	bz,pt	%ncc, .ci_med_word1	! word alignment
2803	  nop
2804.ci_med_word0:
2805	lduba	[%o0]ASI_USER, %o3	! load one byte
2806	inc	%o0
2807	stb	%o3,[%o1]		! store byte
2808	inc	%o1
2809	btst	3, %o0
2810	bnz,pt	%ncc, .ci_med_word0
2811	  dec	%o2
2812!
2813!  Now word aligned and have at least 36 bytes to move
2814!
2815.ci_med_word1:
2816	sub	%o2, 15, %o2		! adjust count to allow cc zero test
2817.ci_med_wmove:
2818	lduwa	[%o0]ASI_USER, %o3	! read word
2819	subcc	%o2, 16, %o2		! reduce count by 16
2820	stw	%o3, [%o1]		! write word
2821	add	%o0, 4, %o0		! advance SRC by 4
2822	lduwa	[%o0]ASI_USER, %o3	! repeat for a total for 4 words
2823	add	%o0, 4, %o0		! advance SRC by 4
2824	stw	%o3, [%o1 + 4]
2825	add	%o1, 16, %o1		! advance DST by 16
2826	lduwa	[%o0]ASI_USER, %o3
2827	add	%o0, 4, %o0		! advance SRC by 4
2828	stw	%o3, [%o1 - 8]
2829	lduwa	[%o0]ASI_USER, %o3
2830	add	%o0, 4, %o0		! advance SRC by 4
2831	bgt,pt	%ncc, .ci_med_wmove	! loop til 15 or fewer bytes left
2832	  stw	%o3, [%o1 - 4]
2833	addcc	%o2, 12, %o2		! restore count to word offset
2834	ble,pt	%ncc, .ci_med_wextra	! check for more words to move
2835	  nop
2836.ci_med_word2:
2837	lduwa	[%o0]ASI_USER, %o3	! read word
2838	subcc	%o2, 4, %o2		! reduce count by 4
2839	stw	%o3, [%o1]		! write word
2840	add	%o0, 4, %o0		! advance SRC by 4
2841	bgt,pt	%ncc, .ci_med_word2	! loop til 3 or fewer bytes left
2842	  add	%o1, 4, %o1		! advance DST by 4
2843.ci_med_wextra:
2844	addcc	%o2, 3, %o2		! restore rest of count
2845	bz,pt	%ncc, .ci_sm_exit	! if zero, then done
2846	  deccc	%o2
2847	bz,pt	%ncc, .ci_sm_byte
2848	  nop
2849	ba,pt	%ncc, .ci_sm_half
2850	  nop
2851
2852	.align 16
2853	nop				! instruction alignment
2854					! see discussion at start of file
2855.ci_med_half:
2856	btst	1, %o0			! check for
2857	bz,pt	%ncc, .ci_med_half1	! half word alignment
2858	  nop
2859	lduba	[%o0]ASI_USER, %o3	! load one byte
2860	inc	%o0
2861	stb	%o3,[%o1]		! store byte
2862	inc	%o1
2863	dec	%o2
2864!
2865!  Now half word aligned and have at least 38 bytes to move
2866!
2867.ci_med_half1:
2868	sub	%o2, 7, %o2		! adjust count to allow cc zero test
2869.ci_med_hmove:
2870	lduha	[%o0]ASI_USER, %o3	! read half word
2871	subcc	%o2, 8, %o2		! reduce count by 8
2872	sth	%o3, [%o1]		! write half word
2873	add	%o0, 2, %o0		! advance SRC by 2
2874	lduha	[%o0]ASI_USER, %o3	! repeat for a total for 4 halfwords
2875	add	%o0, 2, %o0		! advance SRC by 2
2876	sth	%o3, [%o1 + 2]
2877	add	%o1, 8, %o1		! advance DST by 8
2878	lduha	[%o0]ASI_USER, %o3
2879	add	%o0, 2, %o0		! advance SRC by 2
2880	sth	%o3, [%o1 - 4]
2881	lduha	[%o0]ASI_USER, %o3
2882	add	%o0, 2, %o0		! advance SRC by 2
2883	bgt,pt	%ncc, .ci_med_hmove	! loop til 7 or fewer bytes left
2884	  sth	%o3, [%o1 - 2]
2885	addcc	%o2, 7, %o2		! restore count
2886	bz,pt	%ncc, .ci_sm_exit
2887	  deccc	%o2
2888	bz,pt	%ncc, .ci_sm_byte
2889	  nop
2890	ba,pt	%ncc, .ci_sm_half
2891	  nop
2892
2893.sm_copyin_err:
2894	membar	#Sync
2895	stn	%o4, [THREAD_REG + T_LOFAULT]   ! restore old t_lofault
2896	mov	SM_SAVE_SRC, %o0
2897	mov	SM_SAVE_DST, %o1
2898	mov	SM_SAVE_COUNT, %o2
2899	ldn	[THREAD_REG + T_COPYOPS], %o3	! check for copyop handler
2900	tst	%o3
2901	bz,pt	%ncc, 3f			! if not, return error
2902	  nop
2903	ldn	[%o3 + CP_COPYIN], %o5		! if handler, invoke it with
2904	jmp	%o5				! original arguments
2905	  nop
29063:
2907	retl
2908	  or	%g0, -1, %o0		! return errno value
2909
2910	SET_SIZE(copyin)
2911
2912
2913/*
2914 * The _more entry points are not intended to be used directly by
2915 * any caller from outside this file.  They are provided to allow
2916 * profiling and dtrace of the portions of the copy code that uses
2917 * the floating point registers.
2918 * This entry is particularly important as DTRACE (at least as of
2919 * 4/2004) does not support leaf functions.
2920 */
2921
2922	ENTRY(copyin_more)
2923.copyin_more:
2924	save	%sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
2925	set	.copyin_err, REAL_LOFAULT
2926
2927/*
2928 * Copy ins that reach here are larger than VIS_COPY_THRESHOLD bytes
2929 */
2930.do_copyin:
2931	set	copyio_fault, %l7		! .copyio_fault is lofault val
2932
2933	ldn	[THREAD_REG + T_LOFAULT], %l6	! save existing handler
2934	membar	#Sync				! sync error barrier
2935	stn	%l7, [THREAD_REG + T_LOFAULT]	! set t_lofault
2936
2937	mov	%i0, SAVE_SRC
2938	mov	%i1, SAVE_DST
2939	mov	%i2, SAVE_COUNT
2940
2941	FP_NOMIGRATE(6, 7)
2942
2943	rd	%fprs, %o2		! check for unused fp
2944	st	%o2, [%fp + STACK_BIAS - SAVED_FPRS_OFFSET] ! save orig %fprs
2945	btst	FPRS_FEF, %o2
2946	bz,a,pt	%icc, .do_blockcopyin
2947	  wr	%g0, FPRS_FEF, %fprs
2948
2949	BST_FPQ2Q4_TOSTACK(%o2)
2950
2951.do_blockcopyin:
2952	rd	%gsr, %o2
2953	stx	%o2, [%fp + STACK_BIAS - SAVED_GSR_OFFSET]	! save gsr
2954	or	%l6, FPUSED_FLAG, %l6
2955
2956	andcc	DST, VIS_BLOCKSIZE - 1, TMP
2957	mov	ASI_USER, %asi
2958	bz,pt	%ncc, 2f
2959	  neg	TMP
2960	add	TMP, VIS_BLOCKSIZE, TMP
2961
2962	! TMP = bytes required to align DST on FP_BLOCK boundary
2963	! Using SRC as a tmp here
2964	cmp	TMP, 3
2965	bleu,pt	%ncc, 1f
2966	  sub	CNT,TMP,CNT		! adjust main count
2967	sub	TMP, 3, TMP		! adjust for end of loop test
2968.ci_blkalign:
2969	lduba	[REALSRC]%asi, SRC	! move 4 bytes per loop iteration
2970	stb	SRC, [DST]
2971	subcc	TMP, 4, TMP
2972	lduba	[REALSRC + 1]%asi, SRC
2973	add	REALSRC, 4, REALSRC
2974	stb	SRC, [DST + 1]
2975	lduba	[REALSRC - 2]%asi, SRC
2976	add	DST, 4, DST
2977	stb	SRC, [DST - 2]
2978	lduba	[REALSRC - 1]%asi, SRC
2979	bgu,pt	%ncc, .ci_blkalign
2980	  stb	SRC, [DST - 1]
2981
2982	addcc	TMP, 3, TMP		! restore count adjustment
2983	bz,pt	%ncc, 2f		! no bytes left?
2984	  nop
29851:	lduba	[REALSRC]%asi, SRC
2986	inc	REALSRC
2987	inc	DST
2988	deccc	TMP
2989	bgu	%ncc, 1b
2990	  stb	SRC, [DST - 1]
2991
29922:
2993	andn	REALSRC, 0x7, SRC
2994	alignaddr REALSRC, %g0, %g0
2995
2996	! SRC - 8-byte aligned
2997	! DST - 64-byte aligned
2998	prefetcha [SRC]%asi, #one_read
2999	prefetcha [SRC + (1 * VIS_BLOCKSIZE)]%asi, #one_read
3000	prefetcha [SRC + (2 * VIS_BLOCKSIZE)]%asi, #one_read
3001	prefetcha [SRC + (3 * VIS_BLOCKSIZE)]%asi, #one_read
3002	ldda	[SRC]%asi, %f16
3003#if CHEETAH_PREFETCH > 4
3004	prefetcha [SRC + (4 * VIS_BLOCKSIZE)]%asi, #one_read
3005#endif
3006	ldda	[SRC + 0x08]%asi, %f18
3007#if CHEETAH_PREFETCH > 5
3008	prefetcha [SRC + (5 * VIS_BLOCKSIZE)]%asi, #one_read
3009#endif
3010	ldda	[SRC + 0x10]%asi, %f20
3011#if CHEETAH_PREFETCH > 6
3012	prefetcha [SRC + (6 * VIS_BLOCKSIZE)]%asi, #one_read
3013#endif
3014	faligndata %f16, %f18, %f48
3015	ldda	[SRC + 0x18]%asi, %f22
3016#if CHEETAH_PREFETCH > 7
3017	prefetcha [SRC + (7 * VIS_BLOCKSIZE)]%asi, #one_read
3018#endif
3019	faligndata %f18, %f20, %f50
3020	ldda	[SRC + 0x20]%asi, %f24
3021	faligndata %f20, %f22, %f52
3022	ldda	[SRC + 0x28]%asi, %f26
3023	faligndata %f22, %f24, %f54
3024	ldda	[SRC + 0x30]%asi, %f28
3025	faligndata %f24, %f26, %f56
3026	ldda	[SRC + 0x38]%asi, %f30
3027	faligndata %f26, %f28, %f58
3028	ldda	[SRC + VIS_BLOCKSIZE]%asi, %f16
3029	sub	CNT, VIS_BLOCKSIZE, CNT
3030	add	SRC, VIS_BLOCKSIZE, SRC
3031	add	REALSRC, VIS_BLOCKSIZE, REALSRC
3032	ba,a,pt	%ncc, 1f
3033	  nop
3034	.align	16
30351:
3036	ldda	[SRC + 0x08]%asi, %f18
3037	faligndata %f28, %f30, %f60
3038	ldda	[SRC + 0x10]%asi, %f20
3039	faligndata %f30, %f16, %f62
3040	stda	%f48, [DST]ASI_BLK_P
3041	ldda	[SRC + 0x18]%asi, %f22
3042	faligndata %f16, %f18, %f48
3043	ldda	[SRC + 0x20]%asi, %f24
3044	faligndata %f18, %f20, %f50
3045	ldda	[SRC + 0x28]%asi, %f26
3046	faligndata %f20, %f22, %f52
3047	ldda	[SRC + 0x30]%asi, %f28
3048	faligndata %f22, %f24, %f54
3049	ldda	[SRC + 0x38]%asi, %f30
3050	faligndata %f24, %f26, %f56
3051	sub	CNT, VIS_BLOCKSIZE, CNT
3052	ldda	[SRC + VIS_BLOCKSIZE]%asi, %f16
3053	faligndata %f26, %f28, %f58
3054	prefetcha [SRC + ((CHEETAH_PREFETCH) * VIS_BLOCKSIZE) + 8]%asi, #one_read
3055	add	DST, VIS_BLOCKSIZE, DST
3056	prefetcha [SRC + ((CHEETAH_2ND_PREFETCH) * VIS_BLOCKSIZE)]%asi, #one_read
3057	add	REALSRC, VIS_BLOCKSIZE, REALSRC
3058	cmp	CNT, VIS_BLOCKSIZE + 8
3059	bgu,pt	%ncc, 1b
3060	  add	SRC, VIS_BLOCKSIZE, SRC
3061
3062	! only if REALSRC & 0x7 is 0
3063	cmp	CNT, VIS_BLOCKSIZE
3064	bne	%ncc, 3f
3065	  andcc	REALSRC, 0x7, %g0
3066	bz,pt	%ncc, 2f
3067	  nop
30683:
3069	faligndata %f28, %f30, %f60
3070	faligndata %f30, %f16, %f62
3071	stda	%f48, [DST]ASI_BLK_P
3072	add	DST, VIS_BLOCKSIZE, DST
3073	ba,pt	%ncc, 3f
3074	  nop
30752:
3076	ldda	[SRC + 0x08]%asi, %f18
3077	fsrc1	%f28, %f60
3078	ldda	[SRC + 0x10]%asi, %f20
3079	fsrc1	%f30, %f62
3080	stda	%f48, [DST]ASI_BLK_P
3081	ldda	[SRC + 0x18]%asi, %f22
3082	fsrc1	%f16, %f48
3083	ldda	[SRC + 0x20]%asi, %f24
3084	fsrc1	%f18, %f50
3085	ldda	[SRC + 0x28]%asi, %f26
3086	fsrc1	%f20, %f52
3087	ldda	[SRC + 0x30]%asi, %f28
3088	fsrc1	%f22, %f54
3089	ldda	[SRC + 0x38]%asi, %f30
3090	fsrc1	%f24, %f56
3091	sub	CNT, VIS_BLOCKSIZE, CNT
3092	add	DST, VIS_BLOCKSIZE, DST
3093	add	SRC, VIS_BLOCKSIZE, SRC
3094	add	REALSRC, VIS_BLOCKSIZE, REALSRC
3095	fsrc1	%f26, %f58
3096	fsrc1	%f28, %f60
3097	fsrc1	%f30, %f62
3098	stda	%f48, [DST]ASI_BLK_P
3099	add	DST, VIS_BLOCKSIZE, DST
3100	ba,a,pt	%ncc, 4f
3101	  nop
3102
31033:	tst	CNT
3104	bz,a	%ncc, 4f
3105	  nop
3106
31075:	lduba	[REALSRC]ASI_USER, TMP
3108	inc	REALSRC
3109	inc	DST
3110	deccc	CNT
3111	bgu	%ncc, 5b
3112	  stb	TMP, [DST - 1]
31134:
3114
3115.copyin_exit:
3116	membar	#Sync
3117
3118	FPRAS_INTERVAL(FPRAS_COPYIN, 1, %l5, %o2, %o3, %o4, %o5, 8)
3119	FPRAS_REWRITE_TYPE1(1, %l5, %f48, %o2, 9)
3120	FPRAS_CHECK(FPRAS_COPYIN, %l5, 9)	! lose outputs
3121
3122	ldx	[%fp + STACK_BIAS - SAVED_GSR_OFFSET], %o2	! restore gsr
3123	wr	%o2, 0, %gsr
3124
3125	ld	[%fp + STACK_BIAS - SAVED_FPRS_OFFSET], %o3
3126	btst	FPRS_FEF, %o3
3127	bz,pt	%icc, 4f
3128	  nop
3129
3130	BLD_FPQ2Q4_FROMSTACK(%o2)
3131
3132	ba,pt	%ncc, 1f
3133	  wr	%o3, 0, %fprs		! restore fprs
3134
31354:
3136	FZEROQ2Q4
3137	wr	%o3, 0, %fprs		! restore fprs
3138
31391:
3140	membar	#Sync				! sync error barrier
3141	andn	%l6, FPUSED_FLAG, %l6
3142	stn	%l6, [THREAD_REG + T_LOFAULT]	! restore old t_lofault
3143	FP_ALLOWMIGRATE(5, 6)
3144	ret
3145	  restore	%g0, 0, %o0
3146/*
3147 * We got here because of a fault during copyin
3148 * Errno value is in ERRNO, but DDI/DKI says return -1 (sigh).
3149 */
3150.copyin_err:
3151	ldn	[THREAD_REG + T_COPYOPS], %o4	! check for copyop handler
3152	tst	%o4
3153	bz,pt	%ncc, 2f			! if not, return error
3154	nop
3155	ldn	[%o4 + CP_COPYIN], %g2		! if handler, invoke it with
3156	jmp	%g2				! original arguments
3157	restore %g0, 0, %g0			! dispose of copy window
31582:
3159	ret
3160	restore %g0, -1, %o0			! return error value
3161
3162
3163	SET_SIZE(copyin_more)
3164
3165	ENTRY(xcopyin)
3166
3167	cmp	%o2, VIS_COPY_THRESHOLD		! check for leaf rtn case
3168	bleu,pt	%ncc, .xcopyin_small		! go to larger cases
3169	  xor	%o0, %o1, %o3			! are src, dst alignable?
3170	btst	7, %o3				!
3171	bz,pt	%ncc, .xcopyin_8		! check for longword alignment
3172	  nop
3173	btst	1, %o3				!
3174	bz,pt	%ncc, .xcopyin_2		! check for half-word
3175	  nop
3176	sethi	%hi(hw_copy_limit_1), %o3	! Check copy limit
3177	ld	[%o3 + %lo(hw_copy_limit_1)], %o3
3178	tst	%o3
3179	bz,pn	%icc, .xcopyin_small		! if zero, disable HW copy
3180	  cmp	%o2, %o3			! if length <= limit
3181	bleu,pt	%ncc, .xcopyin_small		! go to small copy
3182	  nop
3183	ba,pt	%ncc, .xcopyin_more		! otherwise go to large copy
3184	  nop
3185.xcopyin_2:
3186	btst	3, %o3				!
3187	bz,pt	%ncc, .xcopyin_4		! check for word alignment
3188	  nop
3189	sethi	%hi(hw_copy_limit_2), %o3	! Check copy limit
3190	ld	[%o3 + %lo(hw_copy_limit_2)], %o3
3191	tst	%o3
3192	bz,pn	%icc, .xcopyin_small		! if zero, disable HW copy
3193	  cmp	%o2, %o3			! if length <= limit
3194	bleu,pt	%ncc, .xcopyin_small		! go to small copy
3195	  nop
3196	ba,pt	%ncc, .xcopyin_more		! otherwise go to large copy
3197	  nop
3198.xcopyin_4:
3199	! already checked longword, must be word aligned
3200	sethi	%hi(hw_copy_limit_4), %o3	! Check copy limit
3201	ld	[%o3 + %lo(hw_copy_limit_4)], %o3
3202	tst	%o3
3203	bz,pn	%icc, .xcopyin_small		! if zero, disable HW copy
3204	  cmp	%o2, %o3			! if length <= limit
3205	bleu,pt	%ncc, .xcopyin_small		! go to small copy
3206	  nop
3207	ba,pt	%ncc, .xcopyin_more		! otherwise go to large copy
3208	  nop
3209.xcopyin_8:
3210	sethi	%hi(hw_copy_limit_8), %o3	! Check copy limit
3211	ld	[%o3 + %lo(hw_copy_limit_8)], %o3
3212	tst	%o3
3213	bz,pn	%icc, .xcopyin_small		! if zero, disable HW copy
3214	  cmp	%o2, %o3			! if length <= limit
3215	bleu,pt	%ncc, .xcopyin_small		! go to small copy
3216	  nop
3217	ba,pt	%ncc, .xcopyin_more		! otherwise go to large copy
3218	  nop
3219
3220.xcopyin_small:
3221	sethi	%hi(.sm_xcopyin_err), %o5  ! .sm_xcopyin_err is lofault value
3222	or	%o5, %lo(.sm_xcopyin_err), %o5
3223	ldn	[THREAD_REG + T_LOFAULT], %o4	! set/save t_lofaul
3224	membar	#Sync				! sync error barrier
3225	ba,pt	%ncc, .sm_do_copyin		! common code
3226	  stn	%o5, [THREAD_REG + T_LOFAULT]
3227
3228.xcopyin_more:
3229	save	%sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
3230	sethi	%hi(.xcopyin_err), REAL_LOFAULT	! .xcopyin_err is lofault value
3231	ba,pt	%ncc, .do_copyin
3232	  or	REAL_LOFAULT, %lo(.xcopyin_err), REAL_LOFAULT
3233
3234/*
3235 * We got here because of fault during xcopyin
3236 * Errno value is in ERRNO
3237 */
3238.xcopyin_err:
3239	ldn	[THREAD_REG + T_COPYOPS], %o4	! check for copyop handler
3240	tst	%o4
3241	bz,pt	%ncc, 2f			! if not, return error
3242	  nop
3243	ldn	[%o4 + CP_XCOPYIN], %g2		! if handler, invoke it with
3244	jmp	%g2				! original arguments
3245	  restore %g0, 0, %g0			! dispose of copy window
32462:
3247        ret
3248	  restore ERRNO, 0, %o0			! return errno value
3249
3250.sm_xcopyin_err:
3251
3252	membar	#Sync
3253	stn	%o4, [THREAD_REG + T_LOFAULT]   ! restore old t_lofault
3254	mov	SM_SAVE_SRC, %o0
3255	mov	SM_SAVE_DST, %o1
3256	mov	SM_SAVE_COUNT, %o2
3257	ldn	[THREAD_REG + T_COPYOPS], %o3	! check for copyop handler
3258	tst	%o3
3259	bz,pt	%ncc, 3f			! if not, return error
3260	  nop
3261	ldn	[%o3 + CP_XCOPYIN], %o5		! if handler, invoke it with
3262	jmp	%o5				! original arguments
3263	  nop
32643:
3265	retl
3266	  or	%g1, 0, %o0		! return errno value
3267
3268	SET_SIZE(xcopyin)
3269
3270	ENTRY(xcopyin_little)
3271	sethi	%hi(.xcopyio_err), %o5
3272	or	%o5, %lo(.xcopyio_err), %o5
3273	ldn	[THREAD_REG + T_LOFAULT], %o4
3274	membar	#Sync				! sync error barrier
3275	stn	%o5, [THREAD_REG + T_LOFAULT]
3276	mov	%o4, %o5
3277
3278	subcc	%g0, %o2, %o3
3279	add	%o0, %o2, %o0
3280	bz,pn	%ncc, 2f		! check for zero bytes
3281	  sub	%o2, 1, %o4
3282	add	%o0, %o4, %o0		! start w/last byte
3283	add	%o1, %o2, %o1
3284	lduba	[%o0 + %o3]ASI_AIUSL, %o4
3285
32861:	stb	%o4, [%o1 + %o3]
3287	inccc	%o3
3288	sub	%o0, 2, %o0		! get next byte
3289	bcc,a,pt %ncc, 1b
3290	  lduba	[%o0 + %o3]ASI_AIUSL, %o4
3291
32922:
3293	membar	#Sync				! sync error barrier
3294	stn	%o5, [THREAD_REG + T_LOFAULT]	! restore old t_lofault
3295	retl
3296	  mov	%g0, %o0		! return (0)
3297
3298.xcopyio_err:
3299	membar	#Sync				! sync error barrier
3300	stn	%o5, [THREAD_REG + T_LOFAULT]	! restore old t_lofault
3301	retl
3302	  mov	%g1, %o0
3303
3304	SET_SIZE(xcopyin_little)
3305
3306
3307/*
3308 * Copy a block of storage - must not overlap (from + len <= to).
3309 * No fault handler installed (to be called under on_fault())
3310 */
3311	ENTRY(copyin_noerr)
3312
3313	cmp	%o2, VIS_COPY_THRESHOLD		! check for leaf rtn case
3314	bleu,pt	%ncc, .copyin_ne_small		! go to larger cases
3315	  xor	%o0, %o1, %o3			! are src, dst alignable?
3316	btst	7, %o3				!
3317	bz,pt	%ncc, .copyin_ne_8		! check for longword alignment
3318	  nop
3319	btst	1, %o3				!
3320	bz,pt	%ncc, .copyin_ne_2		! check for half-word
3321	  nop
3322	sethi	%hi(hw_copy_limit_1), %o3	! Check copy limit
3323	ld	[%o3 + %lo(hw_copy_limit_1)], %o3
3324	tst	%o3
3325	bz,pn	%icc, .copyin_ne_small		! if zero, disable HW copy
3326	  cmp	%o2, %o3			! if length <= limit
3327	bleu,pt	%ncc, .copyin_ne_small		! go to small copy
3328	  nop
3329	ba,pt	%ncc, .copyin_noerr_more	! otherwise go to large copy
3330	  nop
3331.copyin_ne_2:
3332	btst	3, %o3				!
3333	bz,pt	%ncc, .copyin_ne_4		! check for word alignment
3334	  nop
3335	sethi	%hi(hw_copy_limit_2), %o3	! Check copy limit
3336	ld	[%o3 + %lo(hw_copy_limit_2)], %o3
3337	tst	%o3
3338	bz,pn	%icc, .copyin_ne_small		! if zero, disable HW copy
3339	  cmp	%o2, %o3			! if length <= limit
3340	bleu,pt	%ncc, .copyin_ne_small		! go to small copy
3341	  nop
3342	ba,pt	%ncc, .copyin_noerr_more	! otherwise go to large copy
3343	  nop
3344.copyin_ne_4:
3345	! already checked longword, must be word aligned
3346	sethi	%hi(hw_copy_limit_4), %o3	! Check copy limit
3347	ld	[%o3 + %lo(hw_copy_limit_4)], %o3
3348	tst	%o3
3349	bz,pn	%icc, .copyin_ne_small		! if zero, disable HW copy
3350	  cmp	%o2, %o3			! if length <= limit
3351	bleu,pt	%ncc, .copyin_ne_small		! go to small copy
3352	  nop
3353	ba,pt	%ncc, .copyin_noerr_more	! otherwise go to large copy
3354	  nop
3355.copyin_ne_8:
3356	sethi	%hi(hw_copy_limit_8), %o3	! Check copy limit
3357	ld	[%o3 + %lo(hw_copy_limit_8)], %o3
3358	tst	%o3
3359	bz,pn	%icc, .copyin_ne_small		! if zero, disable HW copy
3360	  cmp	%o2, %o3			! if length <= limit
3361	bleu,pt	%ncc, .copyin_ne_small		! go to small copy
3362	  nop
3363	ba,pt	%ncc, .copyin_noerr_more	! otherwise go to large copy
3364	  nop
3365
3366.copyin_ne_small:
3367	ldn	[THREAD_REG + T_LOFAULT], %o4
3368	tst	%o4
3369	bz,pn	%ncc, .sm_do_copyin
3370	  nop
3371	sethi	%hi(.sm_copyio_noerr), %o5
3372	or	%o5, %lo(.sm_copyio_noerr), %o5
3373	membar	#Sync				! sync error barrier
3374	ba,pt	%ncc, .sm_do_copyin
3375	  stn	%o5, [THREAD_REG + T_LOFAULT]	! set/save t_lofault
3376
3377.copyin_noerr_more:
3378	save	%sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
3379	sethi	%hi(.copyio_noerr), REAL_LOFAULT
3380	ba,pt	%ncc, .do_copyin
3381	  or	REAL_LOFAULT, %lo(.copyio_noerr), REAL_LOFAULT
3382
3383.copyio_noerr:
3384	jmp	%l6
3385	  restore %g0,0,%g0
3386
3387.sm_copyio_noerr:
3388	membar	#Sync
3389	stn	%o4, [THREAD_REG + T_LOFAULT]	! restore t_lofault
3390	jmp	%o4
3391	  nop
3392
3393	SET_SIZE(copyin_noerr)
3394
3395/*
3396 * Copy a block of storage - must not overlap (from + len <= to).
3397 * No fault handler installed (to be called under on_fault())
3398 */
3399
3400	ENTRY(copyout_noerr)
3401
3402	cmp	%o2, VIS_COPY_THRESHOLD		! check for leaf rtn case
3403	bleu,pt	%ncc, .copyout_ne_small		! go to larger cases
3404	  xor	%o0, %o1, %o3			! are src, dst alignable?
3405	btst	7, %o3				!
3406	bz,pt	%ncc, .copyout_ne_8		! check for longword alignment
3407	  nop
3408	btst	1, %o3				!
3409	bz,pt	%ncc, .copyout_ne_2		! check for half-word
3410	  nop
3411	sethi	%hi(hw_copy_limit_1), %o3	! Check copy limit
3412	ld	[%o3 + %lo(hw_copy_limit_1)], %o3
3413	tst	%o3
3414	bz,pn	%icc, .copyout_ne_small		! if zero, disable HW copy
3415	  cmp	%o2, %o3			! if length <= limit
3416	bleu,pt	%ncc, .copyout_ne_small		! go to small copy
3417	  nop
3418	ba,pt	%ncc, .copyout_noerr_more	! otherwise go to large copy
3419	  nop
3420.copyout_ne_2:
3421	btst	3, %o3				!
3422	bz,pt	%ncc, .copyout_ne_4		! check for word alignment
3423	  nop
3424	sethi	%hi(hw_copy_limit_2), %o3	! Check copy limit
3425	ld	[%o3 + %lo(hw_copy_limit_2)], %o3
3426	tst	%o3
3427	bz,pn	%icc, .copyout_ne_small		! if zero, disable HW copy
3428	  cmp	%o2, %o3			! if length <= limit
3429	bleu,pt	%ncc, .copyout_ne_small		! go to small copy
3430	  nop
3431	ba,pt	%ncc, .copyout_noerr_more	! otherwise go to large copy
3432	  nop
3433.copyout_ne_4:
3434	! already checked longword, must be word aligned
3435	sethi	%hi(hw_copy_limit_4), %o3	! Check copy limit
3436	ld	[%o3 + %lo(hw_copy_limit_4)], %o3
3437	tst	%o3
3438	bz,pn	%icc, .copyout_ne_small		! if zero, disable HW copy
3439	  cmp	%o2, %o3			! if length <= limit
3440	bleu,pt	%ncc, .copyout_ne_small		! go to small copy
3441	  nop
3442	ba,pt	%ncc, .copyout_noerr_more	! otherwise go to large copy
3443	  nop
3444.copyout_ne_8:
3445	sethi	%hi(hw_copy_limit_8), %o3	! Check copy limit
3446	ld	[%o3 + %lo(hw_copy_limit_8)], %o3
3447	tst	%o3
3448	bz,pn	%icc, .copyout_ne_small		! if zero, disable HW copy
3449	  cmp	%o2, %o3			! if length <= limit
3450	bleu,pt	%ncc, .copyout_ne_small		! go to small copy
3451	  nop
3452	ba,pt	%ncc, .copyout_noerr_more	! otherwise go to large copy
3453	  nop
3454
3455.copyout_ne_small:
3456	ldn	[THREAD_REG + T_LOFAULT], %o4
3457	tst	%o4
3458	bz,pn	%ncc, .sm_do_copyout
3459	  nop
3460	sethi	%hi(.sm_copyio_noerr), %o5
3461	or	%o5, %lo(.sm_copyio_noerr), %o5
3462	membar	#Sync				! sync error barrier
3463	ba,pt	%ncc, .sm_do_copyout
3464	stn	%o5, [THREAD_REG + T_LOFAULT]	! set/save t_lofault
3465
3466.copyout_noerr_more:
3467	save	%sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
3468	sethi	%hi(.copyio_noerr), REAL_LOFAULT
3469	ba,pt	%ncc, .do_copyout
3470	  or	REAL_LOFAULT, %lo(.copyio_noerr), REAL_LOFAULT
3471
3472	SET_SIZE(copyout_noerr)
3473
3474
3475/*
3476 * hwblkclr - clears block-aligned, block-multiple-sized regions that are
3477 * longer than 256 bytes in length using spitfire's block stores.  If
3478 * the criteria for using this routine are not met then it calls bzero
3479 * and returns 1.  Otherwise 0 is returned indicating success.
3480 * Caller is responsible for ensuring use_hw_bzero is true and that
3481 * kpreempt_disable() has been called.
3482 */
3483	! %i0 - start address
3484	! %i1 - length of region (multiple of 64)
3485	! %l0 - saved fprs
3486	! %l1 - pointer to saved %d0 block
3487	! %l2 - saved curthread->t_lwp
3488
3489	ENTRY(hwblkclr)
3490	! get another window w/space for one aligned block of saved fpregs
3491	save	%sp, -SA(MINFRAME + 2*VIS_BLOCKSIZE), %sp
3492
3493	! Must be block-aligned
3494	andcc	%i0, (VIS_BLOCKSIZE-1), %g0
3495	bnz,pn	%ncc, 1f
3496	  nop
3497
3498	! ... and must be 256 bytes or more
3499	cmp	%i1, 256
3500	blu,pn	%ncc, 1f
3501	  nop
3502
3503	! ... and length must be a multiple of VIS_BLOCKSIZE
3504	andcc	%i1, (VIS_BLOCKSIZE-1), %g0
3505	bz,pn	%ncc, 2f
3506	  nop
3507
35081:	! punt, call bzero but notify the caller that bzero was used
3509	mov	%i0, %o0
3510	call	bzero
3511	mov	%i1, %o1
3512	ret
3513	  restore	%g0, 1, %o0 ! return (1) - did not use block operations
3514
35152:	rd	%fprs, %l0		! check for unused fp
3516	btst	FPRS_FEF, %l0
3517	bz,pt	%icc, 1f
3518	  nop
3519
3520	! save in-use fpregs on stack
3521	membar	#Sync
3522	add	%fp, STACK_BIAS - 65, %l1
3523	and	%l1, -VIS_BLOCKSIZE, %l1
3524	stda	%d0, [%l1]ASI_BLK_P
3525
35261:	membar	#StoreStore|#StoreLoad|#LoadStore
3527	wr	%g0, FPRS_FEF, %fprs
3528	wr	%g0, ASI_BLK_P, %asi
3529
3530	! Clear block
3531	fzero	%d0
3532	fzero	%d2
3533	fzero	%d4
3534	fzero	%d6
3535	fzero	%d8
3536	fzero	%d10
3537	fzero	%d12
3538	fzero	%d14
3539
3540	mov	256, %i3
3541	ba,pt	%ncc, .pz_doblock
3542	  nop
3543
3544.pz_blkstart:
3545      ! stda	%d0, [%i0 + 192]%asi  ! in dly slot of branch that got us here
3546	stda	%d0, [%i0 + 128]%asi
3547	stda	%d0, [%i0 + 64]%asi
3548	stda	%d0, [%i0]%asi
3549.pz_zinst:
3550	add	%i0, %i3, %i0
3551	sub	%i1, %i3, %i1
3552.pz_doblock:
3553	cmp	%i1, 256
3554	bgeu,a	%ncc, .pz_blkstart
3555	  stda	%d0, [%i0 + 192]%asi
3556
3557	cmp	%i1, 64
3558	blu	%ncc, .pz_finish
3559
3560	  andn	%i1, (64-1), %i3
3561	srl	%i3, 4, %i2		! using blocks, 1 instr / 16 words
3562	set	.pz_zinst, %i4
3563	sub	%i4, %i2, %i4
3564	jmp	%i4
3565	  nop
3566
3567.pz_finish:
3568	membar	#Sync
3569	btst	FPRS_FEF, %l0
3570	bz,a	.pz_finished
3571	  wr	%l0, 0, %fprs		! restore fprs
3572
3573	! restore fpregs from stack
3574	ldda	[%l1]ASI_BLK_P, %d0
3575	membar	#Sync
3576	wr	%l0, 0, %fprs		! restore fprs
3577
3578.pz_finished:
3579	ret
3580	  restore	%g0, 0, %o0		! return (bzero or not)
3581
3582	SET_SIZE(hwblkclr)
3583
3584	/*
3585	 * Copy 32 bytes of data from src (%o0) to dst (%o1)
3586	 * using physical addresses.
3587	 */
3588	ENTRY_NP(hw_pa_bcopy32)
3589	rdpr	%pstate, %g1
3590	andn	%g1, PSTATE_IE, %g2
3591	wrpr	%g0, %g2, %pstate
3592
3593	rdpr	%pstate, %g0
3594	ldxa	[%o0]ASI_MEM, %o2
3595	add	%o0, 8, %o0
3596	ldxa	[%o0]ASI_MEM, %o3
3597	add	%o0, 8, %o0
3598	ldxa	[%o0]ASI_MEM, %o4
3599	add	%o0, 8, %o0
3600	ldxa	[%o0]ASI_MEM, %o5
3601
3602    	stxa	%g0, [%o1]ASI_DC_INVAL
3603	membar	#Sync
3604
3605	stxa	%o2, [%o1]ASI_MEM
3606	add	%o1, 8, %o1
3607	stxa	%o3, [%o1]ASI_MEM
3608	add	%o1, 8, %o1
3609	stxa	%o4, [%o1]ASI_MEM
3610	add	%o1, 8, %o1
3611	stxa	%o5, [%o1]ASI_MEM
3612
3613	retl
3614	  wrpr	  %g0, %g1, %pstate
3615
3616	SET_SIZE(hw_pa_bcopy32)
3617
3618	DGDEF(use_hw_bcopy)
3619	.word	1
3620	DGDEF(use_hw_bzero)
3621	.word	1
3622	DGDEF(hw_copy_limit_1)
3623	.word	0
3624	DGDEF(hw_copy_limit_2)
3625	.word	0
3626	DGDEF(hw_copy_limit_4)
3627	.word	0
3628	DGDEF(hw_copy_limit_8)
3629	.word	0
3630
3631	.align	64
3632	.section ".text"
3633