17c478bd9Sstevel@tonic-gate/* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5651ee059Siskreen * Common Development and Distribution License (the "License"). 6651ee059Siskreen * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate/* 22651ee059Siskreen * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 26c2b6e926Sae# ident "%Z%%M% %I% %E% SMI" 277c478bd9Sstevel@tonic-gate 287c478bd9Sstevel@tonic-gate#include <sys/param.h> 297c478bd9Sstevel@tonic-gate#include <sys/errno.h> 307c478bd9Sstevel@tonic-gate#include <sys/asm_linkage.h> 317c478bd9Sstevel@tonic-gate#include <sys/vtrace.h> 327c478bd9Sstevel@tonic-gate#include <sys/machthread.h> 337c478bd9Sstevel@tonic-gate#include <sys/clock.h> 347c478bd9Sstevel@tonic-gate#include <sys/asi.h> 357c478bd9Sstevel@tonic-gate#include <sys/fsr.h> 367c478bd9Sstevel@tonic-gate#include <sys/privregs.h> 377c478bd9Sstevel@tonic-gate 387c478bd9Sstevel@tonic-gate#include "assym.h" 397c478bd9Sstevel@tonic-gate 407c478bd9Sstevel@tonic-gate 417c478bd9Sstevel@tonic-gate/* 427c478bd9Sstevel@tonic-gate * Less then or equal this number of bytes we will always copy byte-for-byte 437c478bd9Sstevel@tonic-gate */ 447c478bd9Sstevel@tonic-gate#define SMALL_LIMIT 7 457c478bd9Sstevel@tonic-gate 467c478bd9Sstevel@tonic-gate/* 47473b13d4Sae * LOFAULT_SET : Flag set by kzero and kcopy to indicate that t_lofault 48473b13d4Sae * handler was set 497c478bd9Sstevel@tonic-gate */ 507c478bd9Sstevel@tonic-gate#define LOFAULT_SET 2 517c478bd9Sstevel@tonic-gate 527c478bd9Sstevel@tonic-gate 537c478bd9Sstevel@tonic-gate/* 547c478bd9Sstevel@tonic-gate * Copy a block of storage, returning an error code if `from' or 557c478bd9Sstevel@tonic-gate * `to' takes a kernel pagefault which cannot be resolved. 567c478bd9Sstevel@tonic-gate * Returns errno value on pagefault error, 0 if all ok 577c478bd9Sstevel@tonic-gate */ 587c478bd9Sstevel@tonic-gate 597c478bd9Sstevel@tonic-gate 607c478bd9Sstevel@tonic-gate 617c478bd9Sstevel@tonic-gate .seg ".text" 627c478bd9Sstevel@tonic-gate .align 4 637c478bd9Sstevel@tonic-gate 647c478bd9Sstevel@tonic-gate ENTRY(kcopy) 657c478bd9Sstevel@tonic-gate 667c478bd9Sstevel@tonic-gate save %sp, -SA(MINFRAME), %sp 67473b13d4Sae set .copyerr, %l7 ! copyerr is lofault value 68473b13d4Sae ldn [THREAD_REG + T_LOFAULT], %o5 ! save existing handler 69473b13d4Sae or %o5, LOFAULT_SET, %o5 70473b13d4Sae membar #Sync ! sync error barrier 71473b13d4Sae b .do_copy ! common code 72473b13d4Sae stn %l7, [THREAD_REG + T_LOFAULT] ! set t_lofault 737c478bd9Sstevel@tonic-gate 747c478bd9Sstevel@tonic-gate/* 757c478bd9Sstevel@tonic-gate * We got here because of a fault during kcopy. 767c478bd9Sstevel@tonic-gate * Errno value is in %g1. 777c478bd9Sstevel@tonic-gate */ 787c478bd9Sstevel@tonic-gate.copyerr: 79473b13d4Sae ! The kcopy() *always* sets a t_lofault handler and it ORs LOFAULT_SET 80473b13d4Sae ! into %o5 to indicate it has set t_lofault handler. Need to clear 81473b13d4Sae ! LOFAULT_SET flag before restoring the error handler. 82473b13d4Sae andn %o5, LOFAULT_SET, %o5 837c478bd9Sstevel@tonic-gate membar #Sync ! sync error barrier 847c478bd9Sstevel@tonic-gate stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault 857c478bd9Sstevel@tonic-gate ret 867c478bd9Sstevel@tonic-gate restore %g1, 0, %o0 877c478bd9Sstevel@tonic-gate 887c478bd9Sstevel@tonic-gate SET_SIZE(kcopy) 897c478bd9Sstevel@tonic-gate 907c478bd9Sstevel@tonic-gate 917c478bd9Sstevel@tonic-gate/* 927c478bd9Sstevel@tonic-gate * Copy a block of storage - must not overlap (from + len <= to). 937c478bd9Sstevel@tonic-gate */ 947c478bd9Sstevel@tonic-gate 957c478bd9Sstevel@tonic-gate ENTRY(bcopy) 967c478bd9Sstevel@tonic-gate 977c478bd9Sstevel@tonic-gate save %sp, -SA(MINFRAME), %sp 98473b13d4Sae clr %o5 ! flag LOFAULT_SET is not set for bcopy 997c478bd9Sstevel@tonic-gate 1007c478bd9Sstevel@tonic-gate.do_copy: 101651ee059Siskreen mov %i1, %g5 ! save dest addr start 102651ee059Siskreen 103651ee059Siskreen mov %i2, %l6 ! save size 104651ee059Siskreen 1057c478bd9Sstevel@tonic-gate cmp %i2, 12 ! for small counts 1067c478bd9Sstevel@tonic-gate blu %ncc, .bytecp ! just copy bytes 1077c478bd9Sstevel@tonic-gate .empty 1087c478bd9Sstevel@tonic-gate 1097c478bd9Sstevel@tonic-gate ! 1107c478bd9Sstevel@tonic-gate ! use aligned transfers where possible 1117c478bd9Sstevel@tonic-gate ! 1127c478bd9Sstevel@tonic-gate xor %i0, %i1, %o4 ! xor from and to address 1137c478bd9Sstevel@tonic-gate btst 7, %o4 ! if lower three bits zero 1147c478bd9Sstevel@tonic-gate bz .aldoubcp ! can align on double boundary 1157c478bd9Sstevel@tonic-gate .empty ! assembler complaints about label 1167c478bd9Sstevel@tonic-gate 1177c478bd9Sstevel@tonic-gate xor %i0, %i1, %o4 ! xor from and to address 1187c478bd9Sstevel@tonic-gate btst 3, %o4 ! if lower two bits zero 1197c478bd9Sstevel@tonic-gate bz .alwordcp ! can align on word boundary 1207c478bd9Sstevel@tonic-gate btst 3, %i0 ! delay slot, from address unaligned? 1217c478bd9Sstevel@tonic-gate ! 1227c478bd9Sstevel@tonic-gate ! use aligned reads and writes where possible 1237c478bd9Sstevel@tonic-gate ! this differs from wordcp in that it copes 1247c478bd9Sstevel@tonic-gate ! with odd alignment between source and destnation 1257c478bd9Sstevel@tonic-gate ! using word reads and writes with the proper shifts 1267c478bd9Sstevel@tonic-gate ! in between to align transfers to and from memory 1277c478bd9Sstevel@tonic-gate ! i0 - src address, i1 - dest address, i2 - count 1287c478bd9Sstevel@tonic-gate ! i3, i4 - tmps for used generating complete word 1297c478bd9Sstevel@tonic-gate ! i5 (word to write) 1307c478bd9Sstevel@tonic-gate ! l0 size in bits of upper part of source word (US) 1317c478bd9Sstevel@tonic-gate ! l1 size in bits of lower part of source word (LS = 32 - US) 1327c478bd9Sstevel@tonic-gate ! l2 size in bits of upper part of destination word (UD) 1337c478bd9Sstevel@tonic-gate ! l3 size in bits of lower part of destination word (LD = 32 - UD) 1347c478bd9Sstevel@tonic-gate ! l4 number of bytes leftover after aligned transfers complete 1357c478bd9Sstevel@tonic-gate ! l5 the number 32 1367c478bd9Sstevel@tonic-gate ! 1377c478bd9Sstevel@tonic-gate mov 32, %l5 ! load an oft-needed constant 1387c478bd9Sstevel@tonic-gate bz .align_dst_only 1397c478bd9Sstevel@tonic-gate btst 3, %i1 ! is destnation address aligned? 1407c478bd9Sstevel@tonic-gate clr %i4 ! clear registers used in either case 1417c478bd9Sstevel@tonic-gate bz .align_src_only 1427c478bd9Sstevel@tonic-gate clr %l0 1437c478bd9Sstevel@tonic-gate ! 1447c478bd9Sstevel@tonic-gate ! both source and destination addresses are unaligned 1457c478bd9Sstevel@tonic-gate ! 1467c478bd9Sstevel@tonic-gate1: ! align source 1477c478bd9Sstevel@tonic-gate ldub [%i0], %i3 ! read a byte from source address 1487c478bd9Sstevel@tonic-gate add %i0, 1, %i0 ! increment source address 1497c478bd9Sstevel@tonic-gate or %i4, %i3, %i4 ! or in with previous bytes (if any) 1507c478bd9Sstevel@tonic-gate btst 3, %i0 ! is source aligned? 1517c478bd9Sstevel@tonic-gate add %l0, 8, %l0 ! increment size of upper source (US) 1527c478bd9Sstevel@tonic-gate bnz,a 1b 1537c478bd9Sstevel@tonic-gate sll %i4, 8, %i4 ! make room for next byte 1547c478bd9Sstevel@tonic-gate 1557c478bd9Sstevel@tonic-gate sub %l5, %l0, %l1 ! generate shift left count (LS) 1567c478bd9Sstevel@tonic-gate sll %i4, %l1, %i4 ! prepare to get rest 1577c478bd9Sstevel@tonic-gate ld [%i0], %i3 ! read a word 1587c478bd9Sstevel@tonic-gate add %i0, 4, %i0 ! increment source address 1597c478bd9Sstevel@tonic-gate srl %i3, %l0, %i5 ! upper src bits into lower dst bits 1607c478bd9Sstevel@tonic-gate or %i4, %i5, %i5 ! merge 1617c478bd9Sstevel@tonic-gate mov 24, %l3 ! align destination 1627c478bd9Sstevel@tonic-gate1: 1637c478bd9Sstevel@tonic-gate srl %i5, %l3, %i4 ! prepare to write a single byte 1647c478bd9Sstevel@tonic-gate stb %i4, [%i1] ! write a byte 1657c478bd9Sstevel@tonic-gate add %i1, 1, %i1 ! increment destination address 1667c478bd9Sstevel@tonic-gate sub %i2, 1, %i2 ! decrement count 1677c478bd9Sstevel@tonic-gate btst 3, %i1 ! is destination aligned? 1687c478bd9Sstevel@tonic-gate bnz,a 1b 1697c478bd9Sstevel@tonic-gate sub %l3, 8, %l3 ! delay slot, decrement shift count (LD) 1707c478bd9Sstevel@tonic-gate sub %l5, %l3, %l2 ! generate shift left count (UD) 1717c478bd9Sstevel@tonic-gate sll %i5, %l2, %i5 ! move leftover into upper bytes 1727c478bd9Sstevel@tonic-gate cmp %l2, %l0 ! cmp # reqd to fill dst w old src left 1737c478bd9Sstevel@tonic-gate bgu %ncc, .more_needed ! need more to fill than we have 1747c478bd9Sstevel@tonic-gate nop 1757c478bd9Sstevel@tonic-gate 1767c478bd9Sstevel@tonic-gate sll %i3, %l1, %i3 ! clear upper used byte(s) 1777c478bd9Sstevel@tonic-gate srl %i3, %l1, %i3 1787c478bd9Sstevel@tonic-gate ! get the odd bytes between alignments 1797c478bd9Sstevel@tonic-gate sub %l0, %l2, %l0 ! regenerate shift count 1807c478bd9Sstevel@tonic-gate sub %l5, %l0, %l1 ! generate new shift left count (LS) 1817c478bd9Sstevel@tonic-gate and %i2, 3, %l4 ! must do remaining bytes if count%4 > 0 1827c478bd9Sstevel@tonic-gate andn %i2, 3, %i2 ! # of aligned bytes that can be moved 1837c478bd9Sstevel@tonic-gate srl %i3, %l0, %i4 1847c478bd9Sstevel@tonic-gate or %i5, %i4, %i5 1857c478bd9Sstevel@tonic-gate st %i5, [%i1] ! write a word 1867c478bd9Sstevel@tonic-gate subcc %i2, 4, %i2 ! decrement count 1877c478bd9Sstevel@tonic-gate bz %ncc, .unalign_out 1887c478bd9Sstevel@tonic-gate add %i1, 4, %i1 ! increment destination address 1897c478bd9Sstevel@tonic-gate 1907c478bd9Sstevel@tonic-gate b 2f 1917c478bd9Sstevel@tonic-gate sll %i3, %l1, %i5 ! get leftover into upper bits 1927c478bd9Sstevel@tonic-gate.more_needed: 1937c478bd9Sstevel@tonic-gate sll %i3, %l0, %i3 ! save remaining byte(s) 1947c478bd9Sstevel@tonic-gate srl %i3, %l0, %i3 1957c478bd9Sstevel@tonic-gate sub %l2, %l0, %l1 ! regenerate shift count 1967c478bd9Sstevel@tonic-gate sub %l5, %l1, %l0 ! generate new shift left count 1977c478bd9Sstevel@tonic-gate sll %i3, %l1, %i4 ! move to fill empty space 1987c478bd9Sstevel@tonic-gate b 3f 1997c478bd9Sstevel@tonic-gate or %i5, %i4, %i5 ! merge to complete word 2007c478bd9Sstevel@tonic-gate ! 2017c478bd9Sstevel@tonic-gate ! the source address is aligned and destination is not 2027c478bd9Sstevel@tonic-gate ! 2037c478bd9Sstevel@tonic-gate.align_dst_only: 2047c478bd9Sstevel@tonic-gate ld [%i0], %i4 ! read a word 2057c478bd9Sstevel@tonic-gate add %i0, 4, %i0 ! increment source address 2067c478bd9Sstevel@tonic-gate mov 24, %l0 ! initial shift alignment count 2077c478bd9Sstevel@tonic-gate1: 2087c478bd9Sstevel@tonic-gate srl %i4, %l0, %i3 ! prepare to write a single byte 2097c478bd9Sstevel@tonic-gate stb %i3, [%i1] ! write a byte 2107c478bd9Sstevel@tonic-gate add %i1, 1, %i1 ! increment destination address 2117c478bd9Sstevel@tonic-gate sub %i2, 1, %i2 ! decrement count 2127c478bd9Sstevel@tonic-gate btst 3, %i1 ! is destination aligned? 2137c478bd9Sstevel@tonic-gate bnz,a 1b 2147c478bd9Sstevel@tonic-gate sub %l0, 8, %l0 ! delay slot, decrement shift count 2157c478bd9Sstevel@tonic-gate.xfer: 2167c478bd9Sstevel@tonic-gate sub %l5, %l0, %l1 ! generate shift left count 2177c478bd9Sstevel@tonic-gate sll %i4, %l1, %i5 ! get leftover 2187c478bd9Sstevel@tonic-gate3: 2197c478bd9Sstevel@tonic-gate and %i2, 3, %l4 ! must do remaining bytes if count%4 > 0 2207c478bd9Sstevel@tonic-gate andn %i2, 3, %i2 ! # of aligned bytes that can be moved 2217c478bd9Sstevel@tonic-gate2: 2227c478bd9Sstevel@tonic-gate ld [%i0], %i3 ! read a source word 2237c478bd9Sstevel@tonic-gate add %i0, 4, %i0 ! increment source address 2247c478bd9Sstevel@tonic-gate srl %i3, %l0, %i4 ! upper src bits into lower dst bits 2257c478bd9Sstevel@tonic-gate or %i5, %i4, %i5 ! merge with upper dest bits (leftover) 2267c478bd9Sstevel@tonic-gate st %i5, [%i1] ! write a destination word 2277c478bd9Sstevel@tonic-gate subcc %i2, 4, %i2 ! decrement count 2287c478bd9Sstevel@tonic-gate bz %ncc, .unalign_out ! check if done 2297c478bd9Sstevel@tonic-gate add %i1, 4, %i1 ! increment destination address 2307c478bd9Sstevel@tonic-gate b 2b ! loop 2317c478bd9Sstevel@tonic-gate sll %i3, %l1, %i5 ! get leftover 2327c478bd9Sstevel@tonic-gate.unalign_out: 2337c478bd9Sstevel@tonic-gate tst %l4 ! any bytes leftover? 2347c478bd9Sstevel@tonic-gate bz %ncc, .cpdone 2357c478bd9Sstevel@tonic-gate .empty ! allow next instruction in delay slot 2367c478bd9Sstevel@tonic-gate1: 2377c478bd9Sstevel@tonic-gate sub %l0, 8, %l0 ! decrement shift 2387c478bd9Sstevel@tonic-gate srl %i3, %l0, %i4 ! upper src byte into lower dst byte 2397c478bd9Sstevel@tonic-gate stb %i4, [%i1] ! write a byte 2407c478bd9Sstevel@tonic-gate subcc %l4, 1, %l4 ! decrement count 2417c478bd9Sstevel@tonic-gate bz %ncc, .cpdone ! done? 2427c478bd9Sstevel@tonic-gate add %i1, 1, %i1 ! increment destination 2437c478bd9Sstevel@tonic-gate tst %l0 ! any more previously read bytes 2447c478bd9Sstevel@tonic-gate bnz %ncc, 1b ! we have leftover bytes 2457c478bd9Sstevel@tonic-gate mov %l4, %i2 ! delay slot, mv cnt where dbytecp wants 2467c478bd9Sstevel@tonic-gate b .dbytecp ! let dbytecp do the rest 2477c478bd9Sstevel@tonic-gate sub %i0, %i1, %i0 ! i0 gets the difference of src and dst 2487c478bd9Sstevel@tonic-gate ! 2497c478bd9Sstevel@tonic-gate ! the destination address is aligned and the source is not 2507c478bd9Sstevel@tonic-gate ! 2517c478bd9Sstevel@tonic-gate.align_src_only: 2527c478bd9Sstevel@tonic-gate ldub [%i0], %i3 ! read a byte from source address 2537c478bd9Sstevel@tonic-gate add %i0, 1, %i0 ! increment source address 2547c478bd9Sstevel@tonic-gate or %i4, %i3, %i4 ! or in with previous bytes (if any) 2557c478bd9Sstevel@tonic-gate btst 3, %i0 ! is source aligned? 2567c478bd9Sstevel@tonic-gate add %l0, 8, %l0 ! increment shift count (US) 2577c478bd9Sstevel@tonic-gate bnz,a .align_src_only 2587c478bd9Sstevel@tonic-gate sll %i4, 8, %i4 ! make room for next byte 2597c478bd9Sstevel@tonic-gate b,a .xfer 2607c478bd9Sstevel@tonic-gate ! 2617c478bd9Sstevel@tonic-gate ! if from address unaligned for double-word moves, 2627c478bd9Sstevel@tonic-gate ! move bytes till it is, if count is < 56 it could take 2637c478bd9Sstevel@tonic-gate ! longer to align the thing than to do the transfer 2647c478bd9Sstevel@tonic-gate ! in word size chunks right away 2657c478bd9Sstevel@tonic-gate ! 2667c478bd9Sstevel@tonic-gate.aldoubcp: 2677c478bd9Sstevel@tonic-gate cmp %i2, 56 ! if count < 56, use wordcp, it takes 2687c478bd9Sstevel@tonic-gate blu,a %ncc, .alwordcp ! longer to align doubles than words 2697c478bd9Sstevel@tonic-gate mov 3, %o0 ! mask for word alignment 2707c478bd9Sstevel@tonic-gate call .alignit ! copy bytes until aligned 2717c478bd9Sstevel@tonic-gate mov 7, %o0 ! mask for double alignment 2727c478bd9Sstevel@tonic-gate ! 2737c478bd9Sstevel@tonic-gate ! source and destination are now double-word aligned 2747c478bd9Sstevel@tonic-gate ! i3 has aligned count returned by alignit 2757c478bd9Sstevel@tonic-gate ! 2767c478bd9Sstevel@tonic-gate and %i2, 7, %i2 ! unaligned leftover count 2777c478bd9Sstevel@tonic-gate sub %i0, %i1, %i0 ! i0 gets the difference of src and dst 2787c478bd9Sstevel@tonic-gate5: 2797c478bd9Sstevel@tonic-gate ldx [%i0+%i1], %o4 ! read from address 2807c478bd9Sstevel@tonic-gate stx %o4, [%i1] ! write at destination address 2817c478bd9Sstevel@tonic-gate subcc %i3, 8, %i3 ! dec count 2827c478bd9Sstevel@tonic-gate bgu %ncc, 5b 2837c478bd9Sstevel@tonic-gate add %i1, 8, %i1 ! delay slot, inc to address 2847c478bd9Sstevel@tonic-gate cmp %i2, 4 ! see if we can copy a word 2857c478bd9Sstevel@tonic-gate blu %ncc, .dbytecp ! if 3 or less bytes use bytecp 2867c478bd9Sstevel@tonic-gate .empty 2877c478bd9Sstevel@tonic-gate ! 2887c478bd9Sstevel@tonic-gate ! for leftover bytes we fall into wordcp, if needed 2897c478bd9Sstevel@tonic-gate ! 2907c478bd9Sstevel@tonic-gate.wordcp: 2917c478bd9Sstevel@tonic-gate and %i2, 3, %i2 ! unaligned leftover count 2927c478bd9Sstevel@tonic-gate5: 2937c478bd9Sstevel@tonic-gate ld [%i0+%i1], %o4 ! read from address 2947c478bd9Sstevel@tonic-gate st %o4, [%i1] ! write at destination address 2957c478bd9Sstevel@tonic-gate subcc %i3, 4, %i3 ! dec count 2967c478bd9Sstevel@tonic-gate bgu %ncc, 5b 2977c478bd9Sstevel@tonic-gate add %i1, 4, %i1 ! delay slot, inc to address 2987c478bd9Sstevel@tonic-gate b,a .dbytecp 2997c478bd9Sstevel@tonic-gate 3007c478bd9Sstevel@tonic-gate ! we come here to align copies on word boundaries 3017c478bd9Sstevel@tonic-gate.alwordcp: 3027c478bd9Sstevel@tonic-gate call .alignit ! go word-align it 3037c478bd9Sstevel@tonic-gate mov 3, %o0 ! bits that must be zero to be aligned 3047c478bd9Sstevel@tonic-gate b .wordcp 3057c478bd9Sstevel@tonic-gate sub %i0, %i1, %i0 ! i0 gets the difference of src and dst 3067c478bd9Sstevel@tonic-gate 3077c478bd9Sstevel@tonic-gate ! 3087c478bd9Sstevel@tonic-gate ! byte copy, works with any alignment 3097c478bd9Sstevel@tonic-gate ! 3107c478bd9Sstevel@tonic-gate.bytecp: 3117c478bd9Sstevel@tonic-gate b .dbytecp 3127c478bd9Sstevel@tonic-gate sub %i0, %i1, %i0 ! i0 gets difference of src and dst 3137c478bd9Sstevel@tonic-gate 3147c478bd9Sstevel@tonic-gate ! 3157c478bd9Sstevel@tonic-gate ! differenced byte copy, works with any alignment 3167c478bd9Sstevel@tonic-gate ! assumes dest in %i1 and (source - dest) in %i0 3177c478bd9Sstevel@tonic-gate ! 3187c478bd9Sstevel@tonic-gate1: 3197c478bd9Sstevel@tonic-gate stb %o4, [%i1] ! write to address 3207c478bd9Sstevel@tonic-gate inc %i1 ! inc to address 3217c478bd9Sstevel@tonic-gate.dbytecp: 3227c478bd9Sstevel@tonic-gate deccc %i2 ! dec count 3237c478bd9Sstevel@tonic-gate bgeu,a %ncc, 1b ! loop till done 3247c478bd9Sstevel@tonic-gate ldub [%i0+%i1], %o4 ! read from address 3257c478bd9Sstevel@tonic-gate.cpdone: 326473b13d4Sae membar #Sync ! sync error barrier 327473b13d4Sae ! Restore t_lofault handler, if came here from kcopy(). 328473b13d4Sae tst %o5 329473b13d4Sae bz %ncc, 1f 330473b13d4Sae andn %o5, LOFAULT_SET, %o5 331473b13d4Sae stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault 332473b13d4Sae1: 333651ee059Siskreen mov %g5, %o0 ! copy dest address 334651ee059Siskreen call sync_icache 335651ee059Siskreen mov %l6, %o1 ! saved size 3367c478bd9Sstevel@tonic-gate ret 3377c478bd9Sstevel@tonic-gate restore %g0, 0, %o0 ! return (0) 3387c478bd9Sstevel@tonic-gate 3397c478bd9Sstevel@tonic-gate/* 3407c478bd9Sstevel@tonic-gate * Common code used to align transfers on word and doubleword 3417c478bd9Sstevel@tonic-gate * boudaries. Aligns source and destination and returns a count 3427c478bd9Sstevel@tonic-gate * of aligned bytes to transfer in %i3 3437c478bd9Sstevel@tonic-gate */ 3447c478bd9Sstevel@tonic-gate1: 3457c478bd9Sstevel@tonic-gate inc %i0 ! inc from 3467c478bd9Sstevel@tonic-gate stb %o4, [%i1] ! write a byte 3477c478bd9Sstevel@tonic-gate inc %i1 ! inc to 3487c478bd9Sstevel@tonic-gate dec %i2 ! dec count 3497c478bd9Sstevel@tonic-gate.alignit: 3507c478bd9Sstevel@tonic-gate btst %o0, %i0 ! %o0 is bit mask to check for alignment 3517c478bd9Sstevel@tonic-gate bnz,a 1b 3527c478bd9Sstevel@tonic-gate ldub [%i0], %o4 ! read next byte 3537c478bd9Sstevel@tonic-gate 3547c478bd9Sstevel@tonic-gate retl 3557c478bd9Sstevel@tonic-gate andn %i2, %o0, %i3 ! return size of aligned bytes 3567c478bd9Sstevel@tonic-gate SET_SIZE(bcopy) 3577c478bd9Sstevel@tonic-gate 3587c478bd9Sstevel@tonic-gate/* 3597c478bd9Sstevel@tonic-gate * Block copy with possibly overlapped operands. 3607c478bd9Sstevel@tonic-gate */ 3617c478bd9Sstevel@tonic-gate 3627c478bd9Sstevel@tonic-gate ENTRY(ovbcopy) 3637c478bd9Sstevel@tonic-gate tst %o2 ! check count 3647c478bd9Sstevel@tonic-gate bgu,a %ncc, 1f ! nothing to do or bad arguments 3657c478bd9Sstevel@tonic-gate subcc %o0, %o1, %o3 ! difference of from and to address 3667c478bd9Sstevel@tonic-gate 3677c478bd9Sstevel@tonic-gate retl ! return 3687c478bd9Sstevel@tonic-gate nop 3697c478bd9Sstevel@tonic-gate1: 3707c478bd9Sstevel@tonic-gate bneg,a %ncc, 2f 3717c478bd9Sstevel@tonic-gate neg %o3 ! if < 0, make it positive 3727c478bd9Sstevel@tonic-gate2: cmp %o2, %o3 ! cmp size and abs(from - to) 3737c478bd9Sstevel@tonic-gate bleu %ncc, bcopy ! if size <= abs(diff): use bcopy, 3747c478bd9Sstevel@tonic-gate .empty ! no overlap 3757c478bd9Sstevel@tonic-gate cmp %o0, %o1 ! compare from and to addresses 3767c478bd9Sstevel@tonic-gate blu %ncc, .ov_bkwd ! if from < to, copy backwards 3777c478bd9Sstevel@tonic-gate nop 3787c478bd9Sstevel@tonic-gate ! 3797c478bd9Sstevel@tonic-gate ! Copy forwards. 3807c478bd9Sstevel@tonic-gate ! 3817c478bd9Sstevel@tonic-gate.ov_fwd: 3827c478bd9Sstevel@tonic-gate ldub [%o0], %o3 ! read from address 3837c478bd9Sstevel@tonic-gate inc %o0 ! inc from address 3847c478bd9Sstevel@tonic-gate stb %o3, [%o1] ! write to address 3857c478bd9Sstevel@tonic-gate deccc %o2 ! dec count 3867c478bd9Sstevel@tonic-gate bgu %ncc, .ov_fwd ! loop till done 3877c478bd9Sstevel@tonic-gate inc %o1 ! inc to address 3887c478bd9Sstevel@tonic-gate 3897c478bd9Sstevel@tonic-gate retl ! return 3907c478bd9Sstevel@tonic-gate nop 3917c478bd9Sstevel@tonic-gate ! 3927c478bd9Sstevel@tonic-gate ! Copy backwards. 3937c478bd9Sstevel@tonic-gate ! 3947c478bd9Sstevel@tonic-gate.ov_bkwd: 3957c478bd9Sstevel@tonic-gate deccc %o2 ! dec count 3967c478bd9Sstevel@tonic-gate ldub [%o0 + %o2], %o3 ! get byte at end of src 3977c478bd9Sstevel@tonic-gate bgu %ncc, .ov_bkwd ! loop till done 3987c478bd9Sstevel@tonic-gate stb %o3, [%o1 + %o2] ! delay slot, store at end of dst 3997c478bd9Sstevel@tonic-gate 4007c478bd9Sstevel@tonic-gate retl ! return 4017c478bd9Sstevel@tonic-gate nop 4027c478bd9Sstevel@tonic-gate SET_SIZE(ovbcopy) 4037c478bd9Sstevel@tonic-gate 4047c478bd9Sstevel@tonic-gate/* 4057c478bd9Sstevel@tonic-gate * hwblkpagecopy() 4067c478bd9Sstevel@tonic-gate * 4077c478bd9Sstevel@tonic-gate * Copies exactly one page. This routine assumes the caller (ppcopy) 4087c478bd9Sstevel@tonic-gate * has already disabled kernel preemption and has checked 4097c478bd9Sstevel@tonic-gate * use_hw_bcopy. 4107c478bd9Sstevel@tonic-gate */ 4117c478bd9Sstevel@tonic-gate ENTRY(hwblkpagecopy) 4127c478bd9Sstevel@tonic-gate save %sp, -SA(MINFRAME), %sp 4137c478bd9Sstevel@tonic-gate 4147c478bd9Sstevel@tonic-gate ! %i0 - source address (arg) 4157c478bd9Sstevel@tonic-gate ! %i1 - destination address (arg) 4167c478bd9Sstevel@tonic-gate ! %i2 - length of region (not arg) 4177c478bd9Sstevel@tonic-gate 4187c478bd9Sstevel@tonic-gate set PAGESIZE, %i2 419651ee059Siskreen mov %i1, %o0 ! store destination address for flushing 4207c478bd9Sstevel@tonic-gate 4217c478bd9Sstevel@tonic-gate /* 422*55fea89dSDan Cross * Copying exactly one page and PAGESIZE is in mutliple of 0x80. 4237c478bd9Sstevel@tonic-gate */ 4247c478bd9Sstevel@tonic-gate1: 4257c478bd9Sstevel@tonic-gate ldx [%i0+0x0], %l0 4267c478bd9Sstevel@tonic-gate ldx [%i0+0x8], %l1 4277c478bd9Sstevel@tonic-gate ldx [%i0+0x10], %l2 4287c478bd9Sstevel@tonic-gate ldx [%i0+0x18], %l3 4297c478bd9Sstevel@tonic-gate ldx [%i0+0x20], %l4 4307c478bd9Sstevel@tonic-gate ldx [%i0+0x28], %l5 4317c478bd9Sstevel@tonic-gate ldx [%i0+0x30], %l6 4327c478bd9Sstevel@tonic-gate ldx [%i0+0x38], %l7 4337c478bd9Sstevel@tonic-gate stx %l0, [%i1+0x0] 4347c478bd9Sstevel@tonic-gate stx %l1, [%i1+0x8] 4357c478bd9Sstevel@tonic-gate stx %l2, [%i1+0x10] 4367c478bd9Sstevel@tonic-gate stx %l3, [%i1+0x18] 4377c478bd9Sstevel@tonic-gate stx %l4, [%i1+0x20] 4387c478bd9Sstevel@tonic-gate stx %l5, [%i1+0x28] 4397c478bd9Sstevel@tonic-gate stx %l6, [%i1+0x30] 4407c478bd9Sstevel@tonic-gate stx %l7, [%i1+0x38] 4417c478bd9Sstevel@tonic-gate 4427c478bd9Sstevel@tonic-gate ldx [%i0+0x40], %l0 4437c478bd9Sstevel@tonic-gate ldx [%i0+0x48], %l1 4447c478bd9Sstevel@tonic-gate ldx [%i0+0x50], %l2 4457c478bd9Sstevel@tonic-gate ldx [%i0+0x58], %l3 4467c478bd9Sstevel@tonic-gate ldx [%i0+0x60], %l4 4477c478bd9Sstevel@tonic-gate ldx [%i0+0x68], %l5 4487c478bd9Sstevel@tonic-gate ldx [%i0+0x70], %l6 4497c478bd9Sstevel@tonic-gate ldx [%i0+0x78], %l7 4507c478bd9Sstevel@tonic-gate stx %l0, [%i1+0x40] 4517c478bd9Sstevel@tonic-gate stx %l1, [%i1+0x48] 4527c478bd9Sstevel@tonic-gate stx %l2, [%i1+0x50] 4537c478bd9Sstevel@tonic-gate stx %l3, [%i1+0x58] 4547c478bd9Sstevel@tonic-gate stx %l4, [%i1+0x60] 4557c478bd9Sstevel@tonic-gate stx %l5, [%i1+0x68] 4567c478bd9Sstevel@tonic-gate stx %l6, [%i1+0x70] 4577c478bd9Sstevel@tonic-gate stx %l7, [%i1+0x78] 4587c478bd9Sstevel@tonic-gate 4597c478bd9Sstevel@tonic-gate add %i0, 0x80, %i0 4607c478bd9Sstevel@tonic-gate subcc %i2, 0x80, %i2 4617c478bd9Sstevel@tonic-gate bgu,pt %xcc, 1b 4627c478bd9Sstevel@tonic-gate add %i1, 0x80, %i1 4637c478bd9Sstevel@tonic-gate 464651ee059Siskreen ! %o0 contains the dest. address 465651ee059Siskreen set PAGESIZE, %o1 466651ee059Siskreen call sync_icache 467651ee059Siskreen nop 468651ee059Siskreen 4697c478bd9Sstevel@tonic-gate membar #Sync 4707c478bd9Sstevel@tonic-gate ret 4717c478bd9Sstevel@tonic-gate restore %g0, 0, %o0 4727c478bd9Sstevel@tonic-gate SET_SIZE(hwblkpagecopy) 4737c478bd9Sstevel@tonic-gate 4747c478bd9Sstevel@tonic-gate 4757c478bd9Sstevel@tonic-gate/* 4767c478bd9Sstevel@tonic-gate * Transfer data to and from user space - 4777c478bd9Sstevel@tonic-gate * Note that these routines can cause faults 4787c478bd9Sstevel@tonic-gate * It is assumed that the kernel has nothing at 4797c478bd9Sstevel@tonic-gate * less than KERNELBASE in the virtual address space. 4807c478bd9Sstevel@tonic-gate * 4817c478bd9Sstevel@tonic-gate * Note that copyin(9F) and copyout(9F) are part of the 4827c478bd9Sstevel@tonic-gate * DDI/DKI which specifies that they return '-1' on "errors." 4837c478bd9Sstevel@tonic-gate * 4847c478bd9Sstevel@tonic-gate * Sigh. 4857c478bd9Sstevel@tonic-gate * 4867c478bd9Sstevel@tonic-gate * So there's two extremely similar routines - xcopyin() and xcopyout() 4877c478bd9Sstevel@tonic-gate * which return the errno that we've faithfully computed. This 4887c478bd9Sstevel@tonic-gate * allows other callers (e.g. uiomove(9F)) to work correctly. 4897c478bd9Sstevel@tonic-gate * Given that these are used pretty heavily, we expand the calling 4907c478bd9Sstevel@tonic-gate * sequences inline for all flavours (rather than making wrappers). 4917c478bd9Sstevel@tonic-gate * 4927c478bd9Sstevel@tonic-gate * There are also stub routines for xcopyout_little and xcopyin_little, 4937c478bd9Sstevel@tonic-gate * which currently are intended to handle requests of <= 16 bytes from 4947c478bd9Sstevel@tonic-gate * do_unaligned. Future enhancement to make them handle 8k pages efficiently 4957c478bd9Sstevel@tonic-gate * is left as an exercise... 4967c478bd9Sstevel@tonic-gate */ 4977c478bd9Sstevel@tonic-gate 4987c478bd9Sstevel@tonic-gate/* 4997c478bd9Sstevel@tonic-gate * Copy user data to kernel space (copyOP/xcopyOP/copyOP_noerr) 5007c478bd9Sstevel@tonic-gate * 5017c478bd9Sstevel@tonic-gate * General theory of operation: 5027c478bd9Sstevel@tonic-gate * 5037c478bd9Sstevel@tonic-gate * None of the copyops routines grab a window. 5047c478bd9Sstevel@tonic-gate * 5057c478bd9Sstevel@tonic-gate * Flow: 5067c478bd9Sstevel@tonic-gate * 5077c478bd9Sstevel@tonic-gate * If count == zero return zero. 5087c478bd9Sstevel@tonic-gate * 5097c478bd9Sstevel@tonic-gate * Store the previous lo_fault handler into %g6. 5107c478bd9Sstevel@tonic-gate * Place our secondary lofault handler into %g5. 5117c478bd9Sstevel@tonic-gate * Place the address of our fault handler into %o3. 5127c478bd9Sstevel@tonic-gate * 5137c478bd9Sstevel@tonic-gate * If count is less than or equal to SMALL_LIMIT (7) we 5147c478bd9Sstevel@tonic-gate * always do a byte for byte copy. 5157c478bd9Sstevel@tonic-gate * 5167c478bd9Sstevel@tonic-gate * If count is > SMALL_LIMIT, we check the alignment of the input 5177c478bd9Sstevel@tonic-gate * and output pointers. We store -count in %o3, we store the number 5187c478bd9Sstevel@tonic-gate * of chunks (8, 4, 2 or 1 byte) operated on in our basic copy loop 5197c478bd9Sstevel@tonic-gate * in %o2. Following this we branch to the appropriate copy loop and 5207c478bd9Sstevel@tonic-gate * copy that many chunks. Since we've been adding the chunk size 5217c478bd9Sstevel@tonic-gate * to %o3 each time through as well as decrementing %o2, we can tell 5227c478bd9Sstevel@tonic-gate * if any data is is left to be copied by examining %o3. If that is 5237c478bd9Sstevel@tonic-gate * zero, we're done and can go home. If not, we figure out what the 5247c478bd9Sstevel@tonic-gate * largest chunk size left to be copied is and branch to that copy 5257c478bd9Sstevel@tonic-gate * loop unless there's only one byte left. We load that as we're 5267c478bd9Sstevel@tonic-gate * branching to code that stores it just before we return. 5277c478bd9Sstevel@tonic-gate * 5287c478bd9Sstevel@tonic-gate * Fault handlers are invoked if we reference memory that has no 5297c478bd9Sstevel@tonic-gate * current mapping. All forms share the same copyio_fault handler. 5307c478bd9Sstevel@tonic-gate * This routine handles fixing up the stack and general housecleaning. 5317c478bd9Sstevel@tonic-gate * Each copy operation has a simple fault handler that is then called 5327c478bd9Sstevel@tonic-gate * to do the work specific to the invidual operation. The handler 5337c478bd9Sstevel@tonic-gate * for copyOP and xcopyOP are found at the end of individual function. 5347c478bd9Sstevel@tonic-gate * The handlers for xcopyOP_little are found at the end of xcopyin_little. 5357c478bd9Sstevel@tonic-gate * The handlers for copyOP_noerr are found at the end of copyin_noerr. 5367c478bd9Sstevel@tonic-gate */ 5377c478bd9Sstevel@tonic-gate 5387c478bd9Sstevel@tonic-gate/* 5397c478bd9Sstevel@tonic-gate * Copy kernel data to user space (copyout/xcopyout/xcopyout_little). 5407c478bd9Sstevel@tonic-gate */ 5417c478bd9Sstevel@tonic-gate 5427c478bd9Sstevel@tonic-gate/* 5437c478bd9Sstevel@tonic-gate * We save the arguments in the following registers in case of a fault: 5447c478bd9Sstevel@tonic-gate * kaddr - %g2 5457c478bd9Sstevel@tonic-gate * uaddr - %g3 5467c478bd9Sstevel@tonic-gate * count - %g4 5477c478bd9Sstevel@tonic-gate */ 5487c478bd9Sstevel@tonic-gate#define SAVE_SRC %g2 5497c478bd9Sstevel@tonic-gate#define SAVE_DST %g3 5507c478bd9Sstevel@tonic-gate#define SAVE_COUNT %g4 5517c478bd9Sstevel@tonic-gate 5527c478bd9Sstevel@tonic-gate#define REAL_LOFAULT %g5 5537c478bd9Sstevel@tonic-gate#define SAVED_LOFAULT %g6 5547c478bd9Sstevel@tonic-gate 5557c478bd9Sstevel@tonic-gate/* 556*55fea89dSDan Cross * Generic copyio fault handler. This is the first line of defense when a 5577c478bd9Sstevel@tonic-gate * fault occurs in (x)copyin/(x)copyout. In order for this to function 5587c478bd9Sstevel@tonic-gate * properly, the value of the 'real' lofault handler should be in REAL_LOFAULT. 5597c478bd9Sstevel@tonic-gate * This allows us to share common code for all the flavors of the copy 5607c478bd9Sstevel@tonic-gate * operations, including the _noerr versions. 5617c478bd9Sstevel@tonic-gate * 5627c478bd9Sstevel@tonic-gate * Note that this function will restore the original input parameters before 5637c478bd9Sstevel@tonic-gate * calling REAL_LOFAULT. So the real handler can vector to the appropriate 5647c478bd9Sstevel@tonic-gate * member of the t_copyop structure, if needed. 5657c478bd9Sstevel@tonic-gate */ 5667c478bd9Sstevel@tonic-gate ENTRY(copyio_fault) 5677c478bd9Sstevel@tonic-gate membar #Sync 5687c478bd9Sstevel@tonic-gate stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT] ! restore old t_lofault 5697c478bd9Sstevel@tonic-gate 5707c478bd9Sstevel@tonic-gate mov SAVE_SRC, %o0 5717c478bd9Sstevel@tonic-gate mov SAVE_DST, %o1 5727c478bd9Sstevel@tonic-gate jmp REAL_LOFAULT 5737c478bd9Sstevel@tonic-gate mov SAVE_COUNT, %o2 5747c478bd9Sstevel@tonic-gate SET_SIZE(copyio_fault) 5757c478bd9Sstevel@tonic-gate 5767c478bd9Sstevel@tonic-gate ENTRY(copyout) 5777c478bd9Sstevel@tonic-gate sethi %hi(.copyout_err), REAL_LOFAULT 5787c478bd9Sstevel@tonic-gate or REAL_LOFAULT, %lo(.copyout_err), REAL_LOFAULT 5797c478bd9Sstevel@tonic-gate 5807c478bd9Sstevel@tonic-gate.do_copyout: 5817c478bd9Sstevel@tonic-gate ! 5827c478bd9Sstevel@tonic-gate ! Check the length and bail if zero. 5837c478bd9Sstevel@tonic-gate ! 5847c478bd9Sstevel@tonic-gate tst %o2 5857c478bd9Sstevel@tonic-gate bnz,pt %ncc, 1f 5867c478bd9Sstevel@tonic-gate nop 5877c478bd9Sstevel@tonic-gate retl 5887c478bd9Sstevel@tonic-gate clr %o0 5897c478bd9Sstevel@tonic-gate1: 5907c478bd9Sstevel@tonic-gate sethi %hi(copyio_fault), %o3 5917c478bd9Sstevel@tonic-gate ldn [THREAD_REG + T_LOFAULT], SAVED_LOFAULT 5927c478bd9Sstevel@tonic-gate or %o3, %lo(copyio_fault), %o3 5937c478bd9Sstevel@tonic-gate membar #Sync 5947c478bd9Sstevel@tonic-gate stn %o3, [THREAD_REG + T_LOFAULT] 5957c478bd9Sstevel@tonic-gate 5967c478bd9Sstevel@tonic-gate mov %o0, SAVE_SRC 5977c478bd9Sstevel@tonic-gate mov %o1, SAVE_DST 5987c478bd9Sstevel@tonic-gate mov %o2, SAVE_COUNT 5997c478bd9Sstevel@tonic-gate 6007c478bd9Sstevel@tonic-gate ! 6017c478bd9Sstevel@tonic-gate ! Check to see if we're more than SMALL_LIMIT (7 bytes). 6027c478bd9Sstevel@tonic-gate ! Run in leaf mode, using the %o regs as our input regs. 6037c478bd9Sstevel@tonic-gate ! 6047c478bd9Sstevel@tonic-gate subcc %o2, SMALL_LIMIT, %o3 6057c478bd9Sstevel@tonic-gate bgu,a,pt %ncc, .dco_ns 6067c478bd9Sstevel@tonic-gate or %o0, %o1, %o3 6077c478bd9Sstevel@tonic-gate 6087c478bd9Sstevel@tonic-gate.dcobcp: 6097c478bd9Sstevel@tonic-gate sub %g0, %o2, %o3 ! negate count 6107c478bd9Sstevel@tonic-gate add %o0, %o2, %o0 ! make %o0 point at the end 6117c478bd9Sstevel@tonic-gate add %o1, %o2, %o1 ! make %o1 point at the end 6127c478bd9Sstevel@tonic-gate ba,pt %ncc, .dcocl 6137c478bd9Sstevel@tonic-gate ldub [%o0 + %o3], %o4 ! load first byte 6147c478bd9Sstevel@tonic-gate ! 6157c478bd9Sstevel@tonic-gate ! %o0 and %o2 point at the end and remain pointing at the end 6167c478bd9Sstevel@tonic-gate ! of their buffers. We pull things out by adding %o3 (which is 6177c478bd9Sstevel@tonic-gate ! the negation of the length) to the buffer end which gives us 6187c478bd9Sstevel@tonic-gate ! the curent location in the buffers. By incrementing %o3 we walk 6197c478bd9Sstevel@tonic-gate ! through both buffers without having to bump each buffer's 6207c478bd9Sstevel@tonic-gate ! pointer. A very fast 4 instruction loop. 6217c478bd9Sstevel@tonic-gate ! 6227c478bd9Sstevel@tonic-gate .align 16 6237c478bd9Sstevel@tonic-gate.dcocl: 6247c478bd9Sstevel@tonic-gate stba %o4, [%o1 + %o3]ASI_USER 6257c478bd9Sstevel@tonic-gate inccc %o3 6267c478bd9Sstevel@tonic-gate bl,a,pt %ncc, .dcocl 6277c478bd9Sstevel@tonic-gate ldub [%o0 + %o3], %o4 6287c478bd9Sstevel@tonic-gate ! 6297c478bd9Sstevel@tonic-gate ! We're done. Go home. 6307c478bd9Sstevel@tonic-gate ! 6317c478bd9Sstevel@tonic-gate membar #Sync 6327c478bd9Sstevel@tonic-gate stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT] 6337c478bd9Sstevel@tonic-gate retl 6347c478bd9Sstevel@tonic-gate clr %o0 6357c478bd9Sstevel@tonic-gate ! 6367c478bd9Sstevel@tonic-gate ! Try aligned copies from here. 6377c478bd9Sstevel@tonic-gate ! 6387c478bd9Sstevel@tonic-gate.dco_ns: 6397c478bd9Sstevel@tonic-gate ! %o0 = kernel addr (to be copied from) 6407c478bd9Sstevel@tonic-gate ! %o1 = user addr (to be copied to) 6417c478bd9Sstevel@tonic-gate ! %o2 = length 6427c478bd9Sstevel@tonic-gate ! %o3 = %o1 | %o2 (used for alignment checking) 6437c478bd9Sstevel@tonic-gate ! %o4 is alternate lo_fault 6447c478bd9Sstevel@tonic-gate ! %o5 is original lo_fault 6457c478bd9Sstevel@tonic-gate ! 6467c478bd9Sstevel@tonic-gate ! See if we're single byte aligned. If we are, check the 6477c478bd9Sstevel@tonic-gate ! limit for single byte copies. If we're smaller or equal, 6487c478bd9Sstevel@tonic-gate ! bounce to the byte for byte copy loop. Otherwise do it in 6497c478bd9Sstevel@tonic-gate ! HW (if enabled). 6507c478bd9Sstevel@tonic-gate ! 6517c478bd9Sstevel@tonic-gate btst 1, %o3 6527c478bd9Sstevel@tonic-gate bz,pt %icc, .dcoh8 6537c478bd9Sstevel@tonic-gate btst 7, %o3 6547c478bd9Sstevel@tonic-gate 6557c478bd9Sstevel@tonic-gate ba .dcobcp 6567c478bd9Sstevel@tonic-gate nop 6577c478bd9Sstevel@tonic-gate.dcoh8: 6587c478bd9Sstevel@tonic-gate ! 6597c478bd9Sstevel@tonic-gate ! 8 byte aligned? 6607c478bd9Sstevel@tonic-gate ! 6617c478bd9Sstevel@tonic-gate bnz,a %ncc, .dcoh4 6627c478bd9Sstevel@tonic-gate btst 3, %o3 6637c478bd9Sstevel@tonic-gate.dcos8: 6647c478bd9Sstevel@tonic-gate ! 6657c478bd9Sstevel@tonic-gate ! Housekeeping for copy loops. Uses same idea as in the byte for 6667c478bd9Sstevel@tonic-gate ! byte copy loop above. 6677c478bd9Sstevel@tonic-gate ! 6687c478bd9Sstevel@tonic-gate add %o0, %o2, %o0 6697c478bd9Sstevel@tonic-gate add %o1, %o2, %o1 6707c478bd9Sstevel@tonic-gate sub %g0, %o2, %o3 6717c478bd9Sstevel@tonic-gate ba,pt %ncc, .dodebc 6727c478bd9Sstevel@tonic-gate srl %o2, 3, %o2 ! Number of 8 byte chunks to copy 6737c478bd9Sstevel@tonic-gate ! 6747c478bd9Sstevel@tonic-gate ! 4 byte aligned? 6757c478bd9Sstevel@tonic-gate ! 6767c478bd9Sstevel@tonic-gate.dcoh4: 6777c478bd9Sstevel@tonic-gate bnz,pn %ncc, .dcoh2 6787c478bd9Sstevel@tonic-gate nop 6797c478bd9Sstevel@tonic-gate.dcos4: 6807c478bd9Sstevel@tonic-gate add %o0, %o2, %o0 6817c478bd9Sstevel@tonic-gate add %o1, %o2, %o1 6827c478bd9Sstevel@tonic-gate sub %g0, %o2, %o3 6837c478bd9Sstevel@tonic-gate ba,pt %ncc, .dodfbc 6847c478bd9Sstevel@tonic-gate srl %o2, 2, %o2 ! Number of 4 byte chunks to copy 6857c478bd9Sstevel@tonic-gate ! 6867c478bd9Sstevel@tonic-gate ! We must be 2 byte aligned. Off we go. 6877c478bd9Sstevel@tonic-gate ! The check for small copies was done in the 6887c478bd9Sstevel@tonic-gate ! delay at .dcoh4 6897c478bd9Sstevel@tonic-gate ! 6907c478bd9Sstevel@tonic-gate.dcoh2: 6917c478bd9Sstevel@tonic-gate.dcos2: 6927c478bd9Sstevel@tonic-gate add %o0, %o2, %o0 6937c478bd9Sstevel@tonic-gate add %o1, %o2, %o1 6947c478bd9Sstevel@tonic-gate sub %g0, %o2, %o3 6957c478bd9Sstevel@tonic-gate ba,pt %ncc, .dodtbc 6967c478bd9Sstevel@tonic-gate srl %o2, 1, %o2 ! Number of 2 byte chunks to copy 6977c478bd9Sstevel@tonic-gate 6987c478bd9Sstevel@tonic-gate.dodebc: 6997c478bd9Sstevel@tonic-gate ldx [%o0 + %o3], %o4 7007c478bd9Sstevel@tonic-gate deccc %o2 7017c478bd9Sstevel@tonic-gate stxa %o4, [%o1 + %o3]ASI_USER 7027c478bd9Sstevel@tonic-gate bg,pt %ncc, .dodebc 7037c478bd9Sstevel@tonic-gate addcc %o3, 8, %o3 7047c478bd9Sstevel@tonic-gate ! 7057c478bd9Sstevel@tonic-gate ! End of copy loop. Check to see if we're done. Most 7067c478bd9Sstevel@tonic-gate ! eight byte aligned copies end here. 7077c478bd9Sstevel@tonic-gate ! 7087c478bd9Sstevel@tonic-gate bz,pt %ncc, .dcofh 7097c478bd9Sstevel@tonic-gate nop 7107c478bd9Sstevel@tonic-gate ! 7117c478bd9Sstevel@tonic-gate ! Something is left - do it byte for byte. 712*55fea89dSDan Cross ! 7137c478bd9Sstevel@tonic-gate ba,pt %ncc, .dcocl 7147c478bd9Sstevel@tonic-gate ldub [%o0 + %o3], %o4 ! load next byte 7157c478bd9Sstevel@tonic-gate ! 7167c478bd9Sstevel@tonic-gate ! Four byte copy loop. %o2 is the number of 4 byte chunks to copy. 7177c478bd9Sstevel@tonic-gate ! 7187c478bd9Sstevel@tonic-gate .align 32 7197c478bd9Sstevel@tonic-gate.dodfbc: 7207c478bd9Sstevel@tonic-gate lduw [%o0 + %o3], %o4 7217c478bd9Sstevel@tonic-gate deccc %o2 7227c478bd9Sstevel@tonic-gate sta %o4, [%o1 + %o3]ASI_USER 7237c478bd9Sstevel@tonic-gate bg,pt %ncc, .dodfbc 7247c478bd9Sstevel@tonic-gate addcc %o3, 4, %o3 7257c478bd9Sstevel@tonic-gate ! 7267c478bd9Sstevel@tonic-gate ! End of copy loop. Check to see if we're done. Most 7277c478bd9Sstevel@tonic-gate ! four byte aligned copies end here. 7287c478bd9Sstevel@tonic-gate ! 7297c478bd9Sstevel@tonic-gate bz,pt %ncc, .dcofh 7307c478bd9Sstevel@tonic-gate nop 7317c478bd9Sstevel@tonic-gate ! 7327c478bd9Sstevel@tonic-gate ! Something is left. Do it byte for byte. 7337c478bd9Sstevel@tonic-gate ! 7347c478bd9Sstevel@tonic-gate ba,pt %ncc, .dcocl 7357c478bd9Sstevel@tonic-gate ldub [%o0 + %o3], %o4 ! load next byte 7367c478bd9Sstevel@tonic-gate ! 7377c478bd9Sstevel@tonic-gate ! two byte aligned copy loop. %o2 is the number of 2 byte chunks to 7387c478bd9Sstevel@tonic-gate ! copy. 7397c478bd9Sstevel@tonic-gate ! 7407c478bd9Sstevel@tonic-gate .align 32 7417c478bd9Sstevel@tonic-gate.dodtbc: 7427c478bd9Sstevel@tonic-gate lduh [%o0 + %o3], %o4 7437c478bd9Sstevel@tonic-gate deccc %o2 7447c478bd9Sstevel@tonic-gate stha %o4, [%o1 + %o3]ASI_USER 7457c478bd9Sstevel@tonic-gate bg,pt %ncc, .dodtbc 7467c478bd9Sstevel@tonic-gate addcc %o3, 2, %o3 7477c478bd9Sstevel@tonic-gate ! 7487c478bd9Sstevel@tonic-gate ! End of copy loop. Anything left? 7497c478bd9Sstevel@tonic-gate ! 7507c478bd9Sstevel@tonic-gate bz,pt %ncc, .dcofh 7517c478bd9Sstevel@tonic-gate nop 7527c478bd9Sstevel@tonic-gate ! 7537c478bd9Sstevel@tonic-gate ! Deal with the last byte 7547c478bd9Sstevel@tonic-gate ! 7557c478bd9Sstevel@tonic-gate ldub [%o0 + %o3], %o4 7567c478bd9Sstevel@tonic-gate stba %o4, [%o1 + %o3]ASI_USER 7577c478bd9Sstevel@tonic-gate.dcofh: 7587c478bd9Sstevel@tonic-gate membar #Sync 7597c478bd9Sstevel@tonic-gate stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT] ! restore old t_lofault 7607c478bd9Sstevel@tonic-gate retl 7617c478bd9Sstevel@tonic-gate clr %o0 7627c478bd9Sstevel@tonic-gate 7637c478bd9Sstevel@tonic-gate.copyout_err: 7647c478bd9Sstevel@tonic-gate ldn [THREAD_REG + T_COPYOPS], %o4 7657c478bd9Sstevel@tonic-gate brz %o4, 2f 7667c478bd9Sstevel@tonic-gate nop 7677c478bd9Sstevel@tonic-gate ldn [%o4 + CP_COPYOUT], %g2 7687c478bd9Sstevel@tonic-gate jmp %g2 7697c478bd9Sstevel@tonic-gate nop 7707c478bd9Sstevel@tonic-gate2: 7717c478bd9Sstevel@tonic-gate retl 7727c478bd9Sstevel@tonic-gate mov -1, %o0 7737c478bd9Sstevel@tonic-gate SET_SIZE(copyout) 7747c478bd9Sstevel@tonic-gate 7757c478bd9Sstevel@tonic-gate 7767c478bd9Sstevel@tonic-gate ENTRY(xcopyout) 7777c478bd9Sstevel@tonic-gate sethi %hi(.xcopyout_err), REAL_LOFAULT 7787c478bd9Sstevel@tonic-gate b .do_copyout 7797c478bd9Sstevel@tonic-gate or REAL_LOFAULT, %lo(.xcopyout_err), REAL_LOFAULT 7807c478bd9Sstevel@tonic-gate.xcopyout_err: 7817c478bd9Sstevel@tonic-gate ldn [THREAD_REG + T_COPYOPS], %o4 7827c478bd9Sstevel@tonic-gate brz %o4, 2f 7837c478bd9Sstevel@tonic-gate nop 7847c478bd9Sstevel@tonic-gate ldn [%o4 + CP_XCOPYOUT], %g2 7857c478bd9Sstevel@tonic-gate jmp %g2 7867c478bd9Sstevel@tonic-gate nop 7877c478bd9Sstevel@tonic-gate2: 7887c478bd9Sstevel@tonic-gate retl 7897c478bd9Sstevel@tonic-gate mov %g1, %o0 7907c478bd9Sstevel@tonic-gate SET_SIZE(xcopyout) 7917c478bd9Sstevel@tonic-gate 7927c478bd9Sstevel@tonic-gate ENTRY(xcopyout_little) 7937c478bd9Sstevel@tonic-gate sethi %hi(.little_err), %o4 7947c478bd9Sstevel@tonic-gate ldn [THREAD_REG + T_LOFAULT], %o5 7957c478bd9Sstevel@tonic-gate or %o4, %lo(.little_err), %o4 7967c478bd9Sstevel@tonic-gate membar #Sync ! sync error barrier 7977c478bd9Sstevel@tonic-gate stn %o4, [THREAD_REG + T_LOFAULT] 7987c478bd9Sstevel@tonic-gate 7997c478bd9Sstevel@tonic-gate subcc %g0, %o2, %o3 8007c478bd9Sstevel@tonic-gate add %o0, %o2, %o0 8017c478bd9Sstevel@tonic-gate bz,pn %ncc, 2f ! check for zero bytes 8027c478bd9Sstevel@tonic-gate sub %o2, 1, %o4 8037c478bd9Sstevel@tonic-gate add %o0, %o4, %o0 ! start w/last byte 8047c478bd9Sstevel@tonic-gate add %o1, %o2, %o1 8057c478bd9Sstevel@tonic-gate ldub [%o0+%o3], %o4 8067c478bd9Sstevel@tonic-gate 8077c478bd9Sstevel@tonic-gate1: stba %o4, [%o1+%o3]ASI_AIUSL 8087c478bd9Sstevel@tonic-gate inccc %o3 8097c478bd9Sstevel@tonic-gate sub %o0, 2, %o0 ! get next byte 8107c478bd9Sstevel@tonic-gate bcc,a,pt %ncc, 1b 8117c478bd9Sstevel@tonic-gate ldub [%o0+%o3], %o4 8127c478bd9Sstevel@tonic-gate 8137c478bd9Sstevel@tonic-gate2: membar #Sync ! sync error barrier 8147c478bd9Sstevel@tonic-gate stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault 8157c478bd9Sstevel@tonic-gate retl 8167c478bd9Sstevel@tonic-gate mov %g0, %o0 ! return (0) 8177c478bd9Sstevel@tonic-gate SET_SIZE(xcopyout_little) 8187c478bd9Sstevel@tonic-gate 8197c478bd9Sstevel@tonic-gate/* 8207c478bd9Sstevel@tonic-gate * Copy user data to kernel space (copyin/xcopyin/xcopyin_little) 8217c478bd9Sstevel@tonic-gate */ 8227c478bd9Sstevel@tonic-gate 8237c478bd9Sstevel@tonic-gate ENTRY(copyin) 8247c478bd9Sstevel@tonic-gate sethi %hi(.copyin_err), REAL_LOFAULT 8257c478bd9Sstevel@tonic-gate or REAL_LOFAULT, %lo(.copyin_err), REAL_LOFAULT 8267c478bd9Sstevel@tonic-gate 8277c478bd9Sstevel@tonic-gate.do_copyin: 8287c478bd9Sstevel@tonic-gate ! 8297c478bd9Sstevel@tonic-gate ! Check the length and bail if zero. 8307c478bd9Sstevel@tonic-gate ! 8317c478bd9Sstevel@tonic-gate tst %o2 8327c478bd9Sstevel@tonic-gate bnz,pt %ncc, 1f 8337c478bd9Sstevel@tonic-gate nop 8347c478bd9Sstevel@tonic-gate retl 8357c478bd9Sstevel@tonic-gate clr %o0 8367c478bd9Sstevel@tonic-gate1: 8377c478bd9Sstevel@tonic-gate sethi %hi(copyio_fault), %o3 8387c478bd9Sstevel@tonic-gate ldn [THREAD_REG + T_LOFAULT], SAVED_LOFAULT 8397c478bd9Sstevel@tonic-gate or %o3, %lo(copyio_fault), %o3 8407c478bd9Sstevel@tonic-gate membar #Sync 8417c478bd9Sstevel@tonic-gate stn %o3, [THREAD_REG + T_LOFAULT] 8427c478bd9Sstevel@tonic-gate 8437c478bd9Sstevel@tonic-gate mov %o0, SAVE_SRC 8447c478bd9Sstevel@tonic-gate mov %o1, SAVE_DST 8457c478bd9Sstevel@tonic-gate mov %o2, SAVE_COUNT 8467c478bd9Sstevel@tonic-gate 8477c478bd9Sstevel@tonic-gate ! 8487c478bd9Sstevel@tonic-gate ! Check to see if we're more than SMALL_LIMIT. 8497c478bd9Sstevel@tonic-gate ! 8507c478bd9Sstevel@tonic-gate subcc %o2, SMALL_LIMIT, %o3 8517c478bd9Sstevel@tonic-gate bgu,a,pt %ncc, .dci_ns 8527c478bd9Sstevel@tonic-gate or %o0, %o1, %o3 8537c478bd9Sstevel@tonic-gate 8547c478bd9Sstevel@tonic-gate.dcibcp: 8557c478bd9Sstevel@tonic-gate sub %g0, %o2, %o3 ! setup for copy loop 8567c478bd9Sstevel@tonic-gate add %o0, %o2, %o0 8577c478bd9Sstevel@tonic-gate add %o1, %o2, %o1 8587c478bd9Sstevel@tonic-gate ba,pt %ncc, .dcicl 8597c478bd9Sstevel@tonic-gate lduba [%o0 + %o3]ASI_USER, %o4 8607c478bd9Sstevel@tonic-gate ! 8617c478bd9Sstevel@tonic-gate ! %o0 and %o1 point at the end and remain pointing at the end 8627c478bd9Sstevel@tonic-gate ! of their buffers. We pull things out by adding %o3 (which is 8637c478bd9Sstevel@tonic-gate ! the negation of the length) to the buffer end which gives us 8647c478bd9Sstevel@tonic-gate ! the curent location in the buffers. By incrementing %o3 we walk 8657c478bd9Sstevel@tonic-gate ! through both buffers without having to bump each buffer's 8667c478bd9Sstevel@tonic-gate ! pointer. A very fast 4 instruction loop. 8677c478bd9Sstevel@tonic-gate ! 8687c478bd9Sstevel@tonic-gate .align 16 8697c478bd9Sstevel@tonic-gate.dcicl: 8707c478bd9Sstevel@tonic-gate stb %o4, [%o1 + %o3] 8717c478bd9Sstevel@tonic-gate inccc %o3 8727c478bd9Sstevel@tonic-gate bl,a,pt %ncc, .dcicl 8737c478bd9Sstevel@tonic-gate lduba [%o0 + %o3]ASI_USER, %o4 8747c478bd9Sstevel@tonic-gate ! 8757c478bd9Sstevel@tonic-gate ! We're done. Go home. 876*55fea89dSDan Cross ! 8777c478bd9Sstevel@tonic-gate membar #Sync 8787c478bd9Sstevel@tonic-gate stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT] 8797c478bd9Sstevel@tonic-gate retl 8807c478bd9Sstevel@tonic-gate clr %o0 8817c478bd9Sstevel@tonic-gate ! 8827c478bd9Sstevel@tonic-gate ! Try aligned copies from here. 8837c478bd9Sstevel@tonic-gate ! 8847c478bd9Sstevel@tonic-gate.dci_ns: 8857c478bd9Sstevel@tonic-gate ! 8867c478bd9Sstevel@tonic-gate ! See if we're single byte aligned. If we are, check the 8877c478bd9Sstevel@tonic-gate ! limit for single byte copies. If we're smaller, or equal, 8887c478bd9Sstevel@tonic-gate ! bounce to the byte for byte copy loop. Otherwise do it in 8897c478bd9Sstevel@tonic-gate ! HW (if enabled). 8907c478bd9Sstevel@tonic-gate ! 8917c478bd9Sstevel@tonic-gate btst 1, %o3 8927c478bd9Sstevel@tonic-gate bz,a,pt %icc, .dcih8 8937c478bd9Sstevel@tonic-gate btst 7, %o3 8947c478bd9Sstevel@tonic-gate ba .dcibcp 8957c478bd9Sstevel@tonic-gate nop 8967c478bd9Sstevel@tonic-gate 8977c478bd9Sstevel@tonic-gate.dcih8: 8987c478bd9Sstevel@tonic-gate ! 8997c478bd9Sstevel@tonic-gate ! 8 byte aligned? 9007c478bd9Sstevel@tonic-gate ! 9017c478bd9Sstevel@tonic-gate bnz,a %ncc, .dcih4 9027c478bd9Sstevel@tonic-gate btst 3, %o3 9037c478bd9Sstevel@tonic-gate.dcis8: 9047c478bd9Sstevel@tonic-gate ! 9057c478bd9Sstevel@tonic-gate ! Housekeeping for copy loops. Uses same idea as in the byte for 9067c478bd9Sstevel@tonic-gate ! byte copy loop above. 9077c478bd9Sstevel@tonic-gate ! 9087c478bd9Sstevel@tonic-gate add %o0, %o2, %o0 9097c478bd9Sstevel@tonic-gate add %o1, %o2, %o1 9107c478bd9Sstevel@tonic-gate sub %g0, %o2, %o3 9117c478bd9Sstevel@tonic-gate ba,pt %ncc, .didebc 9127c478bd9Sstevel@tonic-gate srl %o2, 3, %o2 ! Number of 8 byte chunks to copy 9137c478bd9Sstevel@tonic-gate ! 9147c478bd9Sstevel@tonic-gate ! 4 byte aligned? 9157c478bd9Sstevel@tonic-gate ! 9167c478bd9Sstevel@tonic-gate.dcih4: 9177c478bd9Sstevel@tonic-gate bnz %ncc, .dcih2 9187c478bd9Sstevel@tonic-gate nop 9197c478bd9Sstevel@tonic-gate.dcis4: 9207c478bd9Sstevel@tonic-gate ! 9217c478bd9Sstevel@tonic-gate ! Housekeeping for copy loops. Uses same idea as in the byte 9227c478bd9Sstevel@tonic-gate ! for byte copy loop above. 9237c478bd9Sstevel@tonic-gate ! 9247c478bd9Sstevel@tonic-gate add %o0, %o2, %o0 9257c478bd9Sstevel@tonic-gate add %o1, %o2, %o1 9267c478bd9Sstevel@tonic-gate sub %g0, %o2, %o3 9277c478bd9Sstevel@tonic-gate ba,pt %ncc, .didfbc 9287c478bd9Sstevel@tonic-gate srl %o2, 2, %o2 ! Number of 4 byte chunks to copy 9297c478bd9Sstevel@tonic-gate.dcih2: 9307c478bd9Sstevel@tonic-gate.dcis2: 9317c478bd9Sstevel@tonic-gate add %o0, %o2, %o0 9327c478bd9Sstevel@tonic-gate add %o1, %o2, %o1 9337c478bd9Sstevel@tonic-gate sub %g0, %o2, %o3 9347c478bd9Sstevel@tonic-gate ba,pt %ncc, .didtbc 9357c478bd9Sstevel@tonic-gate srl %o2, 1, %o2 ! Number of 2 byte chunks to copy 9367c478bd9Sstevel@tonic-gate 9377c478bd9Sstevel@tonic-gate.didebc: 9387c478bd9Sstevel@tonic-gate ldxa [%o0 + %o3]ASI_USER, %o4 9397c478bd9Sstevel@tonic-gate deccc %o2 9407c478bd9Sstevel@tonic-gate stx %o4, [%o1 + %o3] 9417c478bd9Sstevel@tonic-gate bg,pt %ncc, .didebc 9427c478bd9Sstevel@tonic-gate addcc %o3, 8, %o3 9437c478bd9Sstevel@tonic-gate ! 9447c478bd9Sstevel@tonic-gate ! End of copy loop. Most 8 byte aligned copies end here. 9457c478bd9Sstevel@tonic-gate ! 9467c478bd9Sstevel@tonic-gate bz,pt %ncc, .dcifh 9477c478bd9Sstevel@tonic-gate nop 9487c478bd9Sstevel@tonic-gate ! 9497c478bd9Sstevel@tonic-gate ! Something is left. Do it byte for byte. 9507c478bd9Sstevel@tonic-gate ! 9517c478bd9Sstevel@tonic-gate ba,pt %ncc, .dcicl 9527c478bd9Sstevel@tonic-gate lduba [%o0 + %o3]ASI_USER, %o4 9537c478bd9Sstevel@tonic-gate ! 9547c478bd9Sstevel@tonic-gate ! 4 byte copy loop. %o2 is number of 4 byte chunks to copy. 9557c478bd9Sstevel@tonic-gate ! 9567c478bd9Sstevel@tonic-gate .align 32 9577c478bd9Sstevel@tonic-gate.didfbc: 9587c478bd9Sstevel@tonic-gate lduwa [%o0 + %o3]ASI_USER, %o4 9597c478bd9Sstevel@tonic-gate deccc %o2 9607c478bd9Sstevel@tonic-gate st %o4, [%o1 + %o3] 9617c478bd9Sstevel@tonic-gate bg,pt %ncc, .didfbc 9627c478bd9Sstevel@tonic-gate addcc %o3, 4, %o3 9637c478bd9Sstevel@tonic-gate ! 9647c478bd9Sstevel@tonic-gate ! End of copy loop. Most 4 byte aligned copies end here. 9657c478bd9Sstevel@tonic-gate ! 9667c478bd9Sstevel@tonic-gate bz,pt %ncc, .dcifh 9677c478bd9Sstevel@tonic-gate nop 9687c478bd9Sstevel@tonic-gate ! 9697c478bd9Sstevel@tonic-gate ! Something is left. Do it byte for byte. 9707c478bd9Sstevel@tonic-gate ! 9717c478bd9Sstevel@tonic-gate ba,pt %ncc, .dcicl 9727c478bd9Sstevel@tonic-gate lduba [%o0 + %o3]ASI_USER, %o4 9737c478bd9Sstevel@tonic-gate ! 9747c478bd9Sstevel@tonic-gate ! 2 byte aligned copy loop. %o2 is number of 2 byte chunks to 9757c478bd9Sstevel@tonic-gate ! copy. 9767c478bd9Sstevel@tonic-gate ! 9777c478bd9Sstevel@tonic-gate .align 32 9787c478bd9Sstevel@tonic-gate.didtbc: 9797c478bd9Sstevel@tonic-gate lduha [%o0 + %o3]ASI_USER, %o4 9807c478bd9Sstevel@tonic-gate deccc %o2 9817c478bd9Sstevel@tonic-gate sth %o4, [%o1 + %o3] 9827c478bd9Sstevel@tonic-gate bg,pt %ncc, .didtbc 9837c478bd9Sstevel@tonic-gate addcc %o3, 2, %o3 9847c478bd9Sstevel@tonic-gate ! 9857c478bd9Sstevel@tonic-gate ! End of copy loop. Most 2 byte aligned copies end here. 9867c478bd9Sstevel@tonic-gate ! 9877c478bd9Sstevel@tonic-gate bz,pt %ncc, .dcifh 9887c478bd9Sstevel@tonic-gate nop 9897c478bd9Sstevel@tonic-gate ! 9907c478bd9Sstevel@tonic-gate ! Deal with the last byte 9917c478bd9Sstevel@tonic-gate ! 9927c478bd9Sstevel@tonic-gate lduba [%o0 + %o3]ASI_USER, %o4 9937c478bd9Sstevel@tonic-gate stb %o4, [%o1 + %o3] 9947c478bd9Sstevel@tonic-gate.dcifh: 9957c478bd9Sstevel@tonic-gate membar #Sync 9967c478bd9Sstevel@tonic-gate stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT] ! restore old t_lofault 9977c478bd9Sstevel@tonic-gate retl 9987c478bd9Sstevel@tonic-gate clr %o0 9997c478bd9Sstevel@tonic-gate 10007c478bd9Sstevel@tonic-gate.copyin_err: 10017c478bd9Sstevel@tonic-gate ldn [THREAD_REG + T_COPYOPS], %o4 10027c478bd9Sstevel@tonic-gate brz %o4, 2f 10037c478bd9Sstevel@tonic-gate nop 10047c478bd9Sstevel@tonic-gate ldn [%o4 + CP_COPYIN], %g2 10057c478bd9Sstevel@tonic-gate jmp %g2 10067c478bd9Sstevel@tonic-gate nop 10077c478bd9Sstevel@tonic-gate2: 10087c478bd9Sstevel@tonic-gate retl 10097c478bd9Sstevel@tonic-gate mov -1, %o0 10107c478bd9Sstevel@tonic-gate SET_SIZE(copyin) 10117c478bd9Sstevel@tonic-gate 10127c478bd9Sstevel@tonic-gate ENTRY(xcopyin) 10137c478bd9Sstevel@tonic-gate sethi %hi(.xcopyin_err), REAL_LOFAULT 10147c478bd9Sstevel@tonic-gate b .do_copyin 10157c478bd9Sstevel@tonic-gate or REAL_LOFAULT, %lo(.xcopyin_err), REAL_LOFAULT 10167c478bd9Sstevel@tonic-gate.xcopyin_err: 10177c478bd9Sstevel@tonic-gate ldn [THREAD_REG + T_COPYOPS], %o4 10187c478bd9Sstevel@tonic-gate brz %o4, 2f 10197c478bd9Sstevel@tonic-gate nop 10207c478bd9Sstevel@tonic-gate ldn [%o4 + CP_XCOPYIN], %g2 10217c478bd9Sstevel@tonic-gate jmp %g2 10227c478bd9Sstevel@tonic-gate nop 10237c478bd9Sstevel@tonic-gate2: 10247c478bd9Sstevel@tonic-gate retl 10257c478bd9Sstevel@tonic-gate mov %g1, %o0 10267c478bd9Sstevel@tonic-gate SET_SIZE(xcopyin) 10277c478bd9Sstevel@tonic-gate 10287c478bd9Sstevel@tonic-gate ENTRY(xcopyin_little) 10297c478bd9Sstevel@tonic-gate sethi %hi(.little_err), %o4 10307c478bd9Sstevel@tonic-gate ldn [THREAD_REG + T_LOFAULT], %o5 10317c478bd9Sstevel@tonic-gate or %o4, %lo(.little_err), %o4 10327c478bd9Sstevel@tonic-gate membar #Sync ! sync error barrier 1033*55fea89dSDan Cross stn %o4, [THREAD_REG + T_LOFAULT] 10347c478bd9Sstevel@tonic-gate 10357c478bd9Sstevel@tonic-gate subcc %g0, %o2, %o3 10367c478bd9Sstevel@tonic-gate add %o0, %o2, %o0 10377c478bd9Sstevel@tonic-gate bz,pn %ncc, 2f ! check for zero bytes 10387c478bd9Sstevel@tonic-gate sub %o2, 1, %o4 1039*55fea89dSDan Cross add %o0, %o4, %o0 ! start w/last byte 10407c478bd9Sstevel@tonic-gate add %o1, %o2, %o1 10417c478bd9Sstevel@tonic-gate lduba [%o0+%o3]ASI_AIUSL, %o4 10427c478bd9Sstevel@tonic-gate 10437c478bd9Sstevel@tonic-gate1: stb %o4, [%o1+%o3] 10447c478bd9Sstevel@tonic-gate inccc %o3 10457c478bd9Sstevel@tonic-gate sub %o0, 2, %o0 ! get next byte 10467c478bd9Sstevel@tonic-gate bcc,a,pt %ncc, 1b 10477c478bd9Sstevel@tonic-gate lduba [%o0+%o3]ASI_AIUSL, %o4 10487c478bd9Sstevel@tonic-gate 10497c478bd9Sstevel@tonic-gate2: membar #Sync ! sync error barrier 10507c478bd9Sstevel@tonic-gate stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault 10517c478bd9Sstevel@tonic-gate retl 10527c478bd9Sstevel@tonic-gate mov %g0, %o0 ! return (0) 10537c478bd9Sstevel@tonic-gate 10547c478bd9Sstevel@tonic-gate.little_err: 10557c478bd9Sstevel@tonic-gate membar #Sync ! sync error barrier 10567c478bd9Sstevel@tonic-gate stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault 10577c478bd9Sstevel@tonic-gate retl 10587c478bd9Sstevel@tonic-gate mov %g1, %o0 10597c478bd9Sstevel@tonic-gate SET_SIZE(xcopyin_little) 10607c478bd9Sstevel@tonic-gate 10617c478bd9Sstevel@tonic-gate 10627c478bd9Sstevel@tonic-gate/* 10637c478bd9Sstevel@tonic-gate * Copy a block of storage - must not overlap (from + len <= to). 10647c478bd9Sstevel@tonic-gate * No fault handler installed (to be called under on_fault()) 10657c478bd9Sstevel@tonic-gate */ 10667c478bd9Sstevel@tonic-gate 10677c478bd9Sstevel@tonic-gate ENTRY(copyin_noerr) 10687c478bd9Sstevel@tonic-gate sethi %hi(.copyio_noerr), REAL_LOFAULT 10697c478bd9Sstevel@tonic-gate b .do_copyin 10707c478bd9Sstevel@tonic-gate or REAL_LOFAULT, %lo(.copyio_noerr), REAL_LOFAULT 10717c478bd9Sstevel@tonic-gate.copyio_noerr: 10727c478bd9Sstevel@tonic-gate jmp SAVED_LOFAULT 10737c478bd9Sstevel@tonic-gate nop 10747c478bd9Sstevel@tonic-gate SET_SIZE(copyin_noerr) 10757c478bd9Sstevel@tonic-gate 10767c478bd9Sstevel@tonic-gate/* 10777c478bd9Sstevel@tonic-gate * Copy a block of storage - must not overlap (from + len <= to). 10787c478bd9Sstevel@tonic-gate * No fault handler installed (to be called under on_fault()) 10797c478bd9Sstevel@tonic-gate */ 10807c478bd9Sstevel@tonic-gate 10817c478bd9Sstevel@tonic-gate ENTRY(copyout_noerr) 10827c478bd9Sstevel@tonic-gate sethi %hi(.copyio_noerr), REAL_LOFAULT 10837c478bd9Sstevel@tonic-gate b .do_copyout 10847c478bd9Sstevel@tonic-gate or REAL_LOFAULT, %lo(.copyio_noerr), REAL_LOFAULT 10857c478bd9Sstevel@tonic-gate SET_SIZE(copyout_noerr) 10867c478bd9Sstevel@tonic-gate 10877c478bd9Sstevel@tonic-gate .align 4 10887c478bd9Sstevel@tonic-gate DGDEF(use_hw_bcopy) 10897c478bd9Sstevel@tonic-gate .word 1 10907c478bd9Sstevel@tonic-gate DGDEF(use_hw_bzero) 10917c478bd9Sstevel@tonic-gate .word 1 10927c478bd9Sstevel@tonic-gate 10937c478bd9Sstevel@tonic-gate .align 64 10947c478bd9Sstevel@tonic-gate .section ".text" 10957c478bd9Sstevel@tonic-gate 10967c478bd9Sstevel@tonic-gate 10977c478bd9Sstevel@tonic-gate/* 10987c478bd9Sstevel@tonic-gate * hwblkclr - clears block-aligned, block-multiple-sized regions that are 1099c2b6e926Sae * longer than 256 bytes in length. For the generic module we will simply 1100c2b6e926Sae * call bzero and return 1 to ensure that the pages in cache should be 1101a4070553Siskreen * flushed to ensure integrity. 11027c478bd9Sstevel@tonic-gate * Caller is responsible for ensuring use_hw_bzero is true and that 11037c478bd9Sstevel@tonic-gate * kpreempt_disable() has been called. 11047c478bd9Sstevel@tonic-gate */ 11057c478bd9Sstevel@tonic-gate ! %i0 - start address 11067c478bd9Sstevel@tonic-gate ! %i1 - length of region (multiple of 64) 11077c478bd9Sstevel@tonic-gate 11087c478bd9Sstevel@tonic-gate ENTRY(hwblkclr) 11097c478bd9Sstevel@tonic-gate save %sp, -SA(MINFRAME), %sp 11107c478bd9Sstevel@tonic-gate 1111c2b6e926Sae ! Simply call bzero and notify the caller that bzero was used 11127c478bd9Sstevel@tonic-gate mov %i0, %o0 11137c478bd9Sstevel@tonic-gate call bzero 11147c478bd9Sstevel@tonic-gate mov %i1, %o1 11157c478bd9Sstevel@tonic-gate ret 11167c478bd9Sstevel@tonic-gate restore %g0, 1, %o0 ! return (1) - did not use block operations 11177c478bd9Sstevel@tonic-gate 11187c478bd9Sstevel@tonic-gate SET_SIZE(hwblkclr) 11197c478bd9Sstevel@tonic-gate 11207c478bd9Sstevel@tonic-gate /* 11217c478bd9Sstevel@tonic-gate * Copy 32 bytes of data from src (%o0) to dst (%o1) 11227c478bd9Sstevel@tonic-gate * using physical addresses. 11237c478bd9Sstevel@tonic-gate */ 11247c478bd9Sstevel@tonic-gate ENTRY_NP(hw_pa_bcopy32) 11257c478bd9Sstevel@tonic-gate rdpr %pstate, %g1 11267c478bd9Sstevel@tonic-gate andn %g1, PSTATE_IE, %g2 11277c478bd9Sstevel@tonic-gate wrpr %g0, %g2, %pstate 11287c478bd9Sstevel@tonic-gate 11297c478bd9Sstevel@tonic-gate ldxa [%o0]ASI_MEM, %o2 11307c478bd9Sstevel@tonic-gate add %o0, 8, %o0 11317c478bd9Sstevel@tonic-gate ldxa [%o0]ASI_MEM, %o3 11327c478bd9Sstevel@tonic-gate add %o0, 8, %o0 11337c478bd9Sstevel@tonic-gate ldxa [%o0]ASI_MEM, %o4 11347c478bd9Sstevel@tonic-gate add %o0, 8, %o0 11357c478bd9Sstevel@tonic-gate ldxa [%o0]ASI_MEM, %o5 11367c478bd9Sstevel@tonic-gate stxa %o2, [%o1]ASI_MEM 11377c478bd9Sstevel@tonic-gate add %o1, 8, %o1 11387c478bd9Sstevel@tonic-gate stxa %o3, [%o1]ASI_MEM 11397c478bd9Sstevel@tonic-gate add %o1, 8, %o1 11407c478bd9Sstevel@tonic-gate stxa %o4, [%o1]ASI_MEM 11417c478bd9Sstevel@tonic-gate add %o1, 8, %o1 11427c478bd9Sstevel@tonic-gate stxa %o5, [%o1]ASI_MEM 11437c478bd9Sstevel@tonic-gate 11447c478bd9Sstevel@tonic-gate membar #Sync 11457c478bd9Sstevel@tonic-gate retl 11467c478bd9Sstevel@tonic-gate wrpr %g0, %g1, %pstate 11477c478bd9Sstevel@tonic-gate SET_SIZE(hw_pa_bcopy32) 11487c478bd9Sstevel@tonic-gate 11497c478bd9Sstevel@tonic-gate/* 11507c478bd9Sstevel@tonic-gate * Zero a block of storage. 11517c478bd9Sstevel@tonic-gate * 11527c478bd9Sstevel@tonic-gate * uzero is used by the kernel to zero a block in user address space. 11537c478bd9Sstevel@tonic-gate */ 11547c478bd9Sstevel@tonic-gate 11557c478bd9Sstevel@tonic-gate 11567c478bd9Sstevel@tonic-gate ENTRY(uzero) 11577c478bd9Sstevel@tonic-gate ! 11587c478bd9Sstevel@tonic-gate ! Set a new lo_fault handler only if we came in with one 11597c478bd9Sstevel@tonic-gate ! already specified. 11607c478bd9Sstevel@tonic-gate ! 11617c478bd9Sstevel@tonic-gate wr %g0, ASI_USER, %asi 11627c478bd9Sstevel@tonic-gate ldn [THREAD_REG + T_LOFAULT], %o5 11637c478bd9Sstevel@tonic-gate tst %o5 11647c478bd9Sstevel@tonic-gate bz,pt %ncc, .do_zero 11657c478bd9Sstevel@tonic-gate sethi %hi(.zeroerr), %o2 11667c478bd9Sstevel@tonic-gate or %o2, %lo(.zeroerr), %o2 11677c478bd9Sstevel@tonic-gate membar #Sync 11687c478bd9Sstevel@tonic-gate ba,pt %ncc, .do_zero 11697c478bd9Sstevel@tonic-gate stn %o2, [THREAD_REG + T_LOFAULT] 11707c478bd9Sstevel@tonic-gate 11717c478bd9Sstevel@tonic-gate ENTRY(kzero) 11727c478bd9Sstevel@tonic-gate ! 11737c478bd9Sstevel@tonic-gate ! Always set a lo_fault handler 11747c478bd9Sstevel@tonic-gate ! 11757c478bd9Sstevel@tonic-gate wr %g0, ASI_P, %asi 11767c478bd9Sstevel@tonic-gate ldn [THREAD_REG + T_LOFAULT], %o5 11777c478bd9Sstevel@tonic-gate sethi %hi(.zeroerr), %o2 11787c478bd9Sstevel@tonic-gate or %o5, LOFAULT_SET, %o5 11797c478bd9Sstevel@tonic-gate or %o2, %lo(.zeroerr), %o2 11807c478bd9Sstevel@tonic-gate membar #Sync 11817c478bd9Sstevel@tonic-gate ba,pt %ncc, .do_zero 11827c478bd9Sstevel@tonic-gate stn %o2, [THREAD_REG + T_LOFAULT] 11837c478bd9Sstevel@tonic-gate 11847c478bd9Sstevel@tonic-gate/* 11857c478bd9Sstevel@tonic-gate * We got here because of a fault during kzero or if 11867c478bd9Sstevel@tonic-gate * uzero or bzero was called with t_lofault non-zero. 11877c478bd9Sstevel@tonic-gate * Otherwise we've already run screaming from the room. 11887c478bd9Sstevel@tonic-gate * Errno value is in %g1. Note that we're here iff 11897c478bd9Sstevel@tonic-gate * we did set t_lofault. 11907c478bd9Sstevel@tonic-gate */ 11917c478bd9Sstevel@tonic-gate.zeroerr: 11927c478bd9Sstevel@tonic-gate ! 11937c478bd9Sstevel@tonic-gate ! Undo asi register setting. Just set it to be the 11947c478bd9Sstevel@tonic-gate ! kernel default without checking. 11957c478bd9Sstevel@tonic-gate ! 11967c478bd9Sstevel@tonic-gate wr %g0, ASI_P, %asi 11977c478bd9Sstevel@tonic-gate 11987c478bd9Sstevel@tonic-gate ! 11997c478bd9Sstevel@tonic-gate ! We did set t_lofault. It may well have been zero coming in. 12007c478bd9Sstevel@tonic-gate ! 12017c478bd9Sstevel@tonic-gate1: 12027c478bd9Sstevel@tonic-gate tst %o5 12037c478bd9Sstevel@tonic-gate membar #Sync 1204*55fea89dSDan Cross bne,pn %ncc, 3f 12057c478bd9Sstevel@tonic-gate andncc %o5, LOFAULT_SET, %o5 12067c478bd9Sstevel@tonic-gate2: 12077c478bd9Sstevel@tonic-gate ! 12087c478bd9Sstevel@tonic-gate ! Old handler was zero. Just return the error. 12097c478bd9Sstevel@tonic-gate ! 12107c478bd9Sstevel@tonic-gate retl ! return 12117c478bd9Sstevel@tonic-gate mov %g1, %o0 ! error code from %g1 12127c478bd9Sstevel@tonic-gate3: 12137c478bd9Sstevel@tonic-gate ! 12147c478bd9Sstevel@tonic-gate ! We're here because %o5 was non-zero. It was non-zero 12157c478bd9Sstevel@tonic-gate ! because either LOFAULT_SET was present, a previous fault 12167c478bd9Sstevel@tonic-gate ! handler was present or both. In all cases we need to reset 12177c478bd9Sstevel@tonic-gate ! T_LOFAULT to the value of %o5 after clearing LOFAULT_SET 12187c478bd9Sstevel@tonic-gate ! before we either simply return the error or we invoke the 12197c478bd9Sstevel@tonic-gate ! previously specified handler. 12207c478bd9Sstevel@tonic-gate ! 12217c478bd9Sstevel@tonic-gate be %ncc, 2b 12227c478bd9Sstevel@tonic-gate stn %o5, [THREAD_REG + T_LOFAULT] 12237c478bd9Sstevel@tonic-gate jmp %o5 ! goto real handler 12247c478bd9Sstevel@tonic-gate nop 12257c478bd9Sstevel@tonic-gate SET_SIZE(kzero) 12267c478bd9Sstevel@tonic-gate SET_SIZE(uzero) 12277c478bd9Sstevel@tonic-gate 12287c478bd9Sstevel@tonic-gate/* 12297c478bd9Sstevel@tonic-gate * Zero a block of storage. 12307c478bd9Sstevel@tonic-gate */ 12317c478bd9Sstevel@tonic-gate 12327c478bd9Sstevel@tonic-gate ENTRY(bzero) 12337c478bd9Sstevel@tonic-gate wr %g0, ASI_P, %asi 12347c478bd9Sstevel@tonic-gate 12357c478bd9Sstevel@tonic-gate ldn [THREAD_REG + T_LOFAULT], %o5 ! save old vector 12367c478bd9Sstevel@tonic-gate tst %o5 12377c478bd9Sstevel@tonic-gate bz,pt %ncc, .do_zero 12387c478bd9Sstevel@tonic-gate sethi %hi(.zeroerr), %o2 12397c478bd9Sstevel@tonic-gate or %o2, %lo(.zeroerr), %o2 12407c478bd9Sstevel@tonic-gate membar #Sync ! sync error barrier 12417c478bd9Sstevel@tonic-gate stn %o2, [THREAD_REG + T_LOFAULT] ! install new vector 12427c478bd9Sstevel@tonic-gate 12437c478bd9Sstevel@tonic-gate.do_zero: 12447c478bd9Sstevel@tonic-gate cmp %o1, 7 12457c478bd9Sstevel@tonic-gate blu,pn %ncc, .byteclr 12467c478bd9Sstevel@tonic-gate nop 12477c478bd9Sstevel@tonic-gate 12487c478bd9Sstevel@tonic-gate cmp %o1, 15 12497c478bd9Sstevel@tonic-gate blu,pn %ncc, .wdalign 12507c478bd9Sstevel@tonic-gate nop 12517c478bd9Sstevel@tonic-gate 12527c478bd9Sstevel@tonic-gate andcc %o0, 7, %o3 ! is add aligned on a 8 byte bound 12537c478bd9Sstevel@tonic-gate bz,pt %ncc, .blkalign ! already double aligned 12547c478bd9Sstevel@tonic-gate sub %o3, 8, %o3 ! -(bytes till double aligned) 12557c478bd9Sstevel@tonic-gate add %o1, %o3, %o1 ! update o1 with new count 12567c478bd9Sstevel@tonic-gate 12577c478bd9Sstevel@tonic-gate1: 12587c478bd9Sstevel@tonic-gate stba %g0, [%o0]%asi 12597c478bd9Sstevel@tonic-gate inccc %o3 12607c478bd9Sstevel@tonic-gate bl,pt %ncc, 1b 12617c478bd9Sstevel@tonic-gate inc %o0 12627c478bd9Sstevel@tonic-gate 12637c478bd9Sstevel@tonic-gate ! Now address is double aligned 12647c478bd9Sstevel@tonic-gate.blkalign: 12657c478bd9Sstevel@tonic-gate cmp %o1, 0x80 ! check if there are 128 bytes to set 12667c478bd9Sstevel@tonic-gate blu,pn %ncc, .bzero_small 12677c478bd9Sstevel@tonic-gate mov %o1, %o3 12687c478bd9Sstevel@tonic-gate 12697c478bd9Sstevel@tonic-gate andcc %o0, 0x3f, %o3 ! is block aligned? 12707c478bd9Sstevel@tonic-gate bz,pt %ncc, .bzero_blk 12717c478bd9Sstevel@tonic-gate sub %o3, 0x40, %o3 ! -(bytes till block aligned) 12727c478bd9Sstevel@tonic-gate add %o1, %o3, %o1 ! o1 is the remainder 1273*55fea89dSDan Cross 12747c478bd9Sstevel@tonic-gate ! Clear -(%o3) bytes till block aligned 12757c478bd9Sstevel@tonic-gate1: 12767c478bd9Sstevel@tonic-gate stxa %g0, [%o0]%asi 12777c478bd9Sstevel@tonic-gate addcc %o3, 8, %o3 12787c478bd9Sstevel@tonic-gate bl,pt %ncc, 1b 12797c478bd9Sstevel@tonic-gate add %o0, 8, %o0 12807c478bd9Sstevel@tonic-gate 12817c478bd9Sstevel@tonic-gate.bzero_blk: 12827c478bd9Sstevel@tonic-gate and %o1, 0x3f, %o3 ! calc bytes left after blk clear 12837c478bd9Sstevel@tonic-gate andn %o1, 0x3f, %o4 ! calc size of blocks in bytes 12847c478bd9Sstevel@tonic-gate 12857c478bd9Sstevel@tonic-gate cmp %o4, 0x100 ! 256 bytes or more 12867c478bd9Sstevel@tonic-gate blu,pn %ncc, 3f 12877c478bd9Sstevel@tonic-gate nop 12887c478bd9Sstevel@tonic-gate 12897c478bd9Sstevel@tonic-gate2: 12907c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x0]%asi 12917c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x40]%asi 12927c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x80]%asi 12937c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0xc0]%asi 12947c478bd9Sstevel@tonic-gate 12957c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x8]%asi 12967c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x10]%asi 12977c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x18]%asi 12987c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x20]%asi 12997c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x28]%asi 13007c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x30]%asi 13017c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x38]%asi 13027c478bd9Sstevel@tonic-gate 13037c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x48]%asi 13047c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x50]%asi 13057c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x58]%asi 13067c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x60]%asi 13077c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x68]%asi 13087c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x70]%asi 13097c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x78]%asi 13107c478bd9Sstevel@tonic-gate 13117c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x88]%asi 13127c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x90]%asi 13137c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x98]%asi 13147c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0xa0]%asi 13157c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0xa8]%asi 13167c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0xb0]%asi 13177c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0xb8]%asi 13187c478bd9Sstevel@tonic-gate 13197c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0xc8]%asi 13207c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0xd0]%asi 13217c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0xd8]%asi 13227c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0xe0]%asi 13237c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0xe8]%asi 13247c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0xf0]%asi 13257c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0xf8]%asi 13267c478bd9Sstevel@tonic-gate 13277c478bd9Sstevel@tonic-gate sub %o4, 0x100, %o4 13287c478bd9Sstevel@tonic-gate cmp %o4, 0x100 13297c478bd9Sstevel@tonic-gate bgu,pt %ncc, 2b 13307c478bd9Sstevel@tonic-gate add %o0, 0x100, %o0 13317c478bd9Sstevel@tonic-gate 13327c478bd9Sstevel@tonic-gate3: 13337c478bd9Sstevel@tonic-gate ! ... check if 64 bytes to set 13347c478bd9Sstevel@tonic-gate cmp %o4, 0x40 13357c478bd9Sstevel@tonic-gate blu %ncc, .bzero_blk_done 13367c478bd9Sstevel@tonic-gate nop 13377c478bd9Sstevel@tonic-gate 13387c478bd9Sstevel@tonic-gate4: 13397c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x0]%asi 13407c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x8]%asi 13417c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x10]%asi 13427c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x18]%asi 13437c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x20]%asi 13447c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x28]%asi 13457c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x30]%asi 13467c478bd9Sstevel@tonic-gate stxa %g0, [%o0+0x38]%asi 13477c478bd9Sstevel@tonic-gate 13487c478bd9Sstevel@tonic-gate subcc %o4, 0x40, %o4 13497c478bd9Sstevel@tonic-gate bgu,pt %ncc, 3b 13507c478bd9Sstevel@tonic-gate add %o0, 0x40, %o0 13517c478bd9Sstevel@tonic-gate 13527c478bd9Sstevel@tonic-gate.bzero_blk_done: 13537c478bd9Sstevel@tonic-gate membar #Sync 13547c478bd9Sstevel@tonic-gate 13557c478bd9Sstevel@tonic-gate.bzero_small: 13567c478bd9Sstevel@tonic-gate ! Set the remaining doubles 13577c478bd9Sstevel@tonic-gate subcc %o3, 8, %o3 ! Can we store any doubles? 13587c478bd9Sstevel@tonic-gate blu,pn %ncc, .byteclr 13597c478bd9Sstevel@tonic-gate and %o1, 7, %o1 ! calc bytes left after doubles 13607c478bd9Sstevel@tonic-gate 13617c478bd9Sstevel@tonic-gate.dbclr: 13627c478bd9Sstevel@tonic-gate stxa %g0, [%o0]%asi ! Clear the doubles 13637c478bd9Sstevel@tonic-gate subcc %o3, 8, %o3 13647c478bd9Sstevel@tonic-gate bgeu,pt %ncc, .dbclr 13657c478bd9Sstevel@tonic-gate add %o0, 8, %o0 13667c478bd9Sstevel@tonic-gate 13677c478bd9Sstevel@tonic-gate ba .byteclr 13687c478bd9Sstevel@tonic-gate nop 13697c478bd9Sstevel@tonic-gate 1370*55fea89dSDan Cross.wdalign: 13717c478bd9Sstevel@tonic-gate andcc %o0, 3, %o3 ! is add aligned on a word boundary 13727c478bd9Sstevel@tonic-gate bz,pn %ncc, .wdclr 13737c478bd9Sstevel@tonic-gate andn %o1, 3, %o3 ! create word sized count in %o3 13747c478bd9Sstevel@tonic-gate 13757c478bd9Sstevel@tonic-gate dec %o1 ! decrement count 13767c478bd9Sstevel@tonic-gate stba %g0, [%o0]%asi ! clear a byte 13777c478bd9Sstevel@tonic-gate ba .wdalign 13787c478bd9Sstevel@tonic-gate inc %o0 ! next byte 13797c478bd9Sstevel@tonic-gate 13807c478bd9Sstevel@tonic-gate.wdclr: 13817c478bd9Sstevel@tonic-gate sta %g0, [%o0]%asi ! 4-byte clearing loop 13827c478bd9Sstevel@tonic-gate subcc %o3, 4, %o3 13837c478bd9Sstevel@tonic-gate bnz,pt %ncc, .wdclr 13847c478bd9Sstevel@tonic-gate inc 4, %o0 13857c478bd9Sstevel@tonic-gate 13867c478bd9Sstevel@tonic-gate and %o1, 3, %o1 ! leftover count, if any 13877c478bd9Sstevel@tonic-gate 13887c478bd9Sstevel@tonic-gate.byteclr: 13897c478bd9Sstevel@tonic-gate ! Set the leftover bytes 13907c478bd9Sstevel@tonic-gate brz %o1, .bzero_exit 13917c478bd9Sstevel@tonic-gate nop 13927c478bd9Sstevel@tonic-gate 13937c478bd9Sstevel@tonic-gate7: 13947c478bd9Sstevel@tonic-gate deccc %o1 ! byte clearing loop 13957c478bd9Sstevel@tonic-gate stba %g0, [%o0]%asi 13967c478bd9Sstevel@tonic-gate bgu,pt %ncc, 7b 13977c478bd9Sstevel@tonic-gate inc %o0 13987c478bd9Sstevel@tonic-gate 13997c478bd9Sstevel@tonic-gate.bzero_exit: 14007c478bd9Sstevel@tonic-gate ! 14017c478bd9Sstevel@tonic-gate ! We're just concerned with whether t_lofault was set 14027c478bd9Sstevel@tonic-gate ! when we came in. We end up here from either kzero() 14037c478bd9Sstevel@tonic-gate ! or bzero(). kzero() *always* sets a lofault handler. 14047c478bd9Sstevel@tonic-gate ! It ors LOFAULT_SET into %o5 to indicate it has done 14057c478bd9Sstevel@tonic-gate ! this even if the value of %o5 is otherwise zero. 14067c478bd9Sstevel@tonic-gate ! bzero() sets a lofault handler *only* if one was 14077c478bd9Sstevel@tonic-gate ! previously set. Accordingly we need to examine 14087c478bd9Sstevel@tonic-gate ! %o5 and if it is non-zero be sure to clear LOFAULT_SET 14097c478bd9Sstevel@tonic-gate ! before resetting the error handler. 14107c478bd9Sstevel@tonic-gate ! 14117c478bd9Sstevel@tonic-gate tst %o5 14127c478bd9Sstevel@tonic-gate bz %ncc, 1f 14137c478bd9Sstevel@tonic-gate andn %o5, LOFAULT_SET, %o5 14147c478bd9Sstevel@tonic-gate membar #Sync ! sync error barrier 14157c478bd9Sstevel@tonic-gate stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault 14167c478bd9Sstevel@tonic-gate1: 14177c478bd9Sstevel@tonic-gate retl 14187c478bd9Sstevel@tonic-gate clr %o0 ! return (0) 14197c478bd9Sstevel@tonic-gate 14207c478bd9Sstevel@tonic-gate SET_SIZE(bzero) 1421