1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27	.file	"memcpy.s"
28
29#include <sys/asm_linkage.h>
30
31	ANSI_PRAGMA_WEAK(memmove,function)
32	ANSI_PRAGMA_WEAK(memcpy,function)
33
34	ENTRY(memmove)
35	movl	0+12(%esp),%ecx	/ get number of bytes to move
36	pushl	%esi		/ save off %edi, %esi and move destination
37	pushl	%edi
38	movl	8+ 4(%esp),%edi	/ destination buffer address
39	movl	8+ 8(%esp),%esi	/ source buffer address
40	movl	%edi, %eax
41	testl	%ecx,%ecx
42	jz	.Return
43
44	cmpl	%esi,%edi	/ if (source addr > dest addr)
45	leal	-1(%esi,%ecx),%edx	/ %edx = src + size - 1
46	jbe	.memcpy_post	/ jump if dst <= src
47	cmpl	%edx,%edi
48	jbe	.CopyLeft	/ jump if dst <= src + size - 1
49	jmp	.memcpy_post
50
51	ENTRY(memcpy)
52	pushl	%esi
53	pushl	%edi
54
55	movl	8+4(%esp),%edi	/ %edi = dest address
56	movl	%edi, %eax	/ save this
57	movl	8+8(%esp),%esi	/ %esi = source address
58	movl	8+12(%esp),%ecx/ %ecx = length of string
59				/ %edx scratch register
60				/ %eax scratch register
61.memcpy_post:
62	nop			/ this really helps, don't know why
63				/ note:	cld is perf death on P4
64	cmpl	$63,%ecx
65	ja	.move_sse	/ not worth doing sse for less
66
67.movew:
68	movl	%ecx,%edx	/ save byte cnt
69	shrl	$2,%ecx		/ %ecx = number of words to move
70	rep ; smovl		/ move the words
71
72
73	andl	$0x3,%edx	/ %edx = number of bytes left to move
74	jz	.Return		/ %edx <= 3, so just unroll the loop
75
76	movb	(%esi), %cl
77	movb	%cl, (%edi)
78	decl	%edx
79	jz	.Return
80	movb	1(%esi), %cl
81	movb	%cl, 1(%edi)
82	decl	%edx
83	jz	.Return
84	movb	2(%esi), %cl
85	movb	%cl, 2(%edi)
86
87.Return:
88	popl	%edi		/ restore register variables
89	popl	%esi
90	ret
91
92.move_sse:
93	/
94	/ time to 16 byte align destination
95	/
96	andl	$15, %eax
97	jnz	.sse_unaligned	/ jmp if dest is unaligned
98.sse:				/ dest is aligned, check source
99	movl	%ecx, %edx	/ get byte count
100	shrl	$6, %edx	/ number of 64 byte blocks to move
101	testl	$15, %esi
102	jnz	.sse_da		/ go to slow loop if source is unaligned
103	cmpl	$65535, %ecx
104	ja	.sse_sa_nt_loop
105
106	/
107	/ use aligned load since we're lucky
108	/
109.sse_sa_loop:
110	prefetcht0 568(%esi)	/ prefetch source & copy 64 byte at a time
111	prefetcht0 568(%edi)	/ prefetch source & copy 64 byte at a time
112	movaps	0(%esi), %xmm0
113	movaps	%xmm0, 0(%edi)
114	movaps	16(%esi), %xmm1
115	movaps	%xmm1, 16(%edi)
116	movaps	32(%esi), %xmm2
117	movaps	%xmm2, 32(%edi)
118	movaps	48(%esi), %xmm3
119	movaps	%xmm3, 48(%edi)
120	addl	$64, %esi
121	addl	$64, %edi
122	decl	%edx
123	jnz	.sse_sa_loop
124
125.sse_cleanup:
126	andl	$63, %ecx	/ compute remaining bytes
127	movl	8+4(%esp), %eax	/ setup return value
128	jz	.Return
129	jmp	.movew
130
131	/
132	/ use aligned load since we're lucky
133	/
134	.align 16
135.sse_sa_nt_loop:
136	prefetchnta 16384(%esi)	/ prefetch source & copy 64 byte at a time
137	movaps	(%esi), %xmm0
138	movntps	%xmm0, 0(%edi)
139	movaps	16(%esi), %xmm1
140	movntps	%xmm1, 16(%edi)
141	movaps	32(%esi), %xmm2
142	movntps	%xmm2, 32(%edi)
143	movaps	48(%esi), %xmm3
144	movntps	%xmm3, 48(%edi)
145	addl	$64, %esi
146	addl	$64, %edi
147	decl	%edx
148	jnz	.sse_sa_nt_loop
149#if defined(_SSE2_INSN)
150	mfence
151#elif defined(_SSE_INSN)
152	sfence
153#else
154#error "Must have either SSE or SSE2"
155#endif
156	jmp	.sse_cleanup
157
158	/
159	/ Make certain that destination buffer becomes aligned
160	/
161.sse_unaligned:
162	neg	%eax		/ subtract from 16 and get destination
163	andl	$15, %eax	/ aligned on a 16 byte boundary
164	movl	%ecx, %edx	/ saved count
165	subl	%eax, %ecx	/ subtract from byte count
166	cmpl	$64, %ecx	/ after aligning, will we still have 64 bytes?
167	cmovb	%edx, %ecx	/ if not, restore original byte count,
168	cmovb	8+4(%esp), %eax	/ and restore return value,
169	jb	.movew		/ and do a non-SSE move.
170	xchg	%ecx, %eax	/ flip for copy
171	rep ; smovb		/ move the bytes
172	xchg	%ecx, %eax	/ flip back
173	jmp	.sse
174
175	.align 16
176.sse_da:
177	cmpl	$65535, %ecx
178	jbe	.sse_da_loop
179
180	/
181	/ use unaligned load since source doesn't line up
182	/
183.sse_da_nt_loop:
184	prefetchnta 16384(%esi)	/ prefetch source & copy 64 byte at a time
185	movups	0(%esi), %xmm0
186	movntps	%xmm0, 0(%edi)
187	movups	16(%esi), %xmm1
188	movntps	%xmm1, 16(%edi)
189	movups	32(%esi), %xmm2
190	movntps	%xmm2, 32(%edi)
191	movups	48(%esi), %xmm3
192	movntps	%xmm3, 48(%edi)
193	addl	$64, %esi
194	addl	$64, %edi
195	decl	%edx
196	jnz	.sse_da_nt_loop
197#if defined(_SSE2_INSN)
198	mfence
199#elif defined(_SSE_INSN)
200	sfence
201#else
202#error "Must have either SSE or SSE2"
203#endif
204	jmp	.sse_cleanup
205	/
206	/ use unaligned load since source doesn't line up
207	/
208	.align	16
209.sse_da_loop:
210	prefetcht0 568(%esi)	/ prefetch source & copy 64 byte at a time
211	prefetcht0 568(%edi)
212	movups	0(%esi), %xmm0
213	movaps	%xmm0, 0(%edi)
214	movups	16(%esi), %xmm1
215	movaps	%xmm1, 16(%edi)
216	movups	32(%esi), %xmm2
217	movaps	%xmm2, 32(%edi)
218	movups	48(%esi), %xmm3
219	movaps	%xmm3, 48(%edi)
220	addl	$64, %esi
221	addl	$64, %edi
222	decl	%edx
223	jnz	.sse_da_loop
224	jmp	.sse_cleanup
225
226	SET_SIZE(memcpy)
227
228
229/ .CopyLeft handles the memmove case where we must perform the copy backwards,
230/ because of overlap between src and dst. This is not particularly optimized.
231
232.CopyLeft:
233	movl	$3,%eax			/ heavily used constant
234	std				/ reverse direction bit (RtoL)
235	cmpl	$12,%ecx		/ if (size < 12)
236	ja	.BigCopyLeft		/ {
237	movl	%edx,%esi		/     src = src + size - 1
238	leal	-1(%ecx,%edi),%edi	/     dst = dst + size - 1
239	rep;	smovb			/    do the byte copy
240	cld				/    reset direction flag to LtoR
241	popl	%edi			/  }
242	popl	%esi			/  restore registers
243	movl	4(%esp),%eax		/  set up return value
244	ret				/  return(dba);
245.BigCopyLeft:				/ } else {
246	xchgl	%edx,%ecx
247	movl	%ecx,%esi		/ align source w/byte copy
248	leal	-1(%edx,%edi),%edi
249	andl	%eax,%ecx
250	jz	.SkipAlignLeft
251	addl	$1, %ecx		/ we need to insure that future
252	subl	%ecx,%edx		/ copy is done on aligned boundary
253	rep;	smovb
254.SkipAlignLeft:
255	movl	%edx,%ecx
256	subl	%eax,%esi
257	shrl	$2,%ecx			/ do 4 byte copy RtoL
258	subl	%eax,%edi
259	rep;	smovl
260	andl	%eax,%edx		/ do 1 byte copy whats left
261	jz	.CleanupReturnLeft
262	movl	%edx,%ecx
263	addl	%eax,%esi		/ rep; smovl instruction will decrement
264	addl	%eax,%edi		/ %edi, %esi by four after each copy
265					/ adding 3 will restore pointers to byte
266					/ before last double word copied
267					/ which is where they are expected to
268					/ be for the single byte copy code
269	rep;	smovb
270.CleanupReturnLeft:
271	cld				/ reset direction flag to LtoR
272	popl	%edi
273	popl	%esi			/ restore registers
274	movl	4(%esp),%eax		/ set up return value
275	ret				/ return(dba);
276	SET_SIZE(memmove)
277