xref: /illumos-gate/usr/src/uts/sun4/ml/ip_ocsum.s (revision 9b0bb795)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License").  You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22/*
23 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27#include <sys/param.h>
28#include <sys/errno.h>
29#include <sys/asm_linkage.h>
30#include <sys/vtrace.h>
31#include <sys/machthread.h>
32#include <sys/machparam.h>
33
34#include "assym.h"
35
36/*
37 * Prefetch considerations
38 *
39 * We prefetch one cacheline ahead.  This may not be enough on Serengeti
40 * systems - see default_copyout() etc which prefetch 5 lines ahead.
41 * On the other hand, we expect most of the source buffers to be
42 * recently used enough to be cached.
43 *
44 * On US-I the prefetches are inoperative.  On US-II they preload the E$;
45 * the mainloop unrolling and load-buffer should cover loads from E$.
46 * The stores appear to be the slow point on US-II.
47 *
48 * On US-IIICu the prefetch preloads the L2$ too, but there is no load
49 * buffer so the loads will stall for D$ miss, L2$ hit.  The hardware
50 * auto-prefetch is not activated by integer loads.  No solution
51 * in sight for this, barring odd games with FP read, write, integer read.
52 *
53 * US-IV (Panther) appears similar to US-IIICu, except that a strong
54 * variant of prefetch is available which can take TLB traps.  We don't
55 * use this.  The h/w prefetch stride can be set to 64, 128 or 192,
56 * and they only reach to the L2$ (we don't use these either).
57 * L2$ load-to-use latency is 15 cycles (best).
58 */
59
60
61/*
62 * ip_ocsum(address, halfword_count, sum)
63 * Do a 16 bit one's complement sum of a given number of (16-bit)
64 * halfwords. The halfword pointer must not be odd.
65 *	%o0 address; %o1 count; %o2 sum accumulator; %o4 temp
66 * 	%g2 and %g3 used in main loop
67 *
68 * (from @(#)ocsum.s 1.3 89/02/24 SMI)
69 *
70 */
71
72	ENTRY(ip_ocsum)
73
74/*
75 * On ttcp transmits, called once per ocsum_copyin but with a small
76 * block ( >99.9% ).  Could be the tx hdrs?  How many acks/seg are we rxing?
77 * On ttcp receives, called more than once per ocsum_copyout. Rx hdrs
78 * and tx acks?
79 *
80 * To do: telnet and nfs traffic
81 *
82 * On an NCA'd webserver about 10% of the calls are >64 bytes
83 *	about 10% of those start on a 64byte boundary
84 *	about 30% are >5*64 bytes.
85 * The NCA numbers & proportions don't change with h/w cksum on.
86 *
87 * Tx hdrs are likely to be already in cache.
88 * Rx hdrs depends if already inspected.
89 */
90
91	!
92	! Entry point for checksum-only.
93	! %o0 contains buffer address
94	! %o1 contains count of 16bit words
95	! %o2 contains sum
96	!
97	! %o3 temporary
98	! %o4 temporary
99	! %g1 32bit mask
100	! %g4 16bit mask
101	! %g5 64bit mask (all 1s)
102	!
103	not	%g0, %g5	! all 1's
104	prefetch [%o0], #n_reads	! first hword, dword, cacheline
105
106	clruw	%g5, %g1	! 32 1's at low end
107	srl	%g5, 16, %g4	! 16 1's at low end
108
109	cmp	%o1, 32		! at least a cacheline (64 bytes)?
110	bge,pn %icc, ip_ocsum_long	! yes, do the whole works
111	andn	%o0, 7, %o5	! delay: base src addr
112
113
114	cmp	%o1, 4		! < 4 halfwords?
115	bl,pn	%icc, .tiny	! < 4 halfwords, just do them
116	inc	8, %o5		! delay: next addr (no matter for .tiny)
117
118	/* leading dword with 1-4 hwords: 9 clocks */
119	/* Assumes ok to read the entire dword with the leading hwords */
120
121	ldx	[%o5-8], %o3	! NB base addr
122	sub	%o5, %o0, %g2	! byte count: 2/4/6/8
123	mov	%o5, %o0
124
125	sll	%g2, 2, %g2	! 8/16/24/32 for mask
126
127	sllx	%g5, %g2, %o5
128
129	sllx	%o5, %g2, %o5	! mask: 16/32/48/64 0's at low end
130
131	srl	%g2, 3, %g2	! hw count
132	andn	%o3, %o5, %o3	! select hw's from src
133
134	srlx	%o3, 32, %o4	! hi32
135	b	9f
136	sub	%o1, %g2, %o1	! delay: decr count, 1-4 halfwords
137
138.short_dw:			! max 7 iters of 4 clocks; 1 mispred of 4
139	ldx	[%o0], %o3	! tmp64 = *src++ (groups with the branch)
140
141	inc	8, %o0		! (D-cache load-use delay)
142	dec	4, %o1		! decrement count, 4 halfwords
143
144	srlx	%o3, 32, %o4	! hi32
1459:	and	%o3, %g1, %o3	! lo32
146
147	add	%o4, %o2, %o2	! accumulator
148	andncc	%o1, 3, %g0	! more than 3 hwords left?
149
150	bnz,pt %icc, .short_dw
151	add	%o3, %o2, %o2	! accumulator
152
153.short_hw:			! trailing dw: 0-3 hwords
154	tst	%o1		! 0 seems fairly common...
155	bz,a	.short_fold
156	srlx	%o2, 32, %o4	! delay: hi32
157				! mispredict 4 + 7 clocks for 1-3
158	ldx	[%o0], %o3
159	sll	%o1, 4, %o1	! bitcount: 16/32/48
160
161	srlx	%g5, %o1, %o5	! mask: 16/32/48  0's at high end
162
163	andn	%o3, %o5, %o3	! select hw's from src
164
165	srlx	%o3, 32, %o4	! hi32
166	and	%o3, %g1, %o3	! lo32
167
168	add	%o4, %o2, %o2	! accumulator
169
170	add	%o3, %o2, %o2	! accumulator
171
172	! at this point the 64-bit accumulator
173	! has the result that needs to be returned in 16-bits
174	srlx	%o2, 32, %o4	! hi32
175.short_fold:
176	and	%o2, %g1, %o2	! lo32
177
178	add	%o4, %o2, %o2	! 33b
179
180	srlx	%o2, 16, %o3	! hi17
181	and	%o2, %g4, %o2	! lo16
182
183	add	%o3, %o2, %o2	! 18b
184
185	srlx	%o2, 16, %o3	! hi2
186	and	%o2, %g4, %o2	! lo16
187
188	retl			! return
189	add	%o3, %o2, %o0	! 16b result in %o0
190
191.tiny:				! almost never: less than 4 halfwords total.
192	tst	%o1
193	bz,a	.short_fold
194
195	srlx	%o2, 32, %o4	! delay: hi32
196
197	lduh	[%o0], %o3	! tmp16 = *src++
1981:
199	inc	2, %o0
200				! stall for D-cache
201
202	add	%o3, %o2, %o2	! accumulator
203
204	deccc	%o1		! decrement count
205	bnz,a,pt %icc, 1b
206	lduh	[%o0], %o3	! tmp16 = *src++
207
208	! at this point the 64-bit accumulator
209	! has the result that needs to be returned in 16-bits
210	b	.short_fold
211	srlx	%o2, 32, %o4	! hi32
212
213	SET_SIZE(ip_ocsum)	! 64-bit version
214
215
216	ENTRY(ip_ocsum_long)	! 64-bit, large blocks
217	save	%sp, -SA(MINFRAME), %sp	! get another window
218	!
219	! %i0 contains buffer address
220	! %i1 contains count of 16bit words
221	! %i2 contains sum
222	! %i4 contains the mainloop count
223	! %i5 comes in with the buffer address rounded down to the first dword
224	!
225	! %g1 32bit mask
226	! %g4 16bit mask
227	! %g5 64bit mask (all 1s)
228	! %g6 fetch-ahead offset for Ecache
229	!
230	! %l0-7,%o0-5,%g2-3 mainloop temporaries
231	!
232	!
233				! 1 clock overhead
234	btst	63, %i0		! src 64-byte aligned?
235	bz,a,pt	%icc, .mainsection	! aligned blocks are fairly common
236	andncc	%i1, 31, %i4	! at least 64 bytes for main loop?
237
238
239	! Leading dword, with 1-4 hwords: 9 clocks
240	! Assumes ok to read the entire dword with the leading bytes
241	ldx	[%i5], %l0	! NB base addr
242	inc	8, %i5		! next addr
243
244	sub	%i5, %i0, %l2	! byte count: 2/4/6/8
245	mov	%i5, %i0
246
247	sll	%l2, 2, %l2	! 8/16/24/32 for mask
248
249	sllx	%g5, %l2, %l4
250
251	sllx	%l4, %l2, %l4	! mask: 16, 32, 48, 64 0's at lsb
252
253	srl	%l2, 3, %l2	! 1/2/3/4 for count
254	andn	%l0, %l4, %l0	! select hw's from src
255
256	srlx	%l0, 32, %o0	! hi32
257	b	9f
258	sub	%i1, %l2, %i1	! decr count, 1-4 halfwords
259
260	! Do dwords until source is 64-byte aligned, 0-6 iterations
261	! 4 clocks per + 4 for 1 mispred = 16 clocks avg
262.dw:	ldx	[%i0], %l0	! tmp64 = *src++ (groups with the branch below)
263
264	inc	8, %i0		! (Dcache load-use delay)
265	dec	4, %i1		! decrement count, 4 halfwords
266
267	srlx	%l0, 32, %o0	! hi32
2689:	and	%l0, %g1, %l0	! lo32
269
270	add	%o0, %i2, %i2	! accumulator
271	btst	63, %i0		! src 64-byte aligned?
272
273	bnz,pt	%icc, .dw
274	add	%l0, %i2, %i2	! accumulator
275
276
277	! At this point source address is 64 byte aligned
278	! and we've dealt with 1-32 halfwords.
279	andncc	%i1, 31, %i4	! at least 64 bytes for main loop?
280.mainsection:				! total 18n + 21 clocks
281	bz,pn	%icc, .postamble
282	and	%i1, 31, %i1	! count for postamble
283
284	! preload for main loop - 9 clocks assuming D$ hits at 1 per
285	ldx	[%i0+0], %l0
286	ldx	[%i0+8], %l1
287	ldx	[%i0+16], %l2	! %l0 could be used here if Dcache hit
288	ldx	[%i0+24], %l3	!  but US-II prefetch only loads Ecache
289	ldx	[%i0+32], %l4	!  check on US-III: could mix preloads & splits?
290	ldx	[%i0+40], %l5
291	ldx	[%i0+48], %l6
292	ldx	[%i0+56], %l7
293	inc	64, %i0
294	prefetch [%i0], #n_reads
295
296	! main loop. Read 64 bytes at a time - 18 clocks per iteration
2975:	!					plus 4 for the exit mispredict
298	srlx	%l0, 32, %o0		! hi32 to %o0
299	and	%l0, %g1, %l0		! lo32 to %l0
300
301	srlx	%l1, 32, %o1		! hi32 to %o1
302	and	%l1, %g1, %l1		! lo32 to %l1
303
304	srlx	%l2, 32, %o2		! hi32 to %o2
305	and	%l2, %g1, %l2		! lo32 to %l2
306
307	srlx	%l3, 32, %o3		! hi32 to %o3
308	and	%l3, %g1, %l3		! lo32 to %l3
309
310	srlx	%l4, 32, %o4		! hi32 to %o4
311	and	%l4, %g1, %l4		! lo32 to %l4
312
313	srlx	%l5, 32, %o5		! hi32 to %o5
314	and	%l5, %g1, %l5		! lo32 to %l5
315
316	srlx	%l6, 32, %g2		! hi32 to %g2
317	and	%l6, %g1, %l6		! lo32 to %l6
318
319	srlx	%l7, 32, %g3		! hi32 to %g3
320	and	%l7, %g1, %l7		! lo32 to %l7
321				! splits gave 16 off 32b vals
322	deccc	32, %i4		! mv early,avoid mispredicts? nohelp US-II.
323	bz,pn	%icc, .looptidy	! count now zero?
324	add	%l0, %o0, %o0	! delay
325
326	ldx	[%i0+0], %l0
327	add	%l1, %o1, %o1	! adds and loads
328	add	%l2, %o2, %o2
329
330	ldx	[%i0+8], %l1
331	add	%l3, %o3, %o3
332	add	%l4, %o4, %o4
333
334	ldx	[%i0+16], %l2
335	add	%l5, %o5, %o5
336	add	%l6, %g2, %g2
337
338	ldx	[%i0+24], %l3
339	add	%l7, %g3, %g3		! now 8 off 33b vals
340	add	%o0, %o1, %o0
341
342	ldx	[%i0+32], %l4
343	add	%o2, %o3, %o1
344	add	%o4, %o5, %o2
345
346	ldx	[%i0+40], %l5
347	add	%g2, %g3, %o3		! now 4 off 34b vals
348	add	%o0, %o1, %o0
349
350	ldx	[%i0+48], %l6
351	add	%o2, %o3, %o1		! 2 off 35b
352
353	ldx	[%i0+56], %l7
354	add	%o0, %o1, %o0		! 36b
355	inc	64, %i0		! increment source address
356
357	add	%o0, %i2, %i2	! accumulator
358	ba	5b
359	prefetch [%i0], #n_reads	! next cacheline
360				! end of main loop
361.looptidy:	! compute remaining partial sum - 8 clocks
362	add	%l1, %o1, %o1
363	add	%l2, %o2, %o2
364
365	add	%l3, %o3, %o3
366	add	%l4, %o4, %o4
367
368	add	%l5, %o5, %o5
369	add	%l6, %g2, %g2
370
371	add	%l7, %g3, %g3		! 8 x 33b
372	add	%o0, %o1, %o0
373
374	add	%o2, %o3, %o1
375	add	%o4, %o5, %o2
376
377	add	%g2, %g3, %o3		! 4 x 34b
378	add	%o0, %o1, %o0
379
380	add	%o2, %o3, %o1		! 2 x 35b
381	add	%o0, %i2, %i2	! accumulator
382
383	add	%o1, %i2, %i2	! accumulator
384
385
386.postamble:
387	! postamble hword count is in %i1 (can be zero)
388	! while at least 1 dword, do dwords.   Max 7 iterations.
389	andncc	%i1, 3, %g0	! more than 3 hwords?
390.dotail_dw:
391	bz,a,pn	%icc, .dotail_hw
392	tst	%i1		! delay: any at all left?
3938:
394	ldx	[%i0], %l0	! tmp64 = *src++
395	inc	8, %i0
396	dec	4, %i1		! decrement count, 4 halfwords
397
398				! stall for D-cache
399
400	srlx	%l0, 32, %o0	! hi32
401	and	%l0, %g1, %l0	! lo32
402
403	add	%o0, %i2, %i2	! accumulator
404
405	andncc	%i1, 3, %g0	! more than 3 hwords?
406	bnz,pt	%icc, 8b
407	add	%l0, %i2, %i2	! accumulator
408
409	! while at least 1 hword, do hwords.   Max 3 iterations.
410	tst	%i1
411.dotail_hw:
412	bz,a	.fold
413	srlx	%i2, 32, %o0	! delay: hi32
414	lduh	[%i0], %l0	! tmp16 = *src++
4151:
416	inc	2, %i0
417				! stall for D-cache
418
419	add	%l0, %i2, %i2	! accumulator
420
421	deccc	%i1		! decrement count
422	bnz,a,pt %icc, 1b
423	lduh	[%i0], %l0	! tmp16 = *src++
424
425	! at this point the 64-bit accumulator
426	! has the result that needs to be returned in 16-bits
427	srlx	%i2, 32, %o0	! hi32
428.fold:
429	and	%i2, %g1, %o1	! lo32
430
431	add	%o0, %o1, %o0	! 33b
432
433	srlx	%o0, 16, %o1	! hi17
434	and	%o0, %g4, %o0	! lo16
435
436	add	%o1, %o0, %o0	! 18b
437
438	srlx	%o0, 16, %o1	! hi2
439	and	%o0, %g4, %o0	! lo16
440
441	add	%o1, %o0, %i0	! 16b result in %i0
442
443	ret			! return
444	restore
445
446
447	SET_SIZE(ip_ocsum_long)	! 64-bit version
448
449