xref: /illumos-gate/usr/src/common/zfs/zfs_fletcher.c (revision 770499e1)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 /*
26  * Copyright 2013 Saso Kiselkov. All rights reserved.
27  * Copyright (c) 2016 by Delphix. All rights reserved.
28  */
29 
30 /*
31  * Fletcher Checksums
32  * ------------------
33  *
34  * ZFS's 2nd and 4th order Fletcher checksums are defined by the following
35  * recurrence relations:
36  *
37  *	a  = a    + f
38  *	 i    i-1    i-1
39  *
40  *	b  = b    + a
41  *	 i    i-1    i
42  *
43  *	c  = c    + b		(fletcher-4 only)
44  *	 i    i-1    i
45  *
46  *	d  = d    + c		(fletcher-4 only)
47  *	 i    i-1    i
48  *
49  * Where
50  *	a_0 = b_0 = c_0 = d_0 = 0
51  * and
52  *	f_0 .. f_(n-1) are the input data.
53  *
54  * Using standard techniques, these translate into the following series:
55  *
56  *	     __n_			     __n_
57  *	     \   |			     \   |
58  *	a  =  >     f			b  =  >     i * f
59  *	 n   /___|   n - i		 n   /___|	 n - i
60  *	     i = 1			     i = 1
61  *
62  *
63  *	     __n_			     __n_
64  *	     \   |  i*(i+1)		     \   |  i*(i+1)*(i+2)
65  *	c  =  >     ------- f		d  =  >     ------------- f
66  *	 n   /___|     2     n - i	 n   /___|	  6	   n - i
67  *	     i = 1			     i = 1
68  *
69  * For fletcher-2, the f_is are 64-bit, and [ab]_i are 64-bit accumulators.
70  * Since the additions are done mod (2^64), errors in the high bits may not
71  * be noticed.  For this reason, fletcher-2 is deprecated.
72  *
73  * For fletcher-4, the f_is are 32-bit, and [abcd]_i are 64-bit accumulators.
74  * A conservative estimate of how big the buffer can get before we overflow
75  * can be estimated using f_i = 0xffffffff for all i:
76  *
77  * % bc
78  *  f=2^32-1;d=0; for (i = 1; d<2^64; i++) { d += f*i*(i+1)*(i+2)/6 }; (i-1)*4
79  * 2264
80  *  quit
81  * %
82  *
83  * So blocks of up to 2k will not overflow.  Our largest block size is
84  * 128k, which has 32k 4-byte words, so we can compute the largest possible
85  * accumulators, then divide by 2^64 to figure the max amount of overflow:
86  *
87  * % bc
88  *  a=b=c=d=0; f=2^32-1; for (i=1; i<=32*1024; i++) { a+=f; b+=a; c+=b; d+=c }
89  *  a/2^64;b/2^64;c/2^64;d/2^64
90  * 0
91  * 0
92  * 1365
93  * 11186858
94  *  quit
95  * %
96  *
97  * So a and b cannot overflow.  To make sure each bit of input has some
98  * effect on the contents of c and d, we can look at what the factors of
99  * the coefficients in the equations for c_n and d_n are.  The number of 2s
100  * in the factors determines the lowest set bit in the multiplier.  Running
101  * through the cases for n*(n+1)/2 reveals that the highest power of 2 is
102  * 2^14, and for n*(n+1)*(n+2)/6 it is 2^15.  So while some data may overflow
103  * the 64-bit accumulators, every bit of every f_i effects every accumulator,
104  * even for 128k blocks.
105  *
106  * If we wanted to make a stronger version of fletcher4 (fletcher4c?),
107  * we could do our calculations mod (2^32 - 1) by adding in the carries
108  * periodically, and store the number of carries in the top 32-bits.
109  *
110  * --------------------
111  * Checksum Performance
112  * --------------------
113  *
114  * There are two interesting components to checksum performance: cached and
115  * uncached performance.  With cached data, fletcher-2 is about four times
116  * faster than fletcher-4.  With uncached data, the performance difference is
117  * negligible, since the cost of a cache fill dominates the processing time.
118  * Even though fletcher-4 is slower than fletcher-2, it is still a pretty
119  * efficient pass over the data.
120  *
121  * In normal operation, the data which is being checksummed is in a buffer
122  * which has been filled either by:
123  *
124  *	1. a compression step, which will be mostly cached, or
125  *	2. a bcopy() or copyin(), which will be uncached (because the
126  *	   copy is cache-bypassing).
127  *
128  * For both cached and uncached data, both fletcher checksums are much faster
129  * than sha-256, and slower than 'off', which doesn't touch the data at all.
130  */
131 
132 #include <sys/types.h>
133 #include <sys/sysmacros.h>
134 #include <sys/byteorder.h>
135 #include <sys/zio.h>
136 #include <sys/spa.h>
137 #include <zfs_fletcher.h>
138 
139 void
fletcher_init(zio_cksum_t * zcp)140 fletcher_init(zio_cksum_t *zcp)
141 {
142 	ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0);
143 }
144 
145 int
fletcher_2_incremental_native(void * buf,size_t size,void * data)146 fletcher_2_incremental_native(void *buf, size_t size, void *data)
147 {
148 	zio_cksum_t *zcp = data;
149 
150 	const uint64_t *ip = buf;
151 	const uint64_t *ipend = ip + (size / sizeof (uint64_t));
152 	uint64_t a0, b0, a1, b1;
153 
154 	a0 = zcp->zc_word[0];
155 	a1 = zcp->zc_word[1];
156 	b0 = zcp->zc_word[2];
157 	b1 = zcp->zc_word[3];
158 
159 	for (; ip < ipend; ip += 2) {
160 		a0 += ip[0];
161 		a1 += ip[1];
162 		b0 += a0;
163 		b1 += a1;
164 	}
165 
166 	ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1);
167 	return (0);
168 }
169 
170 /*ARGSUSED*/
171 void
fletcher_2_native(const void * buf,size_t size,const void * ctx_template,zio_cksum_t * zcp)172 fletcher_2_native(const void *buf, size_t size,
173     const void *ctx_template, zio_cksum_t *zcp)
174 {
175 	fletcher_init(zcp);
176 	(void) fletcher_2_incremental_native((void *) buf, size, zcp);
177 }
178 
179 int
fletcher_2_incremental_byteswap(void * buf,size_t size,void * data)180 fletcher_2_incremental_byteswap(void *buf, size_t size, void *data)
181 {
182 	zio_cksum_t *zcp = data;
183 
184 	const uint64_t *ip = buf;
185 	const uint64_t *ipend = ip + (size / sizeof (uint64_t));
186 	uint64_t a0, b0, a1, b1;
187 
188 	a0 = zcp->zc_word[0];
189 	a1 = zcp->zc_word[1];
190 	b0 = zcp->zc_word[2];
191 	b1 = zcp->zc_word[3];
192 
193 	for (; ip < ipend; ip += 2) {
194 		a0 += BSWAP_64(ip[0]);
195 		a1 += BSWAP_64(ip[1]);
196 		b0 += a0;
197 		b1 += a1;
198 	}
199 
200 	ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1);
201 	return (0);
202 }
203 
204 /*ARGSUSED*/
205 void
fletcher_2_byteswap(const void * buf,size_t size,const void * ctx_template,zio_cksum_t * zcp)206 fletcher_2_byteswap(const void *buf, size_t size,
207     const void *ctx_template, zio_cksum_t *zcp)
208 {
209 	fletcher_init(zcp);
210 	(void) fletcher_2_incremental_byteswap((void *) buf, size, zcp);
211 }
212 
213 int
fletcher_4_incremental_native(void * buf,size_t size,void * data)214 fletcher_4_incremental_native(void *buf, size_t size, void *data)
215 {
216 	zio_cksum_t *zcp = data;
217 
218 	const uint32_t *ip = buf;
219 	const uint32_t *ipend = ip + (size / sizeof (uint32_t));
220 	uint64_t a, b, c, d;
221 
222 	a = zcp->zc_word[0];
223 	b = zcp->zc_word[1];
224 	c = zcp->zc_word[2];
225 	d = zcp->zc_word[3];
226 
227 	for (; ip < ipend; ip++) {
228 		a += ip[0];
229 		b += a;
230 		c += b;
231 		d += c;
232 	}
233 
234 	ZIO_SET_CHECKSUM(zcp, a, b, c, d);
235 	return (0);
236 }
237 
238 /*ARGSUSED*/
239 void
fletcher_4_native(const void * buf,size_t size,const void * ctx_template,zio_cksum_t * zcp)240 fletcher_4_native(const void *buf, size_t size,
241     const void *ctx_template, zio_cksum_t *zcp)
242 {
243 	fletcher_init(zcp);
244 	(void) fletcher_4_incremental_native((void *) buf, size, zcp);
245 }
246 
247 int
fletcher_4_incremental_byteswap(void * buf,size_t size,void * data)248 fletcher_4_incremental_byteswap(void *buf, size_t size, void *data)
249 {
250 	zio_cksum_t *zcp = data;
251 
252 	const uint32_t *ip = buf;
253 	const uint32_t *ipend = ip + (size / sizeof (uint32_t));
254 	uint64_t a, b, c, d;
255 
256 	a = zcp->zc_word[0];
257 	b = zcp->zc_word[1];
258 	c = zcp->zc_word[2];
259 	d = zcp->zc_word[3];
260 
261 	for (; ip < ipend; ip++) {
262 		a += BSWAP_32(ip[0]);
263 		b += a;
264 		c += b;
265 		d += c;
266 	}
267 
268 	ZIO_SET_CHECKSUM(zcp, a, b, c, d);
269 	return (0);
270 }
271 
272 /*ARGSUSED*/
273 void
fletcher_4_byteswap(const void * buf,size_t size,const void * ctx_template,zio_cksum_t * zcp)274 fletcher_4_byteswap(const void *buf, size_t size,
275     const void *ctx_template, zio_cksum_t *zcp)
276 {
277 	fletcher_init(zcp);
278 	(void) fletcher_4_incremental_byteswap((void *) buf, size, zcp);
279 }
280