1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25/*
26 * Copyright 2013 Saso Kiselkov. All rights reserved.
27 * Copyright (c) 2016 by Delphix. All rights reserved.
28 */
29
30/*
31 * Fletcher Checksums
32 * ------------------
33 *
34 * ZFS's 2nd and 4th order Fletcher checksums are defined by the following
35 * recurrence relations:
36 *
37 *	a  = a    + f
38 *	 i    i-1    i-1
39 *
40 *	b  = b    + a
41 *	 i    i-1    i
42 *
43 *	c  = c    + b		(fletcher-4 only)
44 *	 i    i-1    i
45 *
46 *	d  = d    + c		(fletcher-4 only)
47 *	 i    i-1    i
48 *
49 * Where
50 *	a_0 = b_0 = c_0 = d_0 = 0
51 * and
52 *	f_0 .. f_(n-1) are the input data.
53 *
54 * Using standard techniques, these translate into the following series:
55 *
56 *	     __n_			     __n_
57 *	     \   |			     \   |
58 *	a  =  >     f			b  =  >     i * f
59 *	 n   /___|   n - i		 n   /___|	 n - i
60 *	     i = 1			     i = 1
61 *
62 *
63 *	     __n_			     __n_
64 *	     \   |  i*(i+1)		     \   |  i*(i+1)*(i+2)
65 *	c  =  >     ------- f		d  =  >     ------------- f
66 *	 n   /___|     2     n - i	 n   /___|	  6	   n - i
67 *	     i = 1			     i = 1
68 *
69 * For fletcher-2, the f_is are 64-bit, and [ab]_i are 64-bit accumulators.
70 * Since the additions are done mod (2^64), errors in the high bits may not
71 * be noticed.  For this reason, fletcher-2 is deprecated.
72 *
73 * For fletcher-4, the f_is are 32-bit, and [abcd]_i are 64-bit accumulators.
74 * A conservative estimate of how big the buffer can get before we overflow
75 * can be estimated using f_i = 0xffffffff for all i:
76 *
77 * % bc
78 *  f=2^32-1;d=0; for (i = 1; d<2^64; i++) { d += f*i*(i+1)*(i+2)/6 }; (i-1)*4
79 * 2264
80 *  quit
81 * %
82 *
83 * So blocks of up to 2k will not overflow.  Our largest block size is
84 * 128k, which has 32k 4-byte words, so we can compute the largest possible
85 * accumulators, then divide by 2^64 to figure the max amount of overflow:
86 *
87 * % bc
88 *  a=b=c=d=0; f=2^32-1; for (i=1; i<=32*1024; i++) { a+=f; b+=a; c+=b; d+=c }
89 *  a/2^64;b/2^64;c/2^64;d/2^64
90 * 0
91 * 0
92 * 1365
93 * 11186858
94 *  quit
95 * %
96 *
97 * So a and b cannot overflow.  To make sure each bit of input has some
98 * effect on the contents of c and d, we can look at what the factors of
99 * the coefficients in the equations for c_n and d_n are.  The number of 2s
100 * in the factors determines the lowest set bit in the multiplier.  Running
101 * through the cases for n*(n+1)/2 reveals that the highest power of 2 is
102 * 2^14, and for n*(n+1)*(n+2)/6 it is 2^15.  So while some data may overflow
103 * the 64-bit accumulators, every bit of every f_i effects every accumulator,
104 * even for 128k blocks.
105 *
106 * If we wanted to make a stronger version of fletcher4 (fletcher4c?),
107 * we could do our calculations mod (2^32 - 1) by adding in the carries
108 * periodically, and store the number of carries in the top 32-bits.
109 *
110 * --------------------
111 * Checksum Performance
112 * --------------------
113 *
114 * There are two interesting components to checksum performance: cached and
115 * uncached performance.  With cached data, fletcher-2 is about four times
116 * faster than fletcher-4.  With uncached data, the performance difference is
117 * negligible, since the cost of a cache fill dominates the processing time.
118 * Even though fletcher-4 is slower than fletcher-2, it is still a pretty
119 * efficient pass over the data.
120 *
121 * In normal operation, the data which is being checksummed is in a buffer
122 * which has been filled either by:
123 *
124 *	1. a compression step, which will be mostly cached, or
125 *	2. a bcopy() or copyin(), which will be uncached (because the
126 *	   copy is cache-bypassing).
127 *
128 * For both cached and uncached data, both fletcher checksums are much faster
129 * than sha-256, and slower than 'off', which doesn't touch the data at all.
130 */
131
132#include <sys/types.h>
133#include <sys/sysmacros.h>
134#include <sys/byteorder.h>
135#include <sys/zio.h>
136#include <sys/spa.h>
137#include <zfs_fletcher.h>
138
139void
140fletcher_init(zio_cksum_t *zcp)
141{
142	ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0);
143}
144
145int
146fletcher_2_incremental_native(void *buf, size_t size, void *data)
147{
148	zio_cksum_t *zcp = data;
149
150	const uint64_t *ip = buf;
151	const uint64_t *ipend = ip + (size / sizeof (uint64_t));
152	uint64_t a0, b0, a1, b1;
153
154	a0 = zcp->zc_word[0];
155	a1 = zcp->zc_word[1];
156	b0 = zcp->zc_word[2];
157	b1 = zcp->zc_word[3];
158
159	for (; ip < ipend; ip += 2) {
160		a0 += ip[0];
161		a1 += ip[1];
162		b0 += a0;
163		b1 += a1;
164	}
165
166	ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1);
167	return (0);
168}
169
170/*ARGSUSED*/
171void
172fletcher_2_native(const void *buf, size_t size,
173    const void *ctx_template, zio_cksum_t *zcp)
174{
175	fletcher_init(zcp);
176	(void) fletcher_2_incremental_native((void *) buf, size, zcp);
177}
178
179int
180fletcher_2_incremental_byteswap(void *buf, size_t size, void *data)
181{
182	zio_cksum_t *zcp = data;
183
184	const uint64_t *ip = buf;
185	const uint64_t *ipend = ip + (size / sizeof (uint64_t));
186	uint64_t a0, b0, a1, b1;
187
188	a0 = zcp->zc_word[0];
189	a1 = zcp->zc_word[1];
190	b0 = zcp->zc_word[2];
191	b1 = zcp->zc_word[3];
192
193	for (; ip < ipend; ip += 2) {
194		a0 += BSWAP_64(ip[0]);
195		a1 += BSWAP_64(ip[1]);
196		b0 += a0;
197		b1 += a1;
198	}
199
200	ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1);
201	return (0);
202}
203
204/*ARGSUSED*/
205void
206fletcher_2_byteswap(const void *buf, size_t size,
207    const void *ctx_template, zio_cksum_t *zcp)
208{
209	fletcher_init(zcp);
210	(void) fletcher_2_incremental_byteswap((void *) buf, size, zcp);
211}
212
213int
214fletcher_4_incremental_native(void *buf, size_t size, void *data)
215{
216	zio_cksum_t *zcp = data;
217
218	const uint32_t *ip = buf;
219	const uint32_t *ipend = ip + (size / sizeof (uint32_t));
220	uint64_t a, b, c, d;
221
222	a = zcp->zc_word[0];
223	b = zcp->zc_word[1];
224	c = zcp->zc_word[2];
225	d = zcp->zc_word[3];
226
227	for (; ip < ipend; ip++) {
228		a += ip[0];
229		b += a;
230		c += b;
231		d += c;
232	}
233
234	ZIO_SET_CHECKSUM(zcp, a, b, c, d);
235	return (0);
236}
237
238/*ARGSUSED*/
239void
240fletcher_4_native(const void *buf, size_t size,
241    const void *ctx_template, zio_cksum_t *zcp)
242{
243	fletcher_init(zcp);
244	(void) fletcher_4_incremental_native((void *) buf, size, zcp);
245}
246
247int
248fletcher_4_incremental_byteswap(void *buf, size_t size, void *data)
249{
250	zio_cksum_t *zcp = data;
251
252	const uint32_t *ip = buf;
253	const uint32_t *ipend = ip + (size / sizeof (uint32_t));
254	uint64_t a, b, c, d;
255
256	a = zcp->zc_word[0];
257	b = zcp->zc_word[1];
258	c = zcp->zc_word[2];
259	d = zcp->zc_word[3];
260
261	for (; ip < ipend; ip++) {
262		a += BSWAP_32(ip[0]);
263		b += a;
264		c += b;
265		d += c;
266	}
267
268	ZIO_SET_CHECKSUM(zcp, a, b, c, d);
269	return (0);
270}
271
272/*ARGSUSED*/
273void
274fletcher_4_byteswap(const void *buf, size_t size,
275    const void *ctx_template, zio_cksum_t *zcp)
276{
277	fletcher_init(zcp);
278	(void) fletcher_4_incremental_byteswap((void *) buf, size, zcp);
279}
280