1*e86372a0SGvozden Neskovic /*
2*e86372a0SGvozden Neskovic * CDDL HEADER START
3*e86372a0SGvozden Neskovic *
4*e86372a0SGvozden Neskovic * The contents of this file are subject to the terms of the
5*e86372a0SGvozden Neskovic * Common Development and Distribution License (the "License").
6*e86372a0SGvozden Neskovic * You may not use this file except in compliance with the License.
7*e86372a0SGvozden Neskovic *
8*e86372a0SGvozden Neskovic * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9*e86372a0SGvozden Neskovic * or http://www.opensolaris.org/os/licensing.
10*e86372a0SGvozden Neskovic * See the License for the specific language governing permissions
11*e86372a0SGvozden Neskovic * and limitations under the License.
12*e86372a0SGvozden Neskovic *
13*e86372a0SGvozden Neskovic * When distributing Covered Code, include this CDDL HEADER in each
14*e86372a0SGvozden Neskovic * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15*e86372a0SGvozden Neskovic * If applicable, add the following below this CDDL HEADER, with the
16*e86372a0SGvozden Neskovic * fields enclosed by brackets "[]" replaced with your own identifying
17*e86372a0SGvozden Neskovic * information: Portions Copyright [yyyy] [name of copyright owner]
18*e86372a0SGvozden Neskovic *
19*e86372a0SGvozden Neskovic * CDDL HEADER END
20*e86372a0SGvozden Neskovic */
21*e86372a0SGvozden Neskovic /*
22*e86372a0SGvozden Neskovic * Copyright (C) 2016 Gvozden Nešković. All rights reserved.
23*e86372a0SGvozden Neskovic */
24*e86372a0SGvozden Neskovic
25*e86372a0SGvozden Neskovic #ifndef _VDEV_RAIDZ_MATH_IMPL_H
26*e86372a0SGvozden Neskovic #define _VDEV_RAIDZ_MATH_IMPL_H
27*e86372a0SGvozden Neskovic
28*e86372a0SGvozden Neskovic #include <sys/types.h>
29*e86372a0SGvozden Neskovic
30*e86372a0SGvozden Neskovic #define raidz_inline inline __attribute__((always_inline))
31*e86372a0SGvozden Neskovic #ifndef noinline
32*e86372a0SGvozden Neskovic #define noinline __attribute__((noinline))
33*e86372a0SGvozden Neskovic #endif
34*e86372a0SGvozden Neskovic
35*e86372a0SGvozden Neskovic /*
36*e86372a0SGvozden Neskovic * Functions calculate multiplication constants for data reconstruction.
37*e86372a0SGvozden Neskovic * Coefficients depend on RAIDZ geometry, indexes of failed child vdevs, and
38*e86372a0SGvozden Neskovic * used parity columns for reconstruction.
39*e86372a0SGvozden Neskovic * @rm RAIDZ map
40*e86372a0SGvozden Neskovic * @tgtidx array of missing data indexes
41*e86372a0SGvozden Neskovic * @coeff output array of coefficients. Array must be provided by
42*e86372a0SGvozden Neskovic * user and must hold minimum MUL_CNT values.
43*e86372a0SGvozden Neskovic */
44*e86372a0SGvozden Neskovic static noinline void
raidz_rec_q_coeff(const raidz_map_t * rm,const int * tgtidx,unsigned * coeff)45*e86372a0SGvozden Neskovic raidz_rec_q_coeff(const raidz_map_t *rm, const int *tgtidx, unsigned *coeff)
46*e86372a0SGvozden Neskovic {
47*e86372a0SGvozden Neskovic const unsigned ncols = raidz_ncols(rm);
48*e86372a0SGvozden Neskovic const unsigned x = tgtidx[TARGET_X];
49*e86372a0SGvozden Neskovic
50*e86372a0SGvozden Neskovic coeff[MUL_Q_X] = gf_exp2(255 - (ncols - x - 1));
51*e86372a0SGvozden Neskovic }
52*e86372a0SGvozden Neskovic
53*e86372a0SGvozden Neskovic static noinline void
raidz_rec_r_coeff(const raidz_map_t * rm,const int * tgtidx,unsigned * coeff)54*e86372a0SGvozden Neskovic raidz_rec_r_coeff(const raidz_map_t *rm, const int *tgtidx, unsigned *coeff)
55*e86372a0SGvozden Neskovic {
56*e86372a0SGvozden Neskovic const unsigned ncols = raidz_ncols(rm);
57*e86372a0SGvozden Neskovic const unsigned x = tgtidx[TARGET_X];
58*e86372a0SGvozden Neskovic
59*e86372a0SGvozden Neskovic coeff[MUL_R_X] = gf_exp4(255 - (ncols - x - 1));
60*e86372a0SGvozden Neskovic }
61*e86372a0SGvozden Neskovic
62*e86372a0SGvozden Neskovic static noinline void
raidz_rec_pq_coeff(const raidz_map_t * rm,const int * tgtidx,unsigned * coeff)63*e86372a0SGvozden Neskovic raidz_rec_pq_coeff(const raidz_map_t *rm, const int *tgtidx, unsigned *coeff)
64*e86372a0SGvozden Neskovic {
65*e86372a0SGvozden Neskovic const unsigned ncols = raidz_ncols(rm);
66*e86372a0SGvozden Neskovic const unsigned x = tgtidx[TARGET_X];
67*e86372a0SGvozden Neskovic const unsigned y = tgtidx[TARGET_Y];
68*e86372a0SGvozden Neskovic gf_t a, b, e;
69*e86372a0SGvozden Neskovic
70*e86372a0SGvozden Neskovic a = gf_exp2(x + 255 - y);
71*e86372a0SGvozden Neskovic b = gf_exp2(255 - (ncols - x - 1));
72*e86372a0SGvozden Neskovic e = a ^ 0x01;
73*e86372a0SGvozden Neskovic
74*e86372a0SGvozden Neskovic coeff[MUL_PQ_X] = gf_div(a, e);
75*e86372a0SGvozden Neskovic coeff[MUL_PQ_Y] = gf_div(b, e);
76*e86372a0SGvozden Neskovic }
77*e86372a0SGvozden Neskovic
78*e86372a0SGvozden Neskovic static noinline void
raidz_rec_pr_coeff(const raidz_map_t * rm,const int * tgtidx,unsigned * coeff)79*e86372a0SGvozden Neskovic raidz_rec_pr_coeff(const raidz_map_t *rm, const int *tgtidx, unsigned *coeff)
80*e86372a0SGvozden Neskovic {
81*e86372a0SGvozden Neskovic const unsigned ncols = raidz_ncols(rm);
82*e86372a0SGvozden Neskovic const unsigned x = tgtidx[TARGET_X];
83*e86372a0SGvozden Neskovic const unsigned y = tgtidx[TARGET_Y];
84*e86372a0SGvozden Neskovic
85*e86372a0SGvozden Neskovic gf_t a, b, e;
86*e86372a0SGvozden Neskovic
87*e86372a0SGvozden Neskovic a = gf_exp4(x + 255 - y);
88*e86372a0SGvozden Neskovic b = gf_exp4(255 - (ncols - x - 1));
89*e86372a0SGvozden Neskovic e = a ^ 0x01;
90*e86372a0SGvozden Neskovic
91*e86372a0SGvozden Neskovic coeff[MUL_PR_X] = gf_div(a, e);
92*e86372a0SGvozden Neskovic coeff[MUL_PR_Y] = gf_div(b, e);
93*e86372a0SGvozden Neskovic }
94*e86372a0SGvozden Neskovic
95*e86372a0SGvozden Neskovic static noinline void
raidz_rec_qr_coeff(const raidz_map_t * rm,const int * tgtidx,unsigned * coeff)96*e86372a0SGvozden Neskovic raidz_rec_qr_coeff(const raidz_map_t *rm, const int *tgtidx, unsigned *coeff)
97*e86372a0SGvozden Neskovic {
98*e86372a0SGvozden Neskovic const unsigned ncols = raidz_ncols(rm);
99*e86372a0SGvozden Neskovic const unsigned x = tgtidx[TARGET_X];
100*e86372a0SGvozden Neskovic const unsigned y = tgtidx[TARGET_Y];
101*e86372a0SGvozden Neskovic
102*e86372a0SGvozden Neskovic gf_t nx, ny, nxxy, nxyy, d;
103*e86372a0SGvozden Neskovic
104*e86372a0SGvozden Neskovic nx = gf_exp2(ncols - x - 1);
105*e86372a0SGvozden Neskovic ny = gf_exp2(ncols - y - 1);
106*e86372a0SGvozden Neskovic nxxy = gf_mul(gf_mul(nx, nx), ny);
107*e86372a0SGvozden Neskovic nxyy = gf_mul(gf_mul(nx, ny), ny);
108*e86372a0SGvozden Neskovic d = nxxy ^ nxyy;
109*e86372a0SGvozden Neskovic
110*e86372a0SGvozden Neskovic coeff[MUL_QR_XQ] = ny;
111*e86372a0SGvozden Neskovic coeff[MUL_QR_X] = gf_div(ny, d);
112*e86372a0SGvozden Neskovic coeff[MUL_QR_YQ] = nx;
113*e86372a0SGvozden Neskovic coeff[MUL_QR_Y] = gf_div(nx, d);
114*e86372a0SGvozden Neskovic }
115*e86372a0SGvozden Neskovic
116*e86372a0SGvozden Neskovic static noinline void
raidz_rec_pqr_coeff(const raidz_map_t * rm,const int * tgtidx,unsigned * coeff)117*e86372a0SGvozden Neskovic raidz_rec_pqr_coeff(const raidz_map_t *rm, const int *tgtidx, unsigned *coeff)
118*e86372a0SGvozden Neskovic {
119*e86372a0SGvozden Neskovic const unsigned ncols = raidz_ncols(rm);
120*e86372a0SGvozden Neskovic const unsigned x = tgtidx[TARGET_X];
121*e86372a0SGvozden Neskovic const unsigned y = tgtidx[TARGET_Y];
122*e86372a0SGvozden Neskovic const unsigned z = tgtidx[TARGET_Z];
123*e86372a0SGvozden Neskovic
124*e86372a0SGvozden Neskovic gf_t nx, ny, nz, nxx, nyy, nzz, nyyz, nyzz, xd, yd;
125*e86372a0SGvozden Neskovic
126*e86372a0SGvozden Neskovic nx = gf_exp2(ncols - x - 1);
127*e86372a0SGvozden Neskovic ny = gf_exp2(ncols - y - 1);
128*e86372a0SGvozden Neskovic nz = gf_exp2(ncols - z - 1);
129*e86372a0SGvozden Neskovic
130*e86372a0SGvozden Neskovic nxx = gf_exp4(ncols - x - 1);
131*e86372a0SGvozden Neskovic nyy = gf_exp4(ncols - y - 1);
132*e86372a0SGvozden Neskovic nzz = gf_exp4(ncols - z - 1);
133*e86372a0SGvozden Neskovic
134*e86372a0SGvozden Neskovic nyyz = gf_mul(gf_mul(ny, nz), ny);
135*e86372a0SGvozden Neskovic nyzz = gf_mul(nzz, ny);
136*e86372a0SGvozden Neskovic
137*e86372a0SGvozden Neskovic xd = gf_mul(nxx, ny) ^ gf_mul(nx, nyy) ^ nyyz ^
138*e86372a0SGvozden Neskovic gf_mul(nxx, nz) ^ gf_mul(nzz, nx) ^ nyzz;
139*e86372a0SGvozden Neskovic
140*e86372a0SGvozden Neskovic yd = gf_inv(ny ^ nz);
141*e86372a0SGvozden Neskovic
142*e86372a0SGvozden Neskovic coeff[MUL_PQR_XP] = gf_div(nyyz ^ nyzz, xd);
143*e86372a0SGvozden Neskovic coeff[MUL_PQR_XQ] = gf_div(nyy ^ nzz, xd);
144*e86372a0SGvozden Neskovic coeff[MUL_PQR_XR] = gf_div(ny ^ nz, xd);
145*e86372a0SGvozden Neskovic coeff[MUL_PQR_YU] = nx;
146*e86372a0SGvozden Neskovic coeff[MUL_PQR_YP] = gf_mul(nz, yd);
147*e86372a0SGvozden Neskovic coeff[MUL_PQR_YQ] = yd;
148*e86372a0SGvozden Neskovic }
149*e86372a0SGvozden Neskovic
150*e86372a0SGvozden Neskovic /*
151*e86372a0SGvozden Neskovic * Method for zeroing a buffer (can be implemented using SIMD).
152*e86372a0SGvozden Neskovic * This method is used by multiple for gen/rec functions.
153*e86372a0SGvozden Neskovic *
154*e86372a0SGvozden Neskovic * @dc Destination buffer
155*e86372a0SGvozden Neskovic * @dsize Destination buffer size
156*e86372a0SGvozden Neskovic * @private Unused
157*e86372a0SGvozden Neskovic */
158*e86372a0SGvozden Neskovic static int
raidz_zero_abd_cb(void * dc,size_t dsize,void * private)159*e86372a0SGvozden Neskovic raidz_zero_abd_cb(void *dc, size_t dsize, void *private)
160*e86372a0SGvozden Neskovic {
161*e86372a0SGvozden Neskovic v_t *dst = (v_t *)dc;
162*e86372a0SGvozden Neskovic size_t i;
163*e86372a0SGvozden Neskovic
164*e86372a0SGvozden Neskovic ZERO_DEFINE();
165*e86372a0SGvozden Neskovic
166*e86372a0SGvozden Neskovic (void) private; /* unused */
167*e86372a0SGvozden Neskovic
168*e86372a0SGvozden Neskovic ZERO(ZERO_D);
169*e86372a0SGvozden Neskovic
170*e86372a0SGvozden Neskovic for (i = 0; i < dsize / sizeof (v_t); i += (2 * ZERO_STRIDE)) {
171*e86372a0SGvozden Neskovic STORE(dst + i, ZERO_D);
172*e86372a0SGvozden Neskovic STORE(dst + i + ZERO_STRIDE, ZERO_D);
173*e86372a0SGvozden Neskovic }
174*e86372a0SGvozden Neskovic
175*e86372a0SGvozden Neskovic return (0);
176*e86372a0SGvozden Neskovic }
177*e86372a0SGvozden Neskovic
178*e86372a0SGvozden Neskovic #define raidz_zero(dabd, size) \
179*e86372a0SGvozden Neskovic { \
180*e86372a0SGvozden Neskovic abd_iterate_func(dabd, 0, size, raidz_zero_abd_cb, NULL); \
181*e86372a0SGvozden Neskovic }
182*e86372a0SGvozden Neskovic
183*e86372a0SGvozden Neskovic /*
184*e86372a0SGvozden Neskovic * Method for copying two buffers (can be implemented using SIMD).
185*e86372a0SGvozden Neskovic * This method is used by multiple for gen/rec functions.
186*e86372a0SGvozden Neskovic *
187*e86372a0SGvozden Neskovic * @dc Destination buffer
188*e86372a0SGvozden Neskovic * @sc Source buffer
189*e86372a0SGvozden Neskovic * @dsize Destination buffer size
190*e86372a0SGvozden Neskovic * @ssize Source buffer size
191*e86372a0SGvozden Neskovic * @private Unused
192*e86372a0SGvozden Neskovic */
193*e86372a0SGvozden Neskovic static int
raidz_copy_abd_cb(void * dc,void * sc,size_t size,void * private)194*e86372a0SGvozden Neskovic raidz_copy_abd_cb(void *dc, void *sc, size_t size, void *private)
195*e86372a0SGvozden Neskovic {
196*e86372a0SGvozden Neskovic v_t *dst = (v_t *)dc;
197*e86372a0SGvozden Neskovic const v_t *src = (v_t *)sc;
198*e86372a0SGvozden Neskovic size_t i;
199*e86372a0SGvozden Neskovic
200*e86372a0SGvozden Neskovic COPY_DEFINE();
201*e86372a0SGvozden Neskovic
202*e86372a0SGvozden Neskovic (void) private; /* unused */
203*e86372a0SGvozden Neskovic
204*e86372a0SGvozden Neskovic for (i = 0; i < size / sizeof (v_t); i += (2 * COPY_STRIDE)) {
205*e86372a0SGvozden Neskovic LOAD(src + i, COPY_D);
206*e86372a0SGvozden Neskovic STORE(dst + i, COPY_D);
207*e86372a0SGvozden Neskovic
208*e86372a0SGvozden Neskovic LOAD(src + i + COPY_STRIDE, COPY_D);
209*e86372a0SGvozden Neskovic STORE(dst + i + COPY_STRIDE, COPY_D);
210*e86372a0SGvozden Neskovic }
211*e86372a0SGvozden Neskovic
212*e86372a0SGvozden Neskovic return (0);
213*e86372a0SGvozden Neskovic }
214*e86372a0SGvozden Neskovic
215*e86372a0SGvozden Neskovic
216*e86372a0SGvozden Neskovic #define raidz_copy(dabd, sabd, size) \
217*e86372a0SGvozden Neskovic { \
218*e86372a0SGvozden Neskovic abd_iterate_func2(dabd, sabd, 0, 0, size, raidz_copy_abd_cb, NULL);\
219*e86372a0SGvozden Neskovic }
220*e86372a0SGvozden Neskovic
221*e86372a0SGvozden Neskovic /*
222*e86372a0SGvozden Neskovic * Method for adding (XORing) two buffers.
223*e86372a0SGvozden Neskovic * Source and destination are XORed together and result is stored in
224*e86372a0SGvozden Neskovic * destination buffer. This method is used by multiple for gen/rec functions.
225*e86372a0SGvozden Neskovic *
226*e86372a0SGvozden Neskovic * @dc Destination buffer
227*e86372a0SGvozden Neskovic * @sc Source buffer
228*e86372a0SGvozden Neskovic * @dsize Destination buffer size
229*e86372a0SGvozden Neskovic * @ssize Source buffer size
230*e86372a0SGvozden Neskovic * @private Unused
231*e86372a0SGvozden Neskovic */
232*e86372a0SGvozden Neskovic static int
raidz_add_abd_cb(void * dc,void * sc,size_t size,void * private)233*e86372a0SGvozden Neskovic raidz_add_abd_cb(void *dc, void *sc, size_t size, void *private)
234*e86372a0SGvozden Neskovic {
235*e86372a0SGvozden Neskovic v_t *dst = (v_t *)dc;
236*e86372a0SGvozden Neskovic const v_t *src = (v_t *)sc;
237*e86372a0SGvozden Neskovic size_t i;
238*e86372a0SGvozden Neskovic
239*e86372a0SGvozden Neskovic ADD_DEFINE();
240*e86372a0SGvozden Neskovic
241*e86372a0SGvozden Neskovic (void) private; /* unused */
242*e86372a0SGvozden Neskovic
243*e86372a0SGvozden Neskovic for (i = 0; i < size / sizeof (v_t); i += (2 * ADD_STRIDE)) {
244*e86372a0SGvozden Neskovic LOAD(dst + i, ADD_D);
245*e86372a0SGvozden Neskovic XOR_ACC(src + i, ADD_D);
246*e86372a0SGvozden Neskovic STORE(dst + i, ADD_D);
247*e86372a0SGvozden Neskovic
248*e86372a0SGvozden Neskovic LOAD(dst + i + ADD_STRIDE, ADD_D);
249*e86372a0SGvozden Neskovic XOR_ACC(src + i + ADD_STRIDE, ADD_D);
250*e86372a0SGvozden Neskovic STORE(dst + i + ADD_STRIDE, ADD_D);
251*e86372a0SGvozden Neskovic }
252*e86372a0SGvozden Neskovic
253*e86372a0SGvozden Neskovic return (0);
254*e86372a0SGvozden Neskovic }
255*e86372a0SGvozden Neskovic
256*e86372a0SGvozden Neskovic #define raidz_add(dabd, sabd, size) \
257*e86372a0SGvozden Neskovic { \
258*e86372a0SGvozden Neskovic abd_iterate_func2(dabd, sabd, 0, 0, size, raidz_add_abd_cb, NULL);\
259*e86372a0SGvozden Neskovic }
260*e86372a0SGvozden Neskovic
261*e86372a0SGvozden Neskovic /*
262*e86372a0SGvozden Neskovic * Method for multiplying a buffer with a constant in GF(2^8).
263*e86372a0SGvozden Neskovic * Symbols from buffer are multiplied by a constant and result is stored
264*e86372a0SGvozden Neskovic * back in the same buffer.
265*e86372a0SGvozden Neskovic *
266*e86372a0SGvozden Neskovic * @dc In/Out data buffer.
267*e86372a0SGvozden Neskovic * @size Size of the buffer
268*e86372a0SGvozden Neskovic * @private pointer to the multiplication constant (unsigned)
269*e86372a0SGvozden Neskovic */
270*e86372a0SGvozden Neskovic static int
raidz_mul_abd_cb(void * dc,size_t size,void * private)271*e86372a0SGvozden Neskovic raidz_mul_abd_cb(void *dc, size_t size, void *private)
272*e86372a0SGvozden Neskovic {
273*e86372a0SGvozden Neskovic const unsigned mul = *((unsigned *)private);
274*e86372a0SGvozden Neskovic v_t *d = (v_t *)dc;
275*e86372a0SGvozden Neskovic size_t i;
276*e86372a0SGvozden Neskovic
277*e86372a0SGvozden Neskovic MUL_DEFINE();
278*e86372a0SGvozden Neskovic
279*e86372a0SGvozden Neskovic for (i = 0; i < size / sizeof (v_t); i += (2 * MUL_STRIDE)) {
280*e86372a0SGvozden Neskovic LOAD(d + i, MUL_D);
281*e86372a0SGvozden Neskovic MUL(mul, MUL_D);
282*e86372a0SGvozden Neskovic STORE(d + i, MUL_D);
283*e86372a0SGvozden Neskovic
284*e86372a0SGvozden Neskovic LOAD(d + i + MUL_STRIDE, MUL_D);
285*e86372a0SGvozden Neskovic MUL(mul, MUL_D);
286*e86372a0SGvozden Neskovic STORE(d + i + MUL_STRIDE, MUL_D);
287*e86372a0SGvozden Neskovic }
288*e86372a0SGvozden Neskovic
289*e86372a0SGvozden Neskovic return (0);
290*e86372a0SGvozden Neskovic }
291*e86372a0SGvozden Neskovic
292*e86372a0SGvozden Neskovic
293*e86372a0SGvozden Neskovic /*
294*e86372a0SGvozden Neskovic * Syndrome generation/update macros
295*e86372a0SGvozden Neskovic *
296*e86372a0SGvozden Neskovic * Require LOAD(), XOR(), STORE(), MUL2(), and MUL4() macros
297*e86372a0SGvozden Neskovic */
298*e86372a0SGvozden Neskovic #define P_D_SYNDROME(D, T, t) \
299*e86372a0SGvozden Neskovic { \
300*e86372a0SGvozden Neskovic LOAD((t), T); \
301*e86372a0SGvozden Neskovic XOR(D, T); \
302*e86372a0SGvozden Neskovic STORE((t), T); \
303*e86372a0SGvozden Neskovic }
304*e86372a0SGvozden Neskovic
305*e86372a0SGvozden Neskovic #define Q_D_SYNDROME(D, T, t) \
306*e86372a0SGvozden Neskovic { \
307*e86372a0SGvozden Neskovic LOAD((t), T); \
308*e86372a0SGvozden Neskovic MUL2(T); \
309*e86372a0SGvozden Neskovic XOR(D, T); \
310*e86372a0SGvozden Neskovic STORE((t), T); \
311*e86372a0SGvozden Neskovic }
312*e86372a0SGvozden Neskovic
313*e86372a0SGvozden Neskovic #define Q_SYNDROME(T, t) \
314*e86372a0SGvozden Neskovic { \
315*e86372a0SGvozden Neskovic LOAD((t), T); \
316*e86372a0SGvozden Neskovic MUL2(T); \
317*e86372a0SGvozden Neskovic STORE((t), T); \
318*e86372a0SGvozden Neskovic }
319*e86372a0SGvozden Neskovic
320*e86372a0SGvozden Neskovic #define R_D_SYNDROME(D, T, t) \
321*e86372a0SGvozden Neskovic { \
322*e86372a0SGvozden Neskovic LOAD((t), T); \
323*e86372a0SGvozden Neskovic MUL4(T); \
324*e86372a0SGvozden Neskovic XOR(D, T); \
325*e86372a0SGvozden Neskovic STORE((t), T); \
326*e86372a0SGvozden Neskovic }
327*e86372a0SGvozden Neskovic
328*e86372a0SGvozden Neskovic #define R_SYNDROME(T, t) \
329*e86372a0SGvozden Neskovic { \
330*e86372a0SGvozden Neskovic LOAD((t), T); \
331*e86372a0SGvozden Neskovic MUL4(T); \
332*e86372a0SGvozden Neskovic STORE((t), T); \
333*e86372a0SGvozden Neskovic }
334*e86372a0SGvozden Neskovic
335*e86372a0SGvozden Neskovic
336*e86372a0SGvozden Neskovic /*
337*e86372a0SGvozden Neskovic * PARITY CALCULATION
338*e86372a0SGvozden Neskovic *
339*e86372a0SGvozden Neskovic * Macros *_SYNDROME are used for parity/syndrome calculation.
340*e86372a0SGvozden Neskovic * *_D_SYNDROME() macros are used to calculate syndrome between 0 and
341*e86372a0SGvozden Neskovic * length of data column, and *_SYNDROME() macros are only for updating
342*e86372a0SGvozden Neskovic * the parity/syndrome if data column is shorter.
343*e86372a0SGvozden Neskovic *
344*e86372a0SGvozden Neskovic * P parity is calculated using raidz_add_abd().
345*e86372a0SGvozden Neskovic */
346*e86372a0SGvozden Neskovic
347*e86372a0SGvozden Neskovic /*
348*e86372a0SGvozden Neskovic * Generate P parity (RAIDZ1)
349*e86372a0SGvozden Neskovic *
350*e86372a0SGvozden Neskovic * @rm RAIDZ map
351*e86372a0SGvozden Neskovic */
352*e86372a0SGvozden Neskovic static raidz_inline void
raidz_generate_p_impl(raidz_map_t * const rm)353*e86372a0SGvozden Neskovic raidz_generate_p_impl(raidz_map_t * const rm)
354*e86372a0SGvozden Neskovic {
355*e86372a0SGvozden Neskovic size_t c;
356*e86372a0SGvozden Neskovic const size_t ncols = raidz_ncols(rm);
357*e86372a0SGvozden Neskovic const size_t psize = rm->rm_col[CODE_P].rc_size;
358*e86372a0SGvozden Neskovic abd_t *pabd = rm->rm_col[CODE_P].rc_abd;
359*e86372a0SGvozden Neskovic size_t size;
360*e86372a0SGvozden Neskovic abd_t *dabd;
361*e86372a0SGvozden Neskovic
362*e86372a0SGvozden Neskovic raidz_math_begin();
363*e86372a0SGvozden Neskovic
364*e86372a0SGvozden Neskovic /* start with first data column */
365*e86372a0SGvozden Neskovic raidz_copy(pabd, rm->rm_col[1].rc_abd, psize);
366*e86372a0SGvozden Neskovic
367*e86372a0SGvozden Neskovic for (c = 2; c < ncols; c++) {
368*e86372a0SGvozden Neskovic dabd = rm->rm_col[c].rc_abd;
369*e86372a0SGvozden Neskovic size = rm->rm_col[c].rc_size;
370*e86372a0SGvozden Neskovic
371*e86372a0SGvozden Neskovic /* add data column */
372*e86372a0SGvozden Neskovic raidz_add(pabd, dabd, size);
373*e86372a0SGvozden Neskovic }
374*e86372a0SGvozden Neskovic
375*e86372a0SGvozden Neskovic raidz_math_end();
376*e86372a0SGvozden Neskovic }
377*e86372a0SGvozden Neskovic
378*e86372a0SGvozden Neskovic
379*e86372a0SGvozden Neskovic /*
380*e86372a0SGvozden Neskovic * Generate PQ parity (RAIDZ2)
381*e86372a0SGvozden Neskovic * The function is called per data column.
382*e86372a0SGvozden Neskovic *
383*e86372a0SGvozden Neskovic * @c array of pointers to parity (code) columns
384*e86372a0SGvozden Neskovic * @dc pointer to data column
385*e86372a0SGvozden Neskovic * @csize size of parity columns
386*e86372a0SGvozden Neskovic * @dsize size of data column
387*e86372a0SGvozden Neskovic */
388*e86372a0SGvozden Neskovic static void
raidz_gen_pq_add(void ** c,const void * dc,const size_t csize,const size_t dsize)389*e86372a0SGvozden Neskovic raidz_gen_pq_add(void **c, const void *dc, const size_t csize,
390*e86372a0SGvozden Neskovic const size_t dsize)
391*e86372a0SGvozden Neskovic {
392*e86372a0SGvozden Neskovic v_t *p = (v_t *)c[0];
393*e86372a0SGvozden Neskovic v_t *q = (v_t *)c[1];
394*e86372a0SGvozden Neskovic const v_t *d = (const v_t *)dc;
395*e86372a0SGvozden Neskovic const v_t * const dend = d + (dsize / sizeof (v_t));
396*e86372a0SGvozden Neskovic const v_t * const qend = q + (csize / sizeof (v_t));
397*e86372a0SGvozden Neskovic
398*e86372a0SGvozden Neskovic GEN_PQ_DEFINE();
399*e86372a0SGvozden Neskovic
400*e86372a0SGvozden Neskovic MUL2_SETUP();
401*e86372a0SGvozden Neskovic
402*e86372a0SGvozden Neskovic for (; d < dend; d += GEN_PQ_STRIDE, p += GEN_PQ_STRIDE,
403*e86372a0SGvozden Neskovic q += GEN_PQ_STRIDE) {
404*e86372a0SGvozden Neskovic LOAD(d, GEN_PQ_D);
405*e86372a0SGvozden Neskovic P_D_SYNDROME(GEN_PQ_D, GEN_PQ_C, p);
406*e86372a0SGvozden Neskovic Q_D_SYNDROME(GEN_PQ_D, GEN_PQ_C, q);
407*e86372a0SGvozden Neskovic }
408*e86372a0SGvozden Neskovic for (; q < qend; q += GEN_PQ_STRIDE) {
409*e86372a0SGvozden Neskovic Q_SYNDROME(GEN_PQ_C, q);
410*e86372a0SGvozden Neskovic }
411*e86372a0SGvozden Neskovic }
412*e86372a0SGvozden Neskovic
413*e86372a0SGvozden Neskovic
414*e86372a0SGvozden Neskovic /*
415*e86372a0SGvozden Neskovic * Generate PQ parity (RAIDZ2)
416*e86372a0SGvozden Neskovic *
417*e86372a0SGvozden Neskovic * @rm RAIDZ map
418*e86372a0SGvozden Neskovic */
419*e86372a0SGvozden Neskovic static raidz_inline void
raidz_generate_pq_impl(raidz_map_t * const rm)420*e86372a0SGvozden Neskovic raidz_generate_pq_impl(raidz_map_t * const rm)
421*e86372a0SGvozden Neskovic {
422*e86372a0SGvozden Neskovic size_t c;
423*e86372a0SGvozden Neskovic const size_t ncols = raidz_ncols(rm);
424*e86372a0SGvozden Neskovic const size_t csize = rm->rm_col[CODE_P].rc_size;
425*e86372a0SGvozden Neskovic size_t dsize;
426*e86372a0SGvozden Neskovic abd_t *dabd;
427*e86372a0SGvozden Neskovic abd_t *cabds[] = {
428*e86372a0SGvozden Neskovic rm->rm_col[CODE_P].rc_abd,
429*e86372a0SGvozden Neskovic rm->rm_col[CODE_Q].rc_abd
430*e86372a0SGvozden Neskovic };
431*e86372a0SGvozden Neskovic
432*e86372a0SGvozden Neskovic raidz_math_begin();
433*e86372a0SGvozden Neskovic
434*e86372a0SGvozden Neskovic raidz_copy(cabds[CODE_P], rm->rm_col[2].rc_abd, csize);
435*e86372a0SGvozden Neskovic raidz_copy(cabds[CODE_Q], rm->rm_col[2].rc_abd, csize);
436*e86372a0SGvozden Neskovic
437*e86372a0SGvozden Neskovic for (c = 3; c < ncols; c++) {
438*e86372a0SGvozden Neskovic dabd = rm->rm_col[c].rc_abd;
439*e86372a0SGvozden Neskovic dsize = rm->rm_col[c].rc_size;
440*e86372a0SGvozden Neskovic
441*e86372a0SGvozden Neskovic abd_raidz_gen_iterate(cabds, dabd, csize, dsize, 2,
442*e86372a0SGvozden Neskovic raidz_gen_pq_add);
443*e86372a0SGvozden Neskovic }
444*e86372a0SGvozden Neskovic
445*e86372a0SGvozden Neskovic raidz_math_end();
446*e86372a0SGvozden Neskovic }
447*e86372a0SGvozden Neskovic
448*e86372a0SGvozden Neskovic
449*e86372a0SGvozden Neskovic /*
450*e86372a0SGvozden Neskovic * Generate PQR parity (RAIDZ3)
451*e86372a0SGvozden Neskovic * The function is called per data column.
452*e86372a0SGvozden Neskovic *
453*e86372a0SGvozden Neskovic * @c array of pointers to parity (code) columns
454*e86372a0SGvozden Neskovic * @dc pointer to data column
455*e86372a0SGvozden Neskovic * @csize size of parity columns
456*e86372a0SGvozden Neskovic * @dsize size of data column
457*e86372a0SGvozden Neskovic */
458*e86372a0SGvozden Neskovic static void
raidz_gen_pqr_add(void ** c,const void * dc,const size_t csize,const size_t dsize)459*e86372a0SGvozden Neskovic raidz_gen_pqr_add(void **c, const void *dc, const size_t csize,
460*e86372a0SGvozden Neskovic const size_t dsize)
461*e86372a0SGvozden Neskovic {
462*e86372a0SGvozden Neskovic v_t *p = (v_t *)c[0];
463*e86372a0SGvozden Neskovic v_t *q = (v_t *)c[1];
464*e86372a0SGvozden Neskovic v_t *r = (v_t *)c[CODE_R];
465*e86372a0SGvozden Neskovic const v_t *d = (const v_t *)dc;
466*e86372a0SGvozden Neskovic const v_t * const dend = d + (dsize / sizeof (v_t));
467*e86372a0SGvozden Neskovic const v_t * const qend = q + (csize / sizeof (v_t));
468*e86372a0SGvozden Neskovic
469*e86372a0SGvozden Neskovic GEN_PQR_DEFINE();
470*e86372a0SGvozden Neskovic
471*e86372a0SGvozden Neskovic MUL2_SETUP();
472*e86372a0SGvozden Neskovic
473*e86372a0SGvozden Neskovic for (; d < dend; d += GEN_PQR_STRIDE, p += GEN_PQR_STRIDE,
474*e86372a0SGvozden Neskovic q += GEN_PQR_STRIDE, r += GEN_PQR_STRIDE) {
475*e86372a0SGvozden Neskovic LOAD(d, GEN_PQR_D);
476*e86372a0SGvozden Neskovic P_D_SYNDROME(GEN_PQR_D, GEN_PQR_C, p);
477*e86372a0SGvozden Neskovic Q_D_SYNDROME(GEN_PQR_D, GEN_PQR_C, q);
478*e86372a0SGvozden Neskovic R_D_SYNDROME(GEN_PQR_D, GEN_PQR_C, r);
479*e86372a0SGvozden Neskovic }
480*e86372a0SGvozden Neskovic for (; q < qend; q += GEN_PQR_STRIDE, r += GEN_PQR_STRIDE) {
481*e86372a0SGvozden Neskovic Q_SYNDROME(GEN_PQR_C, q);
482*e86372a0SGvozden Neskovic R_SYNDROME(GEN_PQR_C, r);
483*e86372a0SGvozden Neskovic }
484*e86372a0SGvozden Neskovic }
485*e86372a0SGvozden Neskovic
486*e86372a0SGvozden Neskovic
487*e86372a0SGvozden Neskovic /*
488*e86372a0SGvozden Neskovic * Generate PQR parity (RAIDZ2)
489*e86372a0SGvozden Neskovic *
490*e86372a0SGvozden Neskovic * @rm RAIDZ map
491*e86372a0SGvozden Neskovic */
492*e86372a0SGvozden Neskovic static raidz_inline void
raidz_generate_pqr_impl(raidz_map_t * const rm)493*e86372a0SGvozden Neskovic raidz_generate_pqr_impl(raidz_map_t * const rm)
494*e86372a0SGvozden Neskovic {
495*e86372a0SGvozden Neskovic size_t c;
496*e86372a0SGvozden Neskovic const size_t ncols = raidz_ncols(rm);
497*e86372a0SGvozden Neskovic const size_t csize = rm->rm_col[CODE_P].rc_size;
498*e86372a0SGvozden Neskovic size_t dsize;
499*e86372a0SGvozden Neskovic abd_t *dabd;
500*e86372a0SGvozden Neskovic abd_t *cabds[] = {
501*e86372a0SGvozden Neskovic rm->rm_col[CODE_P].rc_abd,
502*e86372a0SGvozden Neskovic rm->rm_col[CODE_Q].rc_abd,
503*e86372a0SGvozden Neskovic rm->rm_col[CODE_R].rc_abd
504*e86372a0SGvozden Neskovic };
505*e86372a0SGvozden Neskovic
506*e86372a0SGvozden Neskovic raidz_math_begin();
507*e86372a0SGvozden Neskovic
508*e86372a0SGvozden Neskovic raidz_copy(cabds[CODE_P], rm->rm_col[3].rc_abd, csize);
509*e86372a0SGvozden Neskovic raidz_copy(cabds[CODE_Q], rm->rm_col[3].rc_abd, csize);
510*e86372a0SGvozden Neskovic raidz_copy(cabds[CODE_R], rm->rm_col[3].rc_abd, csize);
511*e86372a0SGvozden Neskovic
512*e86372a0SGvozden Neskovic for (c = 4; c < ncols; c++) {
513*e86372a0SGvozden Neskovic dabd = rm->rm_col[c].rc_abd;
514*e86372a0SGvozden Neskovic dsize = rm->rm_col[c].rc_size;
515*e86372a0SGvozden Neskovic
516*e86372a0SGvozden Neskovic abd_raidz_gen_iterate(cabds, dabd, csize, dsize, 3,
517*e86372a0SGvozden Neskovic raidz_gen_pqr_add);
518*e86372a0SGvozden Neskovic }
519*e86372a0SGvozden Neskovic
520*e86372a0SGvozden Neskovic raidz_math_end();
521*e86372a0SGvozden Neskovic }
522*e86372a0SGvozden Neskovic
523*e86372a0SGvozden Neskovic
524*e86372a0SGvozden Neskovic /*
525*e86372a0SGvozden Neskovic * DATA RECONSTRUCTION
526*e86372a0SGvozden Neskovic *
527*e86372a0SGvozden Neskovic * Data reconstruction process consists of two phases:
528*e86372a0SGvozden Neskovic * - Syndrome calculation
529*e86372a0SGvozden Neskovic * - Data reconstruction
530*e86372a0SGvozden Neskovic *
531*e86372a0SGvozden Neskovic * Syndrome is calculated by generating parity using available data columns
532*e86372a0SGvozden Neskovic * and zeros in places of erasure. Existing parity is added to corresponding
533*e86372a0SGvozden Neskovic * syndrome value to obtain the [P|Q|R]syn values from equation:
534*e86372a0SGvozden Neskovic * P = Psyn + Dx + Dy + Dz
535*e86372a0SGvozden Neskovic * Q = Qsyn + 2^x * Dx + 2^y * Dy + 2^z * Dz
536*e86372a0SGvozden Neskovic * R = Rsyn + 4^x * Dx + 4^y * Dy + 4^z * Dz
537*e86372a0SGvozden Neskovic *
538*e86372a0SGvozden Neskovic * For data reconstruction phase, the corresponding equations are solved
539*e86372a0SGvozden Neskovic * for missing data (Dx, Dy, Dz). This generally involves multiplying known
540*e86372a0SGvozden Neskovic * symbols by an coefficient and adding them together. The multiplication
541*e86372a0SGvozden Neskovic * constant coefficients are calculated ahead of the operation in
542*e86372a0SGvozden Neskovic * raidz_rec_[q|r|pq|pq|qr|pqr]_coeff() functions.
543*e86372a0SGvozden Neskovic *
544*e86372a0SGvozden Neskovic * IMPLEMENTATION NOTE: RAID-Z block can have complex geometry, with "big"
545*e86372a0SGvozden Neskovic * and "short" columns.
546*e86372a0SGvozden Neskovic * For this reason, reconstruction is performed in minimum of
547*e86372a0SGvozden Neskovic * two steps. First, from offset 0 to short_size, then from short_size to
548*e86372a0SGvozden Neskovic * short_size. Calculation functions REC_[*]_BLOCK() are implemented to work
549*e86372a0SGvozden Neskovic * over both ranges. The split also enables removal of conditional expressions
550*e86372a0SGvozden Neskovic * from loop bodies, improving throughput of SIMD implementations.
551*e86372a0SGvozden Neskovic * For the best performance, all functions marked with raidz_inline attribute
552*e86372a0SGvozden Neskovic * must be inlined by compiler.
553*e86372a0SGvozden Neskovic *
554*e86372a0SGvozden Neskovic * parity data
555*e86372a0SGvozden Neskovic * columns columns
556*e86372a0SGvozden Neskovic * <----------> <------------------>
557*e86372a0SGvozden Neskovic * x y <----+ missing columns (x, y)
558*e86372a0SGvozden Neskovic * | |
559*e86372a0SGvozden Neskovic * +---+---+---+---+-v-+---+-v-+---+ ^ 0
560*e86372a0SGvozden Neskovic * | | | | | | | | | |
561*e86372a0SGvozden Neskovic * | | | | | | | | | |
562*e86372a0SGvozden Neskovic * | P | Q | R | D | D | D | D | D | |
563*e86372a0SGvozden Neskovic * | | | | 0 | 1 | 2 | 3 | 4 | |
564*e86372a0SGvozden Neskovic * | | | | | | | | | v
565*e86372a0SGvozden Neskovic * | | | | | +---+---+---+ ^ short_size
566*e86372a0SGvozden Neskovic * | | | | | | |
567*e86372a0SGvozden Neskovic * +---+---+---+---+---+ v big_size
568*e86372a0SGvozden Neskovic * <------------------> <---------->
569*e86372a0SGvozden Neskovic * big columns short columns
570*e86372a0SGvozden Neskovic *
571*e86372a0SGvozden Neskovic */
572*e86372a0SGvozden Neskovic
573*e86372a0SGvozden Neskovic
574*e86372a0SGvozden Neskovic
575*e86372a0SGvozden Neskovic
576*e86372a0SGvozden Neskovic /*
577*e86372a0SGvozden Neskovic * Reconstruct single data column using P parity
578*e86372a0SGvozden Neskovic *
579*e86372a0SGvozden Neskovic * @syn_method raidz_add_abd()
580*e86372a0SGvozden Neskovic * @rec_method not applicable
581*e86372a0SGvozden Neskovic *
582*e86372a0SGvozden Neskovic * @rm RAIDZ map
583*e86372a0SGvozden Neskovic * @tgtidx array of missing data indexes
584*e86372a0SGvozden Neskovic */
585*e86372a0SGvozden Neskovic static raidz_inline int
raidz_reconstruct_p_impl(raidz_map_t * rm,const int * tgtidx)586*e86372a0SGvozden Neskovic raidz_reconstruct_p_impl(raidz_map_t *rm, const int *tgtidx)
587*e86372a0SGvozden Neskovic {
588*e86372a0SGvozden Neskovic size_t c;
589*e86372a0SGvozden Neskovic const size_t firstdc = raidz_parity(rm);
590*e86372a0SGvozden Neskovic const size_t ncols = raidz_ncols(rm);
591*e86372a0SGvozden Neskovic const size_t x = tgtidx[TARGET_X];
592*e86372a0SGvozden Neskovic const size_t xsize = rm->rm_col[x].rc_size;
593*e86372a0SGvozden Neskovic abd_t *xabd = rm->rm_col[x].rc_abd;
594*e86372a0SGvozden Neskovic size_t size;
595*e86372a0SGvozden Neskovic abd_t *dabd;
596*e86372a0SGvozden Neskovic
597*e86372a0SGvozden Neskovic raidz_math_begin();
598*e86372a0SGvozden Neskovic
599*e86372a0SGvozden Neskovic /* copy P into target */
600*e86372a0SGvozden Neskovic raidz_copy(xabd, rm->rm_col[CODE_P].rc_abd, xsize);
601*e86372a0SGvozden Neskovic
602*e86372a0SGvozden Neskovic /* generate p_syndrome */
603*e86372a0SGvozden Neskovic for (c = firstdc; c < ncols; c++) {
604*e86372a0SGvozden Neskovic if (c == x)
605*e86372a0SGvozden Neskovic continue;
606*e86372a0SGvozden Neskovic
607*e86372a0SGvozden Neskovic dabd = rm->rm_col[c].rc_abd;
608*e86372a0SGvozden Neskovic size = MIN(rm->rm_col[c].rc_size, xsize);
609*e86372a0SGvozden Neskovic
610*e86372a0SGvozden Neskovic raidz_add(xabd, dabd, size);
611*e86372a0SGvozden Neskovic }
612*e86372a0SGvozden Neskovic
613*e86372a0SGvozden Neskovic raidz_math_end();
614*e86372a0SGvozden Neskovic
615*e86372a0SGvozden Neskovic return (1 << CODE_P);
616*e86372a0SGvozden Neskovic }
617*e86372a0SGvozden Neskovic
618*e86372a0SGvozden Neskovic
619*e86372a0SGvozden Neskovic /*
620*e86372a0SGvozden Neskovic * Generate Q syndrome (Qsyn)
621*e86372a0SGvozden Neskovic *
622*e86372a0SGvozden Neskovic * @xc array of pointers to syndrome columns
623*e86372a0SGvozden Neskovic * @dc data column (NULL if missing)
624*e86372a0SGvozden Neskovic * @xsize size of syndrome columns
625*e86372a0SGvozden Neskovic * @dsize size of data column (0 if missing)
626*e86372a0SGvozden Neskovic */
627*e86372a0SGvozden Neskovic static void
raidz_syn_q_abd(void ** xc,const void * dc,const size_t xsize,const size_t dsize)628*e86372a0SGvozden Neskovic raidz_syn_q_abd(void **xc, const void *dc, const size_t xsize,
629*e86372a0SGvozden Neskovic const size_t dsize)
630*e86372a0SGvozden Neskovic {
631*e86372a0SGvozden Neskovic v_t *x = (v_t *)xc[TARGET_X];
632*e86372a0SGvozden Neskovic const v_t *d = (const v_t *)dc;
633*e86372a0SGvozden Neskovic const v_t * const dend = d + (dsize / sizeof (v_t));
634*e86372a0SGvozden Neskovic const v_t * const xend = x + (xsize / sizeof (v_t));
635*e86372a0SGvozden Neskovic
636*e86372a0SGvozden Neskovic SYN_Q_DEFINE();
637*e86372a0SGvozden Neskovic
638*e86372a0SGvozden Neskovic MUL2_SETUP();
639*e86372a0SGvozden Neskovic
640*e86372a0SGvozden Neskovic for (; d < dend; d += SYN_STRIDE, x += SYN_STRIDE) {
641*e86372a0SGvozden Neskovic LOAD(d, SYN_Q_D);
642*e86372a0SGvozden Neskovic Q_D_SYNDROME(SYN_Q_D, SYN_Q_X, x);
643*e86372a0SGvozden Neskovic }
644*e86372a0SGvozden Neskovic for (; x < xend; x += SYN_STRIDE) {
645*e86372a0SGvozden Neskovic Q_SYNDROME(SYN_Q_X, x);
646*e86372a0SGvozden Neskovic }
647*e86372a0SGvozden Neskovic }
648*e86372a0SGvozden Neskovic
649*e86372a0SGvozden Neskovic
650*e86372a0SGvozden Neskovic /*
651*e86372a0SGvozden Neskovic * Reconstruct single data column using Q parity
652*e86372a0SGvozden Neskovic *
653*e86372a0SGvozden Neskovic * @syn_method raidz_add_abd()
654*e86372a0SGvozden Neskovic * @rec_method raidz_mul_abd_cb()
655*e86372a0SGvozden Neskovic *
656*e86372a0SGvozden Neskovic * @rm RAIDZ map
657*e86372a0SGvozden Neskovic * @tgtidx array of missing data indexes
658*e86372a0SGvozden Neskovic */
659*e86372a0SGvozden Neskovic static raidz_inline int
raidz_reconstruct_q_impl(raidz_map_t * rm,const int * tgtidx)660*e86372a0SGvozden Neskovic raidz_reconstruct_q_impl(raidz_map_t *rm, const int *tgtidx)
661*e86372a0SGvozden Neskovic {
662*e86372a0SGvozden Neskovic size_t c;
663*e86372a0SGvozden Neskovic size_t dsize;
664*e86372a0SGvozden Neskovic abd_t *dabd;
665*e86372a0SGvozden Neskovic const size_t firstdc = raidz_parity(rm);
666*e86372a0SGvozden Neskovic const size_t ncols = raidz_ncols(rm);
667*e86372a0SGvozden Neskovic const size_t x = tgtidx[TARGET_X];
668*e86372a0SGvozden Neskovic abd_t *xabd = rm->rm_col[x].rc_abd;
669*e86372a0SGvozden Neskovic const size_t xsize = rm->rm_col[x].rc_size;
670*e86372a0SGvozden Neskovic abd_t *tabds[] = { xabd };
671*e86372a0SGvozden Neskovic
672*e86372a0SGvozden Neskovic unsigned coeff[MUL_CNT];
673*e86372a0SGvozden Neskovic raidz_rec_q_coeff(rm, tgtidx, coeff);
674*e86372a0SGvozden Neskovic
675*e86372a0SGvozden Neskovic raidz_math_begin();
676*e86372a0SGvozden Neskovic
677*e86372a0SGvozden Neskovic /* Start with first data column if present */
678*e86372a0SGvozden Neskovic if (firstdc != x) {
679*e86372a0SGvozden Neskovic raidz_copy(xabd, rm->rm_col[firstdc].rc_abd, xsize);
680*e86372a0SGvozden Neskovic } else {
681*e86372a0SGvozden Neskovic raidz_zero(xabd, xsize);
682*e86372a0SGvozden Neskovic }
683*e86372a0SGvozden Neskovic
684*e86372a0SGvozden Neskovic /* generate q_syndrome */
685*e86372a0SGvozden Neskovic for (c = firstdc+1; c < ncols; c++) {
686*e86372a0SGvozden Neskovic if (c == x) {
687*e86372a0SGvozden Neskovic dabd = NULL;
688*e86372a0SGvozden Neskovic dsize = 0;
689*e86372a0SGvozden Neskovic } else {
690*e86372a0SGvozden Neskovic dabd = rm->rm_col[c].rc_abd;
691*e86372a0SGvozden Neskovic dsize = rm->rm_col[c].rc_size;
692*e86372a0SGvozden Neskovic }
693*e86372a0SGvozden Neskovic
694*e86372a0SGvozden Neskovic abd_raidz_gen_iterate(tabds, dabd, xsize, dsize, 1,
695*e86372a0SGvozden Neskovic raidz_syn_q_abd);
696*e86372a0SGvozden Neskovic }
697*e86372a0SGvozden Neskovic
698*e86372a0SGvozden Neskovic /* add Q to the syndrome */
699*e86372a0SGvozden Neskovic raidz_add(xabd, rm->rm_col[CODE_Q].rc_abd, xsize);
700*e86372a0SGvozden Neskovic
701*e86372a0SGvozden Neskovic /* transform the syndrome */
702*e86372a0SGvozden Neskovic abd_iterate_func(xabd, 0, xsize, raidz_mul_abd_cb, (void*) coeff);
703*e86372a0SGvozden Neskovic
704*e86372a0SGvozden Neskovic raidz_math_end();
705*e86372a0SGvozden Neskovic
706*e86372a0SGvozden Neskovic return (1 << CODE_Q);
707*e86372a0SGvozden Neskovic }
708*e86372a0SGvozden Neskovic
709*e86372a0SGvozden Neskovic
710*e86372a0SGvozden Neskovic /*
711*e86372a0SGvozden Neskovic * Generate R syndrome (Rsyn)
712*e86372a0SGvozden Neskovic *
713*e86372a0SGvozden Neskovic * @xc array of pointers to syndrome columns
714*e86372a0SGvozden Neskovic * @dc data column (NULL if missing)
715*e86372a0SGvozden Neskovic * @tsize size of syndrome columns
716*e86372a0SGvozden Neskovic * @dsize size of data column (0 if missing)
717*e86372a0SGvozden Neskovic */
718*e86372a0SGvozden Neskovic static void
raidz_syn_r_abd(void ** xc,const void * dc,const size_t tsize,const size_t dsize)719*e86372a0SGvozden Neskovic raidz_syn_r_abd(void **xc, const void *dc, const size_t tsize,
720*e86372a0SGvozden Neskovic const size_t dsize)
721*e86372a0SGvozden Neskovic {
722*e86372a0SGvozden Neskovic v_t *x = (v_t *)xc[TARGET_X];
723*e86372a0SGvozden Neskovic const v_t *d = (const v_t *)dc;
724*e86372a0SGvozden Neskovic const v_t * const dend = d + (dsize / sizeof (v_t));
725*e86372a0SGvozden Neskovic const v_t * const xend = x + (tsize / sizeof (v_t));
726*e86372a0SGvozden Neskovic
727*e86372a0SGvozden Neskovic SYN_R_DEFINE();
728*e86372a0SGvozden Neskovic
729*e86372a0SGvozden Neskovic MUL2_SETUP();
730*e86372a0SGvozden Neskovic
731*e86372a0SGvozden Neskovic for (; d < dend; d += SYN_STRIDE, x += SYN_STRIDE) {
732*e86372a0SGvozden Neskovic LOAD(d, SYN_R_D);
733*e86372a0SGvozden Neskovic R_D_SYNDROME(SYN_R_D, SYN_R_X, x);
734*e86372a0SGvozden Neskovic }
735*e86372a0SGvozden Neskovic for (; x < xend; x += SYN_STRIDE) {
736*e86372a0SGvozden Neskovic R_SYNDROME(SYN_R_X, x);
737*e86372a0SGvozden Neskovic }
738*e86372a0SGvozden Neskovic }
739*e86372a0SGvozden Neskovic
740*e86372a0SGvozden Neskovic
741*e86372a0SGvozden Neskovic /*
742*e86372a0SGvozden Neskovic * Reconstruct single data column using R parity
743*e86372a0SGvozden Neskovic *
744*e86372a0SGvozden Neskovic * @syn_method raidz_add_abd()
745*e86372a0SGvozden Neskovic * @rec_method raidz_mul_abd_cb()
746*e86372a0SGvozden Neskovic *
747*e86372a0SGvozden Neskovic * @rm RAIDZ map
748*e86372a0SGvozden Neskovic * @tgtidx array of missing data indexes
749*e86372a0SGvozden Neskovic */
750*e86372a0SGvozden Neskovic static raidz_inline int
raidz_reconstruct_r_impl(raidz_map_t * rm,const int * tgtidx)751*e86372a0SGvozden Neskovic raidz_reconstruct_r_impl(raidz_map_t *rm, const int *tgtidx)
752*e86372a0SGvozden Neskovic {
753*e86372a0SGvozden Neskovic size_t c;
754*e86372a0SGvozden Neskovic size_t dsize;
755*e86372a0SGvozden Neskovic abd_t *dabd;
756*e86372a0SGvozden Neskovic const size_t firstdc = raidz_parity(rm);
757*e86372a0SGvozden Neskovic const size_t ncols = raidz_ncols(rm);
758*e86372a0SGvozden Neskovic const size_t x = tgtidx[TARGET_X];
759*e86372a0SGvozden Neskovic const size_t xsize = rm->rm_col[x].rc_size;
760*e86372a0SGvozden Neskovic abd_t *xabd = rm->rm_col[x].rc_abd;
761*e86372a0SGvozden Neskovic abd_t *tabds[] = { xabd };
762*e86372a0SGvozden Neskovic
763*e86372a0SGvozden Neskovic unsigned coeff[MUL_CNT];
764*e86372a0SGvozden Neskovic raidz_rec_r_coeff(rm, tgtidx, coeff);
765*e86372a0SGvozden Neskovic
766*e86372a0SGvozden Neskovic raidz_math_begin();
767*e86372a0SGvozden Neskovic
768*e86372a0SGvozden Neskovic /* Start with first data column if present */
769*e86372a0SGvozden Neskovic if (firstdc != x) {
770*e86372a0SGvozden Neskovic raidz_copy(xabd, rm->rm_col[firstdc].rc_abd, xsize);
771*e86372a0SGvozden Neskovic } else {
772*e86372a0SGvozden Neskovic raidz_zero(xabd, xsize);
773*e86372a0SGvozden Neskovic }
774*e86372a0SGvozden Neskovic
775*e86372a0SGvozden Neskovic
776*e86372a0SGvozden Neskovic /* generate q_syndrome */
777*e86372a0SGvozden Neskovic for (c = firstdc+1; c < ncols; c++) {
778*e86372a0SGvozden Neskovic if (c == x) {
779*e86372a0SGvozden Neskovic dabd = NULL;
780*e86372a0SGvozden Neskovic dsize = 0;
781*e86372a0SGvozden Neskovic } else {
782*e86372a0SGvozden Neskovic dabd = rm->rm_col[c].rc_abd;
783*e86372a0SGvozden Neskovic dsize = rm->rm_col[c].rc_size;
784*e86372a0SGvozden Neskovic }
785*e86372a0SGvozden Neskovic
786*e86372a0SGvozden Neskovic abd_raidz_gen_iterate(tabds, dabd, xsize, dsize, 1,
787*e86372a0SGvozden Neskovic raidz_syn_r_abd);
788*e86372a0SGvozden Neskovic }
789*e86372a0SGvozden Neskovic
790*e86372a0SGvozden Neskovic /* add R to the syndrome */
791*e86372a0SGvozden Neskovic raidz_add(xabd, rm->rm_col[CODE_R].rc_abd, xsize);
792*e86372a0SGvozden Neskovic
793*e86372a0SGvozden Neskovic /* transform the syndrome */
794*e86372a0SGvozden Neskovic abd_iterate_func(xabd, 0, xsize, raidz_mul_abd_cb, (void *)coeff);
795*e86372a0SGvozden Neskovic
796*e86372a0SGvozden Neskovic raidz_math_end();
797*e86372a0SGvozden Neskovic
798*e86372a0SGvozden Neskovic return (1 << CODE_R);
799*e86372a0SGvozden Neskovic }
800*e86372a0SGvozden Neskovic
801*e86372a0SGvozden Neskovic
802*e86372a0SGvozden Neskovic /*
803*e86372a0SGvozden Neskovic * Generate P and Q syndromes
804*e86372a0SGvozden Neskovic *
805*e86372a0SGvozden Neskovic * @xc array of pointers to syndrome columns
806*e86372a0SGvozden Neskovic * @dc data column (NULL if missing)
807*e86372a0SGvozden Neskovic * @tsize size of syndrome columns
808*e86372a0SGvozden Neskovic * @dsize size of data column (0 if missing)
809*e86372a0SGvozden Neskovic */
810*e86372a0SGvozden Neskovic static void
raidz_syn_pq_abd(void ** tc,const void * dc,const size_t tsize,const size_t dsize)811*e86372a0SGvozden Neskovic raidz_syn_pq_abd(void **tc, const void *dc, const size_t tsize,
812*e86372a0SGvozden Neskovic const size_t dsize)
813*e86372a0SGvozden Neskovic {
814*e86372a0SGvozden Neskovic v_t *x = (v_t *)tc[TARGET_X];
815*e86372a0SGvozden Neskovic v_t *y = (v_t *)tc[TARGET_Y];
816*e86372a0SGvozden Neskovic const v_t *d = (const v_t *)dc;
817*e86372a0SGvozden Neskovic const v_t * const dend = d + (dsize / sizeof (v_t));
818*e86372a0SGvozden Neskovic const v_t * const yend = y + (tsize / sizeof (v_t));
819*e86372a0SGvozden Neskovic
820*e86372a0SGvozden Neskovic SYN_PQ_DEFINE();
821*e86372a0SGvozden Neskovic
822*e86372a0SGvozden Neskovic MUL2_SETUP();
823*e86372a0SGvozden Neskovic
824*e86372a0SGvozden Neskovic for (; d < dend; d += SYN_STRIDE, x += SYN_STRIDE, y += SYN_STRIDE) {
825*e86372a0SGvozden Neskovic LOAD(d, SYN_PQ_D);
826*e86372a0SGvozden Neskovic P_D_SYNDROME(SYN_PQ_D, SYN_PQ_X, x);
827*e86372a0SGvozden Neskovic Q_D_SYNDROME(SYN_PQ_D, SYN_PQ_X, y);
828*e86372a0SGvozden Neskovic }
829*e86372a0SGvozden Neskovic for (; y < yend; y += SYN_STRIDE) {
830*e86372a0SGvozden Neskovic Q_SYNDROME(SYN_PQ_X, y);
831*e86372a0SGvozden Neskovic }
832*e86372a0SGvozden Neskovic }
833*e86372a0SGvozden Neskovic
834*e86372a0SGvozden Neskovic /*
835*e86372a0SGvozden Neskovic * Reconstruct data using PQ parity and PQ syndromes
836*e86372a0SGvozden Neskovic *
837*e86372a0SGvozden Neskovic * @tc syndrome/result columns
838*e86372a0SGvozden Neskovic * @tsize size of syndrome/result columns
839*e86372a0SGvozden Neskovic * @c parity columns
840*e86372a0SGvozden Neskovic * @mul array of multiplication constants
841*e86372a0SGvozden Neskovic */
842*e86372a0SGvozden Neskovic static void
raidz_rec_pq_abd(void ** tc,const size_t tsize,void ** c,const unsigned * mul)843*e86372a0SGvozden Neskovic raidz_rec_pq_abd(void **tc, const size_t tsize, void **c,
844*e86372a0SGvozden Neskovic const unsigned *mul)
845*e86372a0SGvozden Neskovic {
846*e86372a0SGvozden Neskovic v_t *x = (v_t *)tc[TARGET_X];
847*e86372a0SGvozden Neskovic v_t *y = (v_t *)tc[TARGET_Y];
848*e86372a0SGvozden Neskovic const v_t * const xend = x + (tsize / sizeof (v_t));
849*e86372a0SGvozden Neskovic const v_t *p = (v_t *)c[CODE_P];
850*e86372a0SGvozden Neskovic const v_t *q = (v_t *)c[CODE_Q];
851*e86372a0SGvozden Neskovic
852*e86372a0SGvozden Neskovic REC_PQ_DEFINE();
853*e86372a0SGvozden Neskovic
854*e86372a0SGvozden Neskovic for (; x < xend; x += REC_PQ_STRIDE, y += REC_PQ_STRIDE,
855*e86372a0SGvozden Neskovic p += REC_PQ_STRIDE, q += REC_PQ_STRIDE) {
856*e86372a0SGvozden Neskovic LOAD(x, REC_PQ_X);
857*e86372a0SGvozden Neskovic LOAD(y, REC_PQ_Y);
858*e86372a0SGvozden Neskovic
859*e86372a0SGvozden Neskovic XOR_ACC(p, REC_PQ_X);
860*e86372a0SGvozden Neskovic XOR_ACC(q, REC_PQ_Y);
861*e86372a0SGvozden Neskovic
862*e86372a0SGvozden Neskovic /* Save Pxy */
863*e86372a0SGvozden Neskovic COPY(REC_PQ_X, REC_PQ_T);
864*e86372a0SGvozden Neskovic
865*e86372a0SGvozden Neskovic /* Calc X */
866*e86372a0SGvozden Neskovic MUL(mul[MUL_PQ_X], REC_PQ_X);
867*e86372a0SGvozden Neskovic MUL(mul[MUL_PQ_Y], REC_PQ_Y);
868*e86372a0SGvozden Neskovic XOR(REC_PQ_Y, REC_PQ_X);
869*e86372a0SGvozden Neskovic STORE(x, REC_PQ_X);
870*e86372a0SGvozden Neskovic
871*e86372a0SGvozden Neskovic /* Calc Y */
872*e86372a0SGvozden Neskovic XOR(REC_PQ_T, REC_PQ_X);
873*e86372a0SGvozden Neskovic STORE(y, REC_PQ_X);
874*e86372a0SGvozden Neskovic }
875*e86372a0SGvozden Neskovic }
876*e86372a0SGvozden Neskovic
877*e86372a0SGvozden Neskovic
878*e86372a0SGvozden Neskovic /*
879*e86372a0SGvozden Neskovic * Reconstruct two data columns using PQ parity
880*e86372a0SGvozden Neskovic *
881*e86372a0SGvozden Neskovic * @syn_method raidz_syn_pq_abd()
882*e86372a0SGvozden Neskovic * @rec_method raidz_rec_pq_abd()
883*e86372a0SGvozden Neskovic *
884*e86372a0SGvozden Neskovic * @rm RAIDZ map
885*e86372a0SGvozden Neskovic * @tgtidx array of missing data indexes
886*e86372a0SGvozden Neskovic */
887*e86372a0SGvozden Neskovic static raidz_inline int
raidz_reconstruct_pq_impl(raidz_map_t * rm,const int * tgtidx)888*e86372a0SGvozden Neskovic raidz_reconstruct_pq_impl(raidz_map_t *rm, const int *tgtidx)
889*e86372a0SGvozden Neskovic {
890*e86372a0SGvozden Neskovic size_t c;
891*e86372a0SGvozden Neskovic size_t dsize;
892*e86372a0SGvozden Neskovic abd_t *dabd;
893*e86372a0SGvozden Neskovic const size_t firstdc = raidz_parity(rm);
894*e86372a0SGvozden Neskovic const size_t ncols = raidz_ncols(rm);
895*e86372a0SGvozden Neskovic const size_t x = tgtidx[TARGET_X];
896*e86372a0SGvozden Neskovic const size_t y = tgtidx[TARGET_Y];
897*e86372a0SGvozden Neskovic const size_t xsize = rm->rm_col[x].rc_size;
898*e86372a0SGvozden Neskovic const size_t ysize = rm->rm_col[y].rc_size;
899*e86372a0SGvozden Neskovic abd_t *xabd = rm->rm_col[x].rc_abd;
900*e86372a0SGvozden Neskovic abd_t *yabd = rm->rm_col[y].rc_abd;
901*e86372a0SGvozden Neskovic abd_t *tabds[2] = { xabd, yabd };
902*e86372a0SGvozden Neskovic abd_t *cabds[] = {
903*e86372a0SGvozden Neskovic rm->rm_col[CODE_P].rc_abd,
904*e86372a0SGvozden Neskovic rm->rm_col[CODE_Q].rc_abd
905*e86372a0SGvozden Neskovic };
906*e86372a0SGvozden Neskovic
907*e86372a0SGvozden Neskovic unsigned coeff[MUL_CNT];
908*e86372a0SGvozden Neskovic raidz_rec_pq_coeff(rm, tgtidx, coeff);
909*e86372a0SGvozden Neskovic
910*e86372a0SGvozden Neskovic /*
911*e86372a0SGvozden Neskovic * Check if some of targets is shorter then others
912*e86372a0SGvozden Neskovic * In this case, shorter target needs to be replaced with
913*e86372a0SGvozden Neskovic * new buffer so that syndrome can be calculated.
914*e86372a0SGvozden Neskovic */
915*e86372a0SGvozden Neskovic if (ysize < xsize) {
916*e86372a0SGvozden Neskovic yabd = abd_alloc(xsize, B_FALSE);
917*e86372a0SGvozden Neskovic tabds[1] = yabd;
918*e86372a0SGvozden Neskovic }
919*e86372a0SGvozden Neskovic
920*e86372a0SGvozden Neskovic raidz_math_begin();
921*e86372a0SGvozden Neskovic
922*e86372a0SGvozden Neskovic /* Start with first data column if present */
923*e86372a0SGvozden Neskovic if (firstdc != x) {
924*e86372a0SGvozden Neskovic raidz_copy(xabd, rm->rm_col[firstdc].rc_abd, xsize);
925*e86372a0SGvozden Neskovic raidz_copy(yabd, rm->rm_col[firstdc].rc_abd, xsize);
926*e86372a0SGvozden Neskovic } else {
927*e86372a0SGvozden Neskovic raidz_zero(xabd, xsize);
928*e86372a0SGvozden Neskovic raidz_zero(yabd, xsize);
929*e86372a0SGvozden Neskovic }
930*e86372a0SGvozden Neskovic
931*e86372a0SGvozden Neskovic /* generate q_syndrome */
932*e86372a0SGvozden Neskovic for (c = firstdc+1; c < ncols; c++) {
933*e86372a0SGvozden Neskovic if (c == x || c == y) {
934*e86372a0SGvozden Neskovic dabd = NULL;
935*e86372a0SGvozden Neskovic dsize = 0;
936*e86372a0SGvozden Neskovic } else {
937*e86372a0SGvozden Neskovic dabd = rm->rm_col[c].rc_abd;
938*e86372a0SGvozden Neskovic dsize = rm->rm_col[c].rc_size;
939*e86372a0SGvozden Neskovic }
940*e86372a0SGvozden Neskovic
941*e86372a0SGvozden Neskovic abd_raidz_gen_iterate(tabds, dabd, xsize, dsize, 2,
942*e86372a0SGvozden Neskovic raidz_syn_pq_abd);
943*e86372a0SGvozden Neskovic }
944*e86372a0SGvozden Neskovic
945*e86372a0SGvozden Neskovic abd_raidz_rec_iterate(cabds, tabds, xsize, 2, raidz_rec_pq_abd, coeff);
946*e86372a0SGvozden Neskovic
947*e86372a0SGvozden Neskovic /* Copy shorter targets back to the original abd buffer */
948*e86372a0SGvozden Neskovic if (ysize < xsize)
949*e86372a0SGvozden Neskovic raidz_copy(rm->rm_col[y].rc_abd, yabd, ysize);
950*e86372a0SGvozden Neskovic
951*e86372a0SGvozden Neskovic raidz_math_end();
952*e86372a0SGvozden Neskovic
953*e86372a0SGvozden Neskovic if (ysize < xsize)
954*e86372a0SGvozden Neskovic abd_free(yabd);
955*e86372a0SGvozden Neskovic
956*e86372a0SGvozden Neskovic return ((1 << CODE_P) | (1 << CODE_Q));
957*e86372a0SGvozden Neskovic }
958*e86372a0SGvozden Neskovic
959*e86372a0SGvozden Neskovic
960*e86372a0SGvozden Neskovic /*
961*e86372a0SGvozden Neskovic * Generate P and R syndromes
962*e86372a0SGvozden Neskovic *
963*e86372a0SGvozden Neskovic * @xc array of pointers to syndrome columns
964*e86372a0SGvozden Neskovic * @dc data column (NULL if missing)
965*e86372a0SGvozden Neskovic * @tsize size of syndrome columns
966*e86372a0SGvozden Neskovic * @dsize size of data column (0 if missing)
967*e86372a0SGvozden Neskovic */
968*e86372a0SGvozden Neskovic static void
raidz_syn_pr_abd(void ** c,const void * dc,const size_t tsize,const size_t dsize)969*e86372a0SGvozden Neskovic raidz_syn_pr_abd(void **c, const void *dc, const size_t tsize,
970*e86372a0SGvozden Neskovic const size_t dsize)
971*e86372a0SGvozden Neskovic {
972*e86372a0SGvozden Neskovic v_t *x = (v_t *)c[TARGET_X];
973*e86372a0SGvozden Neskovic v_t *y = (v_t *)c[TARGET_Y];
974*e86372a0SGvozden Neskovic const v_t *d = (const v_t *)dc;
975*e86372a0SGvozden Neskovic const v_t * const dend = d + (dsize / sizeof (v_t));
976*e86372a0SGvozden Neskovic const v_t * const yend = y + (tsize / sizeof (v_t));
977*e86372a0SGvozden Neskovic
978*e86372a0SGvozden Neskovic SYN_PR_DEFINE();
979*e86372a0SGvozden Neskovic
980*e86372a0SGvozden Neskovic MUL2_SETUP();
981*e86372a0SGvozden Neskovic
982*e86372a0SGvozden Neskovic for (; d < dend; d += SYN_STRIDE, x += SYN_STRIDE, y += SYN_STRIDE) {
983*e86372a0SGvozden Neskovic LOAD(d, SYN_PR_D);
984*e86372a0SGvozden Neskovic P_D_SYNDROME(SYN_PR_D, SYN_PR_X, x);
985*e86372a0SGvozden Neskovic R_D_SYNDROME(SYN_PR_D, SYN_PR_X, y);
986*e86372a0SGvozden Neskovic }
987*e86372a0SGvozden Neskovic for (; y < yend; y += SYN_STRIDE) {
988*e86372a0SGvozden Neskovic R_SYNDROME(SYN_PR_X, y);
989*e86372a0SGvozden Neskovic }
990*e86372a0SGvozden Neskovic }
991*e86372a0SGvozden Neskovic
992*e86372a0SGvozden Neskovic /*
993*e86372a0SGvozden Neskovic * Reconstruct data using PR parity and PR syndromes
994*e86372a0SGvozden Neskovic *
995*e86372a0SGvozden Neskovic * @tc syndrome/result columns
996*e86372a0SGvozden Neskovic * @tsize size of syndrome/result columns
997*e86372a0SGvozden Neskovic * @c parity columns
998*e86372a0SGvozden Neskovic * @mul array of multiplication constants
999*e86372a0SGvozden Neskovic */
1000*e86372a0SGvozden Neskovic static void
raidz_rec_pr_abd(void ** t,const size_t tsize,void ** c,const unsigned * mul)1001*e86372a0SGvozden Neskovic raidz_rec_pr_abd(void **t, const size_t tsize, void **c,
1002*e86372a0SGvozden Neskovic const unsigned *mul)
1003*e86372a0SGvozden Neskovic {
1004*e86372a0SGvozden Neskovic v_t *x = (v_t *)t[TARGET_X];
1005*e86372a0SGvozden Neskovic v_t *y = (v_t *)t[TARGET_Y];
1006*e86372a0SGvozden Neskovic const v_t * const xend = x + (tsize / sizeof (v_t));
1007*e86372a0SGvozden Neskovic const v_t *p = (v_t *)c[CODE_P];
1008*e86372a0SGvozden Neskovic const v_t *q = (v_t *)c[CODE_Q];
1009*e86372a0SGvozden Neskovic
1010*e86372a0SGvozden Neskovic REC_PR_DEFINE();
1011*e86372a0SGvozden Neskovic
1012*e86372a0SGvozden Neskovic for (; x < xend; x += REC_PR_STRIDE, y += REC_PR_STRIDE,
1013*e86372a0SGvozden Neskovic p += REC_PR_STRIDE, q += REC_PR_STRIDE) {
1014*e86372a0SGvozden Neskovic LOAD(x, REC_PR_X);
1015*e86372a0SGvozden Neskovic LOAD(y, REC_PR_Y);
1016*e86372a0SGvozden Neskovic XOR_ACC(p, REC_PR_X);
1017*e86372a0SGvozden Neskovic XOR_ACC(q, REC_PR_Y);
1018*e86372a0SGvozden Neskovic
1019*e86372a0SGvozden Neskovic /* Save Pxy */
1020*e86372a0SGvozden Neskovic COPY(REC_PR_X, REC_PR_T);
1021*e86372a0SGvozden Neskovic
1022*e86372a0SGvozden Neskovic /* Calc X */
1023*e86372a0SGvozden Neskovic MUL(mul[MUL_PR_X], REC_PR_X);
1024*e86372a0SGvozden Neskovic MUL(mul[MUL_PR_Y], REC_PR_Y);
1025*e86372a0SGvozden Neskovic XOR(REC_PR_Y, REC_PR_X);
1026*e86372a0SGvozden Neskovic STORE(x, REC_PR_X);
1027*e86372a0SGvozden Neskovic
1028*e86372a0SGvozden Neskovic /* Calc Y */
1029*e86372a0SGvozden Neskovic XOR(REC_PR_T, REC_PR_X);
1030*e86372a0SGvozden Neskovic STORE(y, REC_PR_X);
1031*e86372a0SGvozden Neskovic }
1032*e86372a0SGvozden Neskovic }
1033*e86372a0SGvozden Neskovic
1034*e86372a0SGvozden Neskovic
1035*e86372a0SGvozden Neskovic /*
1036*e86372a0SGvozden Neskovic * Reconstruct two data columns using PR parity
1037*e86372a0SGvozden Neskovic *
1038*e86372a0SGvozden Neskovic * @syn_method raidz_syn_pr_abd()
1039*e86372a0SGvozden Neskovic * @rec_method raidz_rec_pr_abd()
1040*e86372a0SGvozden Neskovic *
1041*e86372a0SGvozden Neskovic * @rm RAIDZ map
1042*e86372a0SGvozden Neskovic * @tgtidx array of missing data indexes
1043*e86372a0SGvozden Neskovic */
1044*e86372a0SGvozden Neskovic static raidz_inline int
raidz_reconstruct_pr_impl(raidz_map_t * rm,const int * tgtidx)1045*e86372a0SGvozden Neskovic raidz_reconstruct_pr_impl(raidz_map_t *rm, const int *tgtidx)
1046*e86372a0SGvozden Neskovic {
1047*e86372a0SGvozden Neskovic size_t c;
1048*e86372a0SGvozden Neskovic size_t dsize;
1049*e86372a0SGvozden Neskovic abd_t *dabd;
1050*e86372a0SGvozden Neskovic const size_t firstdc = raidz_parity(rm);
1051*e86372a0SGvozden Neskovic const size_t ncols = raidz_ncols(rm);
1052*e86372a0SGvozden Neskovic const size_t x = tgtidx[0];
1053*e86372a0SGvozden Neskovic const size_t y = tgtidx[1];
1054*e86372a0SGvozden Neskovic const size_t xsize = rm->rm_col[x].rc_size;
1055*e86372a0SGvozden Neskovic const size_t ysize = rm->rm_col[y].rc_size;
1056*e86372a0SGvozden Neskovic abd_t *xabd = rm->rm_col[x].rc_abd;
1057*e86372a0SGvozden Neskovic abd_t *yabd = rm->rm_col[y].rc_abd;
1058*e86372a0SGvozden Neskovic abd_t *tabds[2] = { xabd, yabd };
1059*e86372a0SGvozden Neskovic abd_t *cabds[] = {
1060*e86372a0SGvozden Neskovic rm->rm_col[CODE_P].rc_abd,
1061*e86372a0SGvozden Neskovic rm->rm_col[CODE_R].rc_abd
1062*e86372a0SGvozden Neskovic };
1063*e86372a0SGvozden Neskovic unsigned coeff[MUL_CNT];
1064*e86372a0SGvozden Neskovic raidz_rec_pr_coeff(rm, tgtidx, coeff);
1065*e86372a0SGvozden Neskovic
1066*e86372a0SGvozden Neskovic /*
1067*e86372a0SGvozden Neskovic * Check if some of targets are shorter then others.
1068*e86372a0SGvozden Neskovic * They need to be replaced with a new buffer so that syndrome can
1069*e86372a0SGvozden Neskovic * be calculated on full length.
1070*e86372a0SGvozden Neskovic */
1071*e86372a0SGvozden Neskovic if (ysize < xsize) {
1072*e86372a0SGvozden Neskovic yabd = abd_alloc(xsize, B_FALSE);
1073*e86372a0SGvozden Neskovic tabds[1] = yabd;
1074*e86372a0SGvozden Neskovic }
1075*e86372a0SGvozden Neskovic
1076*e86372a0SGvozden Neskovic raidz_math_begin();
1077*e86372a0SGvozden Neskovic
1078*e86372a0SGvozden Neskovic /* Start with first data column if present */
1079*e86372a0SGvozden Neskovic if (firstdc != x) {
1080*e86372a0SGvozden Neskovic raidz_copy(xabd, rm->rm_col[firstdc].rc_abd, xsize);
1081*e86372a0SGvozden Neskovic raidz_copy(yabd, rm->rm_col[firstdc].rc_abd, xsize);
1082*e86372a0SGvozden Neskovic } else {
1083*e86372a0SGvozden Neskovic raidz_zero(xabd, xsize);
1084*e86372a0SGvozden Neskovic raidz_zero(yabd, xsize);
1085*e86372a0SGvozden Neskovic }
1086*e86372a0SGvozden Neskovic
1087*e86372a0SGvozden Neskovic /* generate q_syndrome */
1088*e86372a0SGvozden Neskovic for (c = firstdc+1; c < ncols; c++) {
1089*e86372a0SGvozden Neskovic if (c == x || c == y) {
1090*e86372a0SGvozden Neskovic dabd = NULL;
1091*e86372a0SGvozden Neskovic dsize = 0;
1092*e86372a0SGvozden Neskovic } else {
1093*e86372a0SGvozden Neskovic dabd = rm->rm_col[c].rc_abd;
1094*e86372a0SGvozden Neskovic dsize = rm->rm_col[c].rc_size;
1095*e86372a0SGvozden Neskovic }
1096*e86372a0SGvozden Neskovic
1097*e86372a0SGvozden Neskovic abd_raidz_gen_iterate(tabds, dabd, xsize, dsize, 2,
1098*e86372a0SGvozden Neskovic raidz_syn_pr_abd);
1099*e86372a0SGvozden Neskovic }
1100*e86372a0SGvozden Neskovic
1101*e86372a0SGvozden Neskovic abd_raidz_rec_iterate(cabds, tabds, xsize, 2, raidz_rec_pr_abd, coeff);
1102*e86372a0SGvozden Neskovic
1103*e86372a0SGvozden Neskovic /*
1104*e86372a0SGvozden Neskovic * Copy shorter targets back to the original abd buffer
1105*e86372a0SGvozden Neskovic */
1106*e86372a0SGvozden Neskovic if (ysize < xsize)
1107*e86372a0SGvozden Neskovic raidz_copy(rm->rm_col[y].rc_abd, yabd, ysize);
1108*e86372a0SGvozden Neskovic
1109*e86372a0SGvozden Neskovic raidz_math_end();
1110*e86372a0SGvozden Neskovic
1111*e86372a0SGvozden Neskovic if (ysize < xsize)
1112*e86372a0SGvozden Neskovic abd_free(yabd);
1113*e86372a0SGvozden Neskovic
1114*e86372a0SGvozden Neskovic return ((1 << CODE_P) | (1 << CODE_Q));
1115*e86372a0SGvozden Neskovic }
1116*e86372a0SGvozden Neskovic
1117*e86372a0SGvozden Neskovic
1118*e86372a0SGvozden Neskovic /*
1119*e86372a0SGvozden Neskovic * Generate Q and R syndromes
1120*e86372a0SGvozden Neskovic *
1121*e86372a0SGvozden Neskovic * @xc array of pointers to syndrome columns
1122*e86372a0SGvozden Neskovic * @dc data column (NULL if missing)
1123*e86372a0SGvozden Neskovic * @tsize size of syndrome columns
1124*e86372a0SGvozden Neskovic * @dsize size of data column (0 if missing)
1125*e86372a0SGvozden Neskovic */
1126*e86372a0SGvozden Neskovic static void
raidz_syn_qr_abd(void ** c,const void * dc,const size_t tsize,const size_t dsize)1127*e86372a0SGvozden Neskovic raidz_syn_qr_abd(void **c, const void *dc, const size_t tsize,
1128*e86372a0SGvozden Neskovic const size_t dsize)
1129*e86372a0SGvozden Neskovic {
1130*e86372a0SGvozden Neskovic v_t *x = (v_t *)c[TARGET_X];
1131*e86372a0SGvozden Neskovic v_t *y = (v_t *)c[TARGET_Y];
1132*e86372a0SGvozden Neskovic const v_t * const xend = x + (tsize / sizeof (v_t));
1133*e86372a0SGvozden Neskovic const v_t *d = (const v_t *)dc;
1134*e86372a0SGvozden Neskovic const v_t * const dend = d + (dsize / sizeof (v_t));
1135*e86372a0SGvozden Neskovic
1136*e86372a0SGvozden Neskovic SYN_QR_DEFINE();
1137*e86372a0SGvozden Neskovic
1138*e86372a0SGvozden Neskovic MUL2_SETUP();
1139*e86372a0SGvozden Neskovic
1140*e86372a0SGvozden Neskovic for (; d < dend; d += SYN_STRIDE, x += SYN_STRIDE, y += SYN_STRIDE) {
1141*e86372a0SGvozden Neskovic LOAD(d, SYN_PQ_D);
1142*e86372a0SGvozden Neskovic Q_D_SYNDROME(SYN_QR_D, SYN_QR_X, x);
1143*e86372a0SGvozden Neskovic R_D_SYNDROME(SYN_QR_D, SYN_QR_X, y);
1144*e86372a0SGvozden Neskovic }
1145*e86372a0SGvozden Neskovic for (; x < xend; x += SYN_STRIDE, y += SYN_STRIDE) {
1146*e86372a0SGvozden Neskovic Q_SYNDROME(SYN_QR_X, x);
1147*e86372a0SGvozden Neskovic R_SYNDROME(SYN_QR_X, y);
1148*e86372a0SGvozden Neskovic }
1149*e86372a0SGvozden Neskovic }
1150*e86372a0SGvozden Neskovic
1151*e86372a0SGvozden Neskovic
1152*e86372a0SGvozden Neskovic /*
1153*e86372a0SGvozden Neskovic * Reconstruct data using QR parity and QR syndromes
1154*e86372a0SGvozden Neskovic *
1155*e86372a0SGvozden Neskovic * @tc syndrome/result columns
1156*e86372a0SGvozden Neskovic * @tsize size of syndrome/result columns
1157*e86372a0SGvozden Neskovic * @c parity columns
1158*e86372a0SGvozden Neskovic * @mul array of multiplication constants
1159*e86372a0SGvozden Neskovic */
1160*e86372a0SGvozden Neskovic static void
raidz_rec_qr_abd(void ** t,const size_t tsize,void ** c,const unsigned * mul)1161*e86372a0SGvozden Neskovic raidz_rec_qr_abd(void **t, const size_t tsize, void **c,
1162*e86372a0SGvozden Neskovic const unsigned *mul)
1163*e86372a0SGvozden Neskovic {
1164*e86372a0SGvozden Neskovic v_t *x = (v_t *)t[TARGET_X];
1165*e86372a0SGvozden Neskovic v_t *y = (v_t *)t[TARGET_Y];
1166*e86372a0SGvozden Neskovic const v_t * const xend = x + (tsize / sizeof (v_t));
1167*e86372a0SGvozden Neskovic const v_t *p = (v_t *)c[CODE_P];
1168*e86372a0SGvozden Neskovic const v_t *q = (v_t *)c[CODE_Q];
1169*e86372a0SGvozden Neskovic
1170*e86372a0SGvozden Neskovic REC_QR_DEFINE();
1171*e86372a0SGvozden Neskovic
1172*e86372a0SGvozden Neskovic for (; x < xend; x += REC_QR_STRIDE, y += REC_QR_STRIDE,
1173*e86372a0SGvozden Neskovic p += REC_QR_STRIDE, q += REC_QR_STRIDE) {
1174*e86372a0SGvozden Neskovic LOAD(x, REC_QR_X);
1175*e86372a0SGvozden Neskovic LOAD(y, REC_QR_Y);
1176*e86372a0SGvozden Neskovic
1177*e86372a0SGvozden Neskovic XOR_ACC(p, REC_QR_X);
1178*e86372a0SGvozden Neskovic XOR_ACC(q, REC_QR_Y);
1179*e86372a0SGvozden Neskovic
1180*e86372a0SGvozden Neskovic /* Save Pxy */
1181*e86372a0SGvozden Neskovic COPY(REC_QR_X, REC_QR_T);
1182*e86372a0SGvozden Neskovic
1183*e86372a0SGvozden Neskovic /* Calc X */
1184*e86372a0SGvozden Neskovic MUL(mul[MUL_QR_XQ], REC_QR_X); /* X = Q * xqm */
1185*e86372a0SGvozden Neskovic XOR(REC_QR_Y, REC_QR_X); /* X = R ^ X */
1186*e86372a0SGvozden Neskovic MUL(mul[MUL_QR_X], REC_QR_X); /* X = X * xm */
1187*e86372a0SGvozden Neskovic STORE(x, REC_QR_X);
1188*e86372a0SGvozden Neskovic
1189*e86372a0SGvozden Neskovic /* Calc Y */
1190*e86372a0SGvozden Neskovic MUL(mul[MUL_QR_YQ], REC_QR_T); /* X = Q * xqm */
1191*e86372a0SGvozden Neskovic XOR(REC_QR_Y, REC_QR_T); /* X = R ^ X */
1192*e86372a0SGvozden Neskovic MUL(mul[MUL_QR_Y], REC_QR_T); /* X = X * xm */
1193*e86372a0SGvozden Neskovic STORE(y, REC_QR_T);
1194*e86372a0SGvozden Neskovic }
1195*e86372a0SGvozden Neskovic }
1196*e86372a0SGvozden Neskovic
1197*e86372a0SGvozden Neskovic
1198*e86372a0SGvozden Neskovic /*
1199*e86372a0SGvozden Neskovic * Reconstruct two data columns using QR parity
1200*e86372a0SGvozden Neskovic *
1201*e86372a0SGvozden Neskovic * @syn_method raidz_syn_qr_abd()
1202*e86372a0SGvozden Neskovic * @rec_method raidz_rec_qr_abd()
1203*e86372a0SGvozden Neskovic *
1204*e86372a0SGvozden Neskovic * @rm RAIDZ map
1205*e86372a0SGvozden Neskovic * @tgtidx array of missing data indexes
1206*e86372a0SGvozden Neskovic */
1207*e86372a0SGvozden Neskovic static raidz_inline int
raidz_reconstruct_qr_impl(raidz_map_t * rm,const int * tgtidx)1208*e86372a0SGvozden Neskovic raidz_reconstruct_qr_impl(raidz_map_t *rm, const int *tgtidx)
1209*e86372a0SGvozden Neskovic {
1210*e86372a0SGvozden Neskovic size_t c;
1211*e86372a0SGvozden Neskovic size_t dsize;
1212*e86372a0SGvozden Neskovic abd_t *dabd;
1213*e86372a0SGvozden Neskovic const size_t firstdc = raidz_parity(rm);
1214*e86372a0SGvozden Neskovic const size_t ncols = raidz_ncols(rm);
1215*e86372a0SGvozden Neskovic const size_t x = tgtidx[TARGET_X];
1216*e86372a0SGvozden Neskovic const size_t y = tgtidx[TARGET_Y];
1217*e86372a0SGvozden Neskovic const size_t xsize = rm->rm_col[x].rc_size;
1218*e86372a0SGvozden Neskovic const size_t ysize = rm->rm_col[y].rc_size;
1219*e86372a0SGvozden Neskovic abd_t *xabd = rm->rm_col[x].rc_abd;
1220*e86372a0SGvozden Neskovic abd_t *yabd = rm->rm_col[y].rc_abd;
1221*e86372a0SGvozden Neskovic abd_t *tabds[2] = { xabd, yabd };
1222*e86372a0SGvozden Neskovic abd_t *cabds[] = {
1223*e86372a0SGvozden Neskovic rm->rm_col[CODE_Q].rc_abd,
1224*e86372a0SGvozden Neskovic rm->rm_col[CODE_R].rc_abd
1225*e86372a0SGvozden Neskovic };
1226*e86372a0SGvozden Neskovic unsigned coeff[MUL_CNT];
1227*e86372a0SGvozden Neskovic raidz_rec_qr_coeff(rm, tgtidx, coeff);
1228*e86372a0SGvozden Neskovic
1229*e86372a0SGvozden Neskovic /*
1230*e86372a0SGvozden Neskovic * Check if some of targets is shorter then others
1231*e86372a0SGvozden Neskovic * In this case, shorter target needs to be replaced with
1232*e86372a0SGvozden Neskovic * new buffer so that syndrome can be calculated.
1233*e86372a0SGvozden Neskovic */
1234*e86372a0SGvozden Neskovic if (ysize < xsize) {
1235*e86372a0SGvozden Neskovic yabd = abd_alloc(xsize, B_FALSE);
1236*e86372a0SGvozden Neskovic tabds[1] = yabd;
1237*e86372a0SGvozden Neskovic }
1238*e86372a0SGvozden Neskovic
1239*e86372a0SGvozden Neskovic raidz_math_begin();
1240*e86372a0SGvozden Neskovic
1241*e86372a0SGvozden Neskovic /* Start with first data column if present */
1242*e86372a0SGvozden Neskovic if (firstdc != x) {
1243*e86372a0SGvozden Neskovic raidz_copy(xabd, rm->rm_col[firstdc].rc_abd, xsize);
1244*e86372a0SGvozden Neskovic raidz_copy(yabd, rm->rm_col[firstdc].rc_abd, xsize);
1245*e86372a0SGvozden Neskovic } else {
1246*e86372a0SGvozden Neskovic raidz_zero(xabd, xsize);
1247*e86372a0SGvozden Neskovic raidz_zero(yabd, xsize);
1248*e86372a0SGvozden Neskovic }
1249*e86372a0SGvozden Neskovic
1250*e86372a0SGvozden Neskovic /* generate q_syndrome */
1251*e86372a0SGvozden Neskovic for (c = firstdc+1; c < ncols; c++) {
1252*e86372a0SGvozden Neskovic if (c == x || c == y) {
1253*e86372a0SGvozden Neskovic dabd = NULL;
1254*e86372a0SGvozden Neskovic dsize = 0;
1255*e86372a0SGvozden Neskovic } else {
1256*e86372a0SGvozden Neskovic dabd = rm->rm_col[c].rc_abd;
1257*e86372a0SGvozden Neskovic dsize = rm->rm_col[c].rc_size;
1258*e86372a0SGvozden Neskovic }
1259*e86372a0SGvozden Neskovic
1260*e86372a0SGvozden Neskovic abd_raidz_gen_iterate(tabds, dabd, xsize, dsize, 2,
1261*e86372a0SGvozden Neskovic raidz_syn_qr_abd);
1262*e86372a0SGvozden Neskovic }
1263*e86372a0SGvozden Neskovic
1264*e86372a0SGvozden Neskovic abd_raidz_rec_iterate(cabds, tabds, xsize, 2, raidz_rec_qr_abd, coeff);
1265*e86372a0SGvozden Neskovic
1266*e86372a0SGvozden Neskovic /*
1267*e86372a0SGvozden Neskovic * Copy shorter targets back to the original abd buffer
1268*e86372a0SGvozden Neskovic */
1269*e86372a0SGvozden Neskovic if (ysize < xsize)
1270*e86372a0SGvozden Neskovic raidz_copy(rm->rm_col[y].rc_abd, yabd, ysize);
1271*e86372a0SGvozden Neskovic
1272*e86372a0SGvozden Neskovic raidz_math_end();
1273*e86372a0SGvozden Neskovic
1274*e86372a0SGvozden Neskovic if (ysize < xsize)
1275*e86372a0SGvozden Neskovic abd_free(yabd);
1276*e86372a0SGvozden Neskovic
1277*e86372a0SGvozden Neskovic
1278*e86372a0SGvozden Neskovic return ((1 << CODE_Q) | (1 << CODE_R));
1279*e86372a0SGvozden Neskovic }
1280*e86372a0SGvozden Neskovic
1281*e86372a0SGvozden Neskovic
1282*e86372a0SGvozden Neskovic /*
1283*e86372a0SGvozden Neskovic * Generate P, Q, and R syndromes
1284*e86372a0SGvozden Neskovic *
1285*e86372a0SGvozden Neskovic * @xc array of pointers to syndrome columns
1286*e86372a0SGvozden Neskovic * @dc data column (NULL if missing)
1287*e86372a0SGvozden Neskovic * @tsize size of syndrome columns
1288*e86372a0SGvozden Neskovic * @dsize size of data column (0 if missing)
1289*e86372a0SGvozden Neskovic */
1290*e86372a0SGvozden Neskovic static void
raidz_syn_pqr_abd(void ** c,const void * dc,const size_t tsize,const size_t dsize)1291*e86372a0SGvozden Neskovic raidz_syn_pqr_abd(void **c, const void *dc, const size_t tsize,
1292*e86372a0SGvozden Neskovic const size_t dsize)
1293*e86372a0SGvozden Neskovic {
1294*e86372a0SGvozden Neskovic v_t *x = (v_t *)c[TARGET_X];
1295*e86372a0SGvozden Neskovic v_t *y = (v_t *)c[TARGET_Y];
1296*e86372a0SGvozden Neskovic v_t *z = (v_t *)c[TARGET_Z];
1297*e86372a0SGvozden Neskovic const v_t * const yend = y + (tsize / sizeof (v_t));
1298*e86372a0SGvozden Neskovic const v_t *d = (const v_t *)dc;
1299*e86372a0SGvozden Neskovic const v_t * const dend = d + (dsize / sizeof (v_t));
1300*e86372a0SGvozden Neskovic
1301*e86372a0SGvozden Neskovic SYN_PQR_DEFINE();
1302*e86372a0SGvozden Neskovic
1303*e86372a0SGvozden Neskovic MUL2_SETUP();
1304*e86372a0SGvozden Neskovic
1305*e86372a0SGvozden Neskovic for (; d < dend; d += SYN_STRIDE, x += SYN_STRIDE, y += SYN_STRIDE,
1306*e86372a0SGvozden Neskovic z += SYN_STRIDE) {
1307*e86372a0SGvozden Neskovic LOAD(d, SYN_PQR_D);
1308*e86372a0SGvozden Neskovic P_D_SYNDROME(SYN_PQR_D, SYN_PQR_X, x)
1309*e86372a0SGvozden Neskovic Q_D_SYNDROME(SYN_PQR_D, SYN_PQR_X, y);
1310*e86372a0SGvozden Neskovic R_D_SYNDROME(SYN_PQR_D, SYN_PQR_X, z);
1311*e86372a0SGvozden Neskovic }
1312*e86372a0SGvozden Neskovic for (; y < yend; y += SYN_STRIDE, z += SYN_STRIDE) {
1313*e86372a0SGvozden Neskovic Q_SYNDROME(SYN_PQR_X, y);
1314*e86372a0SGvozden Neskovic R_SYNDROME(SYN_PQR_X, z);
1315*e86372a0SGvozden Neskovic }
1316*e86372a0SGvozden Neskovic }
1317*e86372a0SGvozden Neskovic
1318*e86372a0SGvozden Neskovic
1319*e86372a0SGvozden Neskovic /*
1320*e86372a0SGvozden Neskovic * Reconstruct data using PRQ parity and PQR syndromes
1321*e86372a0SGvozden Neskovic *
1322*e86372a0SGvozden Neskovic * @tc syndrome/result columns
1323*e86372a0SGvozden Neskovic * @tsize size of syndrome/result columns
1324*e86372a0SGvozden Neskovic * @c parity columns
1325*e86372a0SGvozden Neskovic * @mul array of multiplication constants
1326*e86372a0SGvozden Neskovic */
1327*e86372a0SGvozden Neskovic static void
raidz_rec_pqr_abd(void ** t,const size_t tsize,void ** c,const unsigned * const mul)1328*e86372a0SGvozden Neskovic raidz_rec_pqr_abd(void **t, const size_t tsize, void **c,
1329*e86372a0SGvozden Neskovic const unsigned * const mul)
1330*e86372a0SGvozden Neskovic {
1331*e86372a0SGvozden Neskovic v_t *x = (v_t *)t[TARGET_X];
1332*e86372a0SGvozden Neskovic v_t *y = (v_t *)t[TARGET_Y];
1333*e86372a0SGvozden Neskovic v_t *z = (v_t *)t[TARGET_Z];
1334*e86372a0SGvozden Neskovic const v_t * const xend = x + (tsize / sizeof (v_t));
1335*e86372a0SGvozden Neskovic const v_t *p = (v_t *)c[CODE_P];
1336*e86372a0SGvozden Neskovic const v_t *q = (v_t *)c[CODE_Q];
1337*e86372a0SGvozden Neskovic const v_t *r = (v_t *)c[CODE_R];
1338*e86372a0SGvozden Neskovic
1339*e86372a0SGvozden Neskovic REC_PQR_DEFINE();
1340*e86372a0SGvozden Neskovic
1341*e86372a0SGvozden Neskovic for (; x < xend; x += REC_PQR_STRIDE, y += REC_PQR_STRIDE,
1342*e86372a0SGvozden Neskovic z += REC_PQR_STRIDE, p += REC_PQR_STRIDE, q += REC_PQR_STRIDE,
1343*e86372a0SGvozden Neskovic r += REC_PQR_STRIDE) {
1344*e86372a0SGvozden Neskovic LOAD(x, REC_PQR_X);
1345*e86372a0SGvozden Neskovic LOAD(y, REC_PQR_Y);
1346*e86372a0SGvozden Neskovic LOAD(z, REC_PQR_Z);
1347*e86372a0SGvozden Neskovic
1348*e86372a0SGvozden Neskovic XOR_ACC(p, REC_PQR_X);
1349*e86372a0SGvozden Neskovic XOR_ACC(q, REC_PQR_Y);
1350*e86372a0SGvozden Neskovic XOR_ACC(r, REC_PQR_Z);
1351*e86372a0SGvozden Neskovic
1352*e86372a0SGvozden Neskovic /* Save Pxyz and Qxyz */
1353*e86372a0SGvozden Neskovic COPY(REC_PQR_X, REC_PQR_XS);
1354*e86372a0SGvozden Neskovic COPY(REC_PQR_Y, REC_PQR_YS);
1355*e86372a0SGvozden Neskovic
1356*e86372a0SGvozden Neskovic /* Calc X */
1357*e86372a0SGvozden Neskovic MUL(mul[MUL_PQR_XP], REC_PQR_X); /* Xp = Pxyz * xp */
1358*e86372a0SGvozden Neskovic MUL(mul[MUL_PQR_XQ], REC_PQR_Y); /* Xq = Qxyz * xq */
1359*e86372a0SGvozden Neskovic XOR(REC_PQR_Y, REC_PQR_X);
1360*e86372a0SGvozden Neskovic MUL(mul[MUL_PQR_XR], REC_PQR_Z); /* Xr = Rxyz * xr */
1361*e86372a0SGvozden Neskovic XOR(REC_PQR_Z, REC_PQR_X); /* X = Xp + Xq + Xr */
1362*e86372a0SGvozden Neskovic STORE(x, REC_PQR_X);
1363*e86372a0SGvozden Neskovic
1364*e86372a0SGvozden Neskovic /* Calc Y */
1365*e86372a0SGvozden Neskovic XOR(REC_PQR_X, REC_PQR_XS); /* Pyz = Pxyz + X */
1366*e86372a0SGvozden Neskovic MUL(mul[MUL_PQR_YU], REC_PQR_X); /* Xq = X * upd_q */
1367*e86372a0SGvozden Neskovic XOR(REC_PQR_X, REC_PQR_YS); /* Qyz = Qxyz + Xq */
1368*e86372a0SGvozden Neskovic COPY(REC_PQR_XS, REC_PQR_X); /* restore Pyz */
1369*e86372a0SGvozden Neskovic MUL(mul[MUL_PQR_YP], REC_PQR_X); /* Yp = Pyz * yp */
1370*e86372a0SGvozden Neskovic MUL(mul[MUL_PQR_YQ], REC_PQR_YS); /* Yq = Qyz * yq */
1371*e86372a0SGvozden Neskovic XOR(REC_PQR_X, REC_PQR_YS); /* Y = Yp + Yq */
1372*e86372a0SGvozden Neskovic STORE(y, REC_PQR_YS);
1373*e86372a0SGvozden Neskovic
1374*e86372a0SGvozden Neskovic /* Calc Z */
1375*e86372a0SGvozden Neskovic XOR(REC_PQR_XS, REC_PQR_YS); /* Z = Pz = Pyz + Y */
1376*e86372a0SGvozden Neskovic STORE(z, REC_PQR_YS);
1377*e86372a0SGvozden Neskovic }
1378*e86372a0SGvozden Neskovic }
1379*e86372a0SGvozden Neskovic
1380*e86372a0SGvozden Neskovic
1381*e86372a0SGvozden Neskovic /*
1382*e86372a0SGvozden Neskovic * Reconstruct three data columns using PQR parity
1383*e86372a0SGvozden Neskovic *
1384*e86372a0SGvozden Neskovic * @syn_method raidz_syn_pqr_abd()
1385*e86372a0SGvozden Neskovic * @rec_method raidz_rec_pqr_abd()
1386*e86372a0SGvozden Neskovic *
1387*e86372a0SGvozden Neskovic * @rm RAIDZ map
1388*e86372a0SGvozden Neskovic * @tgtidx array of missing data indexes
1389*e86372a0SGvozden Neskovic */
1390*e86372a0SGvozden Neskovic static raidz_inline int
raidz_reconstruct_pqr_impl(raidz_map_t * rm,const int * tgtidx)1391*e86372a0SGvozden Neskovic raidz_reconstruct_pqr_impl(raidz_map_t *rm, const int *tgtidx)
1392*e86372a0SGvozden Neskovic {
1393*e86372a0SGvozden Neskovic size_t c;
1394*e86372a0SGvozden Neskovic size_t dsize;
1395*e86372a0SGvozden Neskovic abd_t *dabd;
1396*e86372a0SGvozden Neskovic const size_t firstdc = raidz_parity(rm);
1397*e86372a0SGvozden Neskovic const size_t ncols = raidz_ncols(rm);
1398*e86372a0SGvozden Neskovic const size_t x = tgtidx[TARGET_X];
1399*e86372a0SGvozden Neskovic const size_t y = tgtidx[TARGET_Y];
1400*e86372a0SGvozden Neskovic const size_t z = tgtidx[TARGET_Z];
1401*e86372a0SGvozden Neskovic const size_t xsize = rm->rm_col[x].rc_size;
1402*e86372a0SGvozden Neskovic const size_t ysize = rm->rm_col[y].rc_size;
1403*e86372a0SGvozden Neskovic const size_t zsize = rm->rm_col[z].rc_size;
1404*e86372a0SGvozden Neskovic abd_t *xabd = rm->rm_col[x].rc_abd;
1405*e86372a0SGvozden Neskovic abd_t *yabd = rm->rm_col[y].rc_abd;
1406*e86372a0SGvozden Neskovic abd_t *zabd = rm->rm_col[z].rc_abd;
1407*e86372a0SGvozden Neskovic abd_t *tabds[] = { xabd, yabd, zabd };
1408*e86372a0SGvozden Neskovic abd_t *cabds[] = {
1409*e86372a0SGvozden Neskovic rm->rm_col[CODE_P].rc_abd,
1410*e86372a0SGvozden Neskovic rm->rm_col[CODE_Q].rc_abd,
1411*e86372a0SGvozden Neskovic rm->rm_col[CODE_R].rc_abd
1412*e86372a0SGvozden Neskovic };
1413*e86372a0SGvozden Neskovic unsigned coeff[MUL_CNT];
1414*e86372a0SGvozden Neskovic raidz_rec_pqr_coeff(rm, tgtidx, coeff);
1415*e86372a0SGvozden Neskovic
1416*e86372a0SGvozden Neskovic /*
1417*e86372a0SGvozden Neskovic * Check if some of targets is shorter then others
1418*e86372a0SGvozden Neskovic * In this case, shorter target needs to be replaced with
1419*e86372a0SGvozden Neskovic * new buffer so that syndrome can be calculated.
1420*e86372a0SGvozden Neskovic */
1421*e86372a0SGvozden Neskovic if (ysize < xsize) {
1422*e86372a0SGvozden Neskovic yabd = abd_alloc(xsize, B_FALSE);
1423*e86372a0SGvozden Neskovic tabds[1] = yabd;
1424*e86372a0SGvozden Neskovic }
1425*e86372a0SGvozden Neskovic if (zsize < xsize) {
1426*e86372a0SGvozden Neskovic zabd = abd_alloc(xsize, B_FALSE);
1427*e86372a0SGvozden Neskovic tabds[2] = zabd;
1428*e86372a0SGvozden Neskovic }
1429*e86372a0SGvozden Neskovic
1430*e86372a0SGvozden Neskovic raidz_math_begin();
1431*e86372a0SGvozden Neskovic
1432*e86372a0SGvozden Neskovic /* Start with first data column if present */
1433*e86372a0SGvozden Neskovic if (firstdc != x) {
1434*e86372a0SGvozden Neskovic raidz_copy(xabd, rm->rm_col[firstdc].rc_abd, xsize);
1435*e86372a0SGvozden Neskovic raidz_copy(yabd, rm->rm_col[firstdc].rc_abd, xsize);
1436*e86372a0SGvozden Neskovic raidz_copy(zabd, rm->rm_col[firstdc].rc_abd, xsize);
1437*e86372a0SGvozden Neskovic } else {
1438*e86372a0SGvozden Neskovic raidz_zero(xabd, xsize);
1439*e86372a0SGvozden Neskovic raidz_zero(yabd, xsize);
1440*e86372a0SGvozden Neskovic raidz_zero(zabd, xsize);
1441*e86372a0SGvozden Neskovic }
1442*e86372a0SGvozden Neskovic
1443*e86372a0SGvozden Neskovic /* generate q_syndrome */
1444*e86372a0SGvozden Neskovic for (c = firstdc+1; c < ncols; c++) {
1445*e86372a0SGvozden Neskovic if (c == x || c == y || c == z) {
1446*e86372a0SGvozden Neskovic dabd = NULL;
1447*e86372a0SGvozden Neskovic dsize = 0;
1448*e86372a0SGvozden Neskovic } else {
1449*e86372a0SGvozden Neskovic dabd = rm->rm_col[c].rc_abd;
1450*e86372a0SGvozden Neskovic dsize = rm->rm_col[c].rc_size;
1451*e86372a0SGvozden Neskovic }
1452*e86372a0SGvozden Neskovic
1453*e86372a0SGvozden Neskovic abd_raidz_gen_iterate(tabds, dabd, xsize, dsize, 3,
1454*e86372a0SGvozden Neskovic raidz_syn_pqr_abd);
1455*e86372a0SGvozden Neskovic }
1456*e86372a0SGvozden Neskovic
1457*e86372a0SGvozden Neskovic abd_raidz_rec_iterate(cabds, tabds, xsize, 3, raidz_rec_pqr_abd, coeff);
1458*e86372a0SGvozden Neskovic
1459*e86372a0SGvozden Neskovic /*
1460*e86372a0SGvozden Neskovic * Copy shorter targets back to the original abd buffer
1461*e86372a0SGvozden Neskovic */
1462*e86372a0SGvozden Neskovic if (ysize < xsize)
1463*e86372a0SGvozden Neskovic raidz_copy(rm->rm_col[y].rc_abd, yabd, ysize);
1464*e86372a0SGvozden Neskovic if (zsize < xsize)
1465*e86372a0SGvozden Neskovic raidz_copy(rm->rm_col[z].rc_abd, zabd, zsize);
1466*e86372a0SGvozden Neskovic
1467*e86372a0SGvozden Neskovic raidz_math_end();
1468*e86372a0SGvozden Neskovic
1469*e86372a0SGvozden Neskovic if (ysize < xsize)
1470*e86372a0SGvozden Neskovic abd_free(yabd);
1471*e86372a0SGvozden Neskovic if (zsize < xsize)
1472*e86372a0SGvozden Neskovic abd_free(zabd);
1473*e86372a0SGvozden Neskovic
1474*e86372a0SGvozden Neskovic return ((1 << CODE_P) | (1 << CODE_Q) | (1 << CODE_R));
1475*e86372a0SGvozden Neskovic }
1476*e86372a0SGvozden Neskovic
1477*e86372a0SGvozden Neskovic #endif /* _VDEV_RAIDZ_MATH_IMPL_H */
1478