1/*
2 * Intel SHA Extensions optimized implementation of a SHA-1 update function
3 *
4 * This file is provided under a dual BSD/GPLv2 license.  When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *
9 * Copyright(c) 2015 Intel Corporation.
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of version 2 of the GNU General Public License as
13 * published by the Free Software Foundation.
14 *
15 * This program is distributed in the hope that it will be useful, but
16 * WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18 * General Public License for more details.
19 *
20 * Contact Information:
21 * 	Sean Gulley <sean.m.gulley@intel.com>
22 * 	Tim Chen <tim.c.chen@linux.intel.com>
23 *
24 * BSD LICENSE
25 *
26 * Copyright(c) 2015 Intel Corporation.
27 *
28 * Redistribution and use in source and binary forms, with or without
29 * modification, are permitted provided that the following conditions
30 * are met:
31 *
32 * 	* Redistributions of source code must retain the above copyright
33 * 	  notice, this list of conditions and the following disclaimer.
34 * 	* Redistributions in binary form must reproduce the above copyright
35 * 	  notice, this list of conditions and the following disclaimer in
36 * 	  the documentation and/or other materials provided with the
37 * 	  distribution.
38 * 	* Neither the name of Intel Corporation nor the names of its
39 * 	  contributors may be used to endorse or promote products derived
40 * 	  from this software without specific prior written permission.
41 *
42 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
43 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
44 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
45 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
46 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
47 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
48 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
49 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
50 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
51 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
52 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
53 *
54 */
55
56/*
57 * Copyright (c) 2018, Joyent, Inc.
58 */
59
60/*
61 * illumos uses this file under the terms of the BSD license.
62 *
63 * The following are a series of changes that we have made to this code:
64 *
65 *  o Changed the include to be sys/asm_linkage.h
66 *  o Use the sys/asm_linkage.h prototypes for assembly functions
67 *  o Renamed the function from sha1_ni_transform to sha1_block_data_order to
68 *    match the illumos name for the function
69 */
70
71#include <sys/asm_linkage.h>
72
73#define DIGEST_PTR	%rdi	/* 1st arg */
74#define DATA_PTR	%rsi	/* 2nd arg */
75#define NUM_BLKS	%rdx	/* 3rd arg */
76
77#define RSPSAVE		%rax
78
79/* gcc conversion */
80#define FRAME_SIZE	32	/* space for 2x16 bytes */
81
82#define ABCD		%xmm0
83#define E0		%xmm1	/* Need two E's b/c they ping pong */
84#define E1		%xmm2
85#define MSG0		%xmm3
86#define MSG1		%xmm4
87#define MSG2		%xmm5
88#define MSG3		%xmm6
89#define SHUF_MASK	%xmm7
90
91
92/*
93 * Intel SHA Extensions optimized implementation of a SHA-1 update function
94 *
95 * The function takes a pointer to the current hash values, a pointer to the
96 * input data, and a number of 64 byte blocks to process.  Once all blocks have
97 * been processed, the digest pointer is  updated with the resulting hash value.
98 * The function only processes complete blocks, there is no functionality to
99 * store partial blocks. All message padding and hash value initialization must
100 * be done outside the update function.
101 *
102 * The indented lines in the loop are instructions related to rounds processing.
103 * The non-indented lines are instructions related to the message schedule.
104 *
105 * void sha1_block_data_order(uint32_t *digest, const void *data,
106		uint32_t numBlocks)
107 * digest : pointer to digest
108 * data: pointer to input data
109 * numBlocks: Number of blocks to process
110 */
111.text
112.align 32
113ENTRY_NP(sha1_block_data_order)
114	mov		%rsp, RSPSAVE
115	sub		$FRAME_SIZE, %rsp
116	and		$~0xF, %rsp
117
118	shl		$6, NUM_BLKS		/* convert to bytes */
119	jz		.Ldone_hash
120	add		DATA_PTR, NUM_BLKS	/* pointer to end of data */
121
122	/* load initial hash values */
123	pinsrd		$3, 1*16(DIGEST_PTR), E0
124	movdqu		0*16(DIGEST_PTR), ABCD
125	pand		UPPER_WORD_MASK(%rip), E0
126	pshufd		$0x1B, ABCD, ABCD
127
128	movdqa		PSHUFFLE_BYTE_FLIP_MASK(%rip), SHUF_MASK
129
130.Lloop0:
131	/* Save hash values for addition after rounds */
132	movdqa		E0, (0*16)(%rsp)
133	movdqa		ABCD, (1*16)(%rsp)
134
135	/* Rounds 0-3 */
136	movdqu		0*16(DATA_PTR), MSG0
137	pshufb		SHUF_MASK, MSG0
138		paddd		MSG0, E0
139		movdqa		ABCD, E1
140		sha1rnds4	$0, E0, ABCD
141
142	/* Rounds 4-7 */
143	movdqu		1*16(DATA_PTR), MSG1
144	pshufb		SHUF_MASK, MSG1
145		sha1nexte	MSG1, E1
146		movdqa		ABCD, E0
147		sha1rnds4	$0, E1, ABCD
148	sha1msg1	MSG1, MSG0
149
150	/* Rounds 8-11 */
151	movdqu		2*16(DATA_PTR), MSG2
152	pshufb		SHUF_MASK, MSG2
153		sha1nexte	MSG2, E0
154		movdqa		ABCD, E1
155		sha1rnds4	$0, E0, ABCD
156	sha1msg1	MSG2, MSG1
157	pxor		MSG2, MSG0
158
159	/* Rounds 12-15 */
160	movdqu		3*16(DATA_PTR), MSG3
161	pshufb		SHUF_MASK, MSG3
162		sha1nexte	MSG3, E1
163		movdqa		ABCD, E0
164	sha1msg2	MSG3, MSG0
165		sha1rnds4	$0, E1, ABCD
166	sha1msg1	MSG3, MSG2
167	pxor		MSG3, MSG1
168
169	/* Rounds 16-19 */
170		sha1nexte	MSG0, E0
171		movdqa		ABCD, E1
172	sha1msg2	MSG0, MSG1
173		sha1rnds4	$0, E0, ABCD
174	sha1msg1	MSG0, MSG3
175	pxor		MSG0, MSG2
176
177	/* Rounds 20-23 */
178		sha1nexte	MSG1, E1
179		movdqa		ABCD, E0
180	sha1msg2	MSG1, MSG2
181		sha1rnds4	$1, E1, ABCD
182	sha1msg1	MSG1, MSG0
183	pxor		MSG1, MSG3
184
185	/* Rounds 24-27 */
186		sha1nexte	MSG2, E0
187		movdqa		ABCD, E1
188	sha1msg2	MSG2, MSG3
189		sha1rnds4	$1, E0, ABCD
190	sha1msg1	MSG2, MSG1
191	pxor		MSG2, MSG0
192
193	/* Rounds 28-31 */
194		sha1nexte	MSG3, E1
195		movdqa		ABCD, E0
196	sha1msg2	MSG3, MSG0
197		sha1rnds4	$1, E1, ABCD
198	sha1msg1	MSG3, MSG2
199	pxor		MSG3, MSG1
200
201	/* Rounds 32-35 */
202		sha1nexte	MSG0, E0
203		movdqa		ABCD, E1
204	sha1msg2	MSG0, MSG1
205		sha1rnds4	$1, E0, ABCD
206	sha1msg1	MSG0, MSG3
207	pxor		MSG0, MSG2
208
209	/* Rounds 36-39 */
210		sha1nexte	MSG1, E1
211		movdqa		ABCD, E0
212	sha1msg2	MSG1, MSG2
213		sha1rnds4	$1, E1, ABCD
214	sha1msg1	MSG1, MSG0
215	pxor		MSG1, MSG3
216
217	/* Rounds 40-43 */
218		sha1nexte	MSG2, E0
219		movdqa		ABCD, E1
220	sha1msg2	MSG2, MSG3
221		sha1rnds4	$2, E0, ABCD
222	sha1msg1	MSG2, MSG1
223	pxor		MSG2, MSG0
224
225	/* Rounds 44-47 */
226		sha1nexte	MSG3, E1
227		movdqa		ABCD, E0
228	sha1msg2	MSG3, MSG0
229		sha1rnds4	$2, E1, ABCD
230	sha1msg1	MSG3, MSG2
231	pxor		MSG3, MSG1
232
233	/* Rounds 48-51 */
234		sha1nexte	MSG0, E0
235		movdqa		ABCD, E1
236	sha1msg2	MSG0, MSG1
237		sha1rnds4	$2, E0, ABCD
238	sha1msg1	MSG0, MSG3
239	pxor		MSG0, MSG2
240
241	/* Rounds 52-55 */
242		sha1nexte	MSG1, E1
243		movdqa		ABCD, E0
244	sha1msg2	MSG1, MSG2
245		sha1rnds4	$2, E1, ABCD
246	sha1msg1	MSG1, MSG0
247	pxor		MSG1, MSG3
248
249	/* Rounds 56-59 */
250		sha1nexte	MSG2, E0
251		movdqa		ABCD, E1
252	sha1msg2	MSG2, MSG3
253		sha1rnds4	$2, E0, ABCD
254	sha1msg1	MSG2, MSG1
255	pxor		MSG2, MSG0
256
257	/* Rounds 60-63 */
258		sha1nexte	MSG3, E1
259		movdqa		ABCD, E0
260	sha1msg2	MSG3, MSG0
261		sha1rnds4	$3, E1, ABCD
262	sha1msg1	MSG3, MSG2
263	pxor		MSG3, MSG1
264
265	/* Rounds 64-67 */
266		sha1nexte	MSG0, E0
267		movdqa		ABCD, E1
268	sha1msg2	MSG0, MSG1
269		sha1rnds4	$3, E0, ABCD
270	sha1msg1	MSG0, MSG3
271	pxor		MSG0, MSG2
272
273	/* Rounds 68-71 */
274		sha1nexte	MSG1, E1
275		movdqa		ABCD, E0
276	sha1msg2	MSG1, MSG2
277		sha1rnds4	$3, E1, ABCD
278	pxor		MSG1, MSG3
279
280	/* Rounds 72-75 */
281		sha1nexte	MSG2, E0
282		movdqa		ABCD, E1
283	sha1msg2	MSG2, MSG3
284		sha1rnds4	$3, E0, ABCD
285
286	/* Rounds 76-79 */
287		sha1nexte	MSG3, E1
288		movdqa		ABCD, E0
289		sha1rnds4	$3, E1, ABCD
290
291	/* Add current hash values with previously saved */
292	sha1nexte	(0*16)(%rsp), E0
293	paddd		(1*16)(%rsp), ABCD
294
295	/* Increment data pointer and loop if more to process */
296	add		$64, DATA_PTR
297	cmp		NUM_BLKS, DATA_PTR
298	jne		.Lloop0
299
300	/* Write hash values back in the correct order */
301	pshufd		$0x1B, ABCD, ABCD
302	movdqu		ABCD, 0*16(DIGEST_PTR)
303	pextrd		$3, E0, 1*16(DIGEST_PTR)
304
305.Ldone_hash:
306	mov		RSPSAVE, %rsp
307
308	ret
309SET_SIZE(sha1_block_data_order)
310
311.section	.rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16
312.align 16
313PSHUFFLE_BYTE_FLIP_MASK:
314	.octa 0x000102030405060708090a0b0c0d0e0f
315
316.section	.rodata.cst16.UPPER_WORD_MASK, "aM", @progbits, 16
317.align 16
318UPPER_WORD_MASK:
319	.octa 0xFFFFFFFF000000000000000000000000
320