1/*-
2 * Copyright (c) 2016 The FreeBSD Foundation
3 * All rights reserved.
4 *
5 * This software was developed by Andrew Turner under
6 * sponsorship from the FreeBSD Foundation.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30/*
31 * This code is built with floating-point enabled. Make sure to have entered
32 * into floating-point context before calling any of these functions.
33 */
34
35#include <sys/cdefs.h>
36__FBSDID("$FreeBSD$");
37
38#include <sys/param.h>
39#include <sys/systm.h>
40#include <sys/malloc.h>
41#include <sys/queue.h>
42
43#include <opencrypto/cryptodev.h>
44#include <crypto/armv8/armv8_crypto.h>
45
46#include <arm_neon.h>
47
48static uint8x16_t
49armv8_aes_enc(int rounds, const uint8x16_t *keysched, const uint8x16_t from)
50{
51	uint8x16_t tmp;
52	int i;
53
54	tmp = from;
55	for (i = 0; i < rounds - 1; i += 2) {
56		tmp = vaeseq_u8(tmp, keysched[i]);
57		tmp = vaesmcq_u8(tmp);
58		tmp = vaeseq_u8(tmp, keysched[i + 1]);
59		tmp = vaesmcq_u8(tmp);
60	}
61
62	tmp = vaeseq_u8(tmp, keysched[rounds - 1]);
63	tmp = vaesmcq_u8(tmp);
64	tmp = vaeseq_u8(tmp, keysched[rounds]);
65	tmp = veorq_u8(tmp, keysched[rounds + 1]);
66
67	return (tmp);
68}
69
70static uint8x16_t
71armv8_aes_dec(int rounds, const uint8x16_t *keysched, const uint8x16_t from)
72{
73	uint8x16_t tmp;
74	int i;
75
76	tmp = from;
77	for (i = 0; i < rounds - 1; i += 2) {
78		tmp = vaesdq_u8(tmp, keysched[i]);
79		tmp = vaesimcq_u8(tmp);
80		tmp = vaesdq_u8(tmp, keysched[i+1]);
81		tmp = vaesimcq_u8(tmp);
82	}
83
84	tmp = vaesdq_u8(tmp, keysched[rounds - 1]);
85	tmp = vaesimcq_u8(tmp);
86	tmp = vaesdq_u8(tmp, keysched[rounds]);
87	tmp = veorq_u8(tmp, keysched[rounds + 1]);
88
89	return (tmp);
90}
91
92void
93armv8_aes_encrypt_cbc(int rounds, const void *key_schedule, size_t len,
94    const uint8_t *from, uint8_t *to, const uint8_t iv[static AES_BLOCK_LEN])
95{
96	uint8x16_t tot, ivreg, tmp;
97	size_t i;
98
99	len /= AES_BLOCK_LEN;
100	ivreg = vld1q_u8(iv);
101	for (i = 0; i < len; i++) {
102		tmp = vld1q_u8(from);
103		tot = armv8_aes_enc(rounds - 1, key_schedule,
104		    veorq_u8(tmp, ivreg));
105		ivreg = tot;
106		vst1q_u8(to, tot);
107		from += AES_BLOCK_LEN;
108		to += AES_BLOCK_LEN;
109	}
110}
111
112void
113armv8_aes_decrypt_cbc(int rounds, const void *key_schedule, size_t len,
114    uint8_t *buf, const uint8_t iv[static AES_BLOCK_LEN])
115{
116	uint8x16_t ivreg, nextiv, tmp;
117	size_t i;
118
119	len /= AES_BLOCK_LEN;
120	ivreg = vld1q_u8(iv);
121	for (i = 0; i < len; i++) {
122		nextiv = vld1q_u8(buf);
123		tmp = armv8_aes_dec(rounds - 1, key_schedule, nextiv);
124		vst1q_u8(buf, veorq_u8(tmp, ivreg));
125		ivreg = nextiv;
126		buf += AES_BLOCK_LEN;
127	}
128}
129