197bbe2fandrew/*-
297bbe2fandrew * Copyright (c) 2016 The FreeBSD Foundation
397bbe2fandrew * All rights reserved.
497bbe2fandrew *
597bbe2fandrew * This software was developed by Andrew Turner under
697bbe2fandrew * sponsorship from the FreeBSD Foundation.
797bbe2fandrew *
897bbe2fandrew * Redistribution and use in source and binary forms, with or without
997bbe2fandrew * modification, are permitted provided that the following conditions
1097bbe2fandrew * are met:
1197bbe2fandrew * 1. Redistributions of source code must retain the above copyright
1297bbe2fandrew *    notice, this list of conditions and the following disclaimer.
1397bbe2fandrew * 2. Redistributions in binary form must reproduce the above copyright
1497bbe2fandrew *    notice, this list of conditions and the following disclaimer in the
1597bbe2fandrew *    documentation and/or other materials provided with the distribution.
1697bbe2fandrew *
1797bbe2fandrew * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
1897bbe2fandrew * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
1997bbe2fandrew * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
2097bbe2fandrew * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
2197bbe2fandrew * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
2297bbe2fandrew * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
2397bbe2fandrew * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
2497bbe2fandrew * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
2597bbe2fandrew * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
2697bbe2fandrew * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
2797bbe2fandrew * SUCH DAMAGE.
2897bbe2fandrew */
2997bbe2fandrew
3097bbe2fandrew/*
3197bbe2fandrew * This code is built with floating-point enabled. Make sure to have entered
3297bbe2fandrew * into floating-point context before calling any of these functions.
3397bbe2fandrew */
3497bbe2fandrew
3597bbe2fandrew#include <sys/cdefs.h>
3697bbe2fandrew__FBSDID("$FreeBSD$");
3797bbe2fandrew
3897bbe2fandrew#include <sys/param.h>
3997bbe2fandrew#include <sys/systm.h>
4097bbe2fandrew#include <sys/malloc.h>
4197bbe2fandrew#include <sys/queue.h>
4297bbe2fandrew
4397bbe2fandrew#include <opencrypto/cryptodev.h>
4497bbe2fandrew#include <crypto/armv8/armv8_crypto.h>
4597bbe2fandrew
4697bbe2fandrew#include <arm_neon.h>
4797bbe2fandrew
4897bbe2fandrewstatic uint8x16_t
4997bbe2fandrewarmv8_aes_enc(int rounds, const uint8x16_t *keysched, const uint8x16_t from)
5097bbe2fandrew{
5197bbe2fandrew	uint8x16_t tmp;
5297bbe2fandrew	int i;
5397bbe2fandrew
5497bbe2fandrew	tmp = from;
5597bbe2fandrew	for (i = 0; i < rounds - 1; i += 2) {
5697bbe2fandrew		tmp = vaeseq_u8(tmp, keysched[i]);
5797bbe2fandrew		tmp = vaesmcq_u8(tmp);
5897bbe2fandrew		tmp = vaeseq_u8(tmp, keysched[i + 1]);
5997bbe2fandrew		tmp = vaesmcq_u8(tmp);
6097bbe2fandrew	}
6197bbe2fandrew
6297bbe2fandrew	tmp = vaeseq_u8(tmp, keysched[rounds - 1]);
6397bbe2fandrew	tmp = vaesmcq_u8(tmp);
6497bbe2fandrew	tmp = vaeseq_u8(tmp, keysched[rounds]);
6597bbe2fandrew	tmp = veorq_u8(tmp, keysched[rounds + 1]);
6697bbe2fandrew
6797bbe2fandrew	return (tmp);
6897bbe2fandrew}
6997bbe2fandrew
7097bbe2fandrewstatic uint8x16_t
7197bbe2fandrewarmv8_aes_dec(int rounds, const uint8x16_t *keysched, const uint8x16_t from)
7297bbe2fandrew{
7397bbe2fandrew	uint8x16_t tmp;
7497bbe2fandrew	int i;
7597bbe2fandrew
7697bbe2fandrew	tmp = from;
7797bbe2fandrew	for (i = 0; i < rounds - 1; i += 2) {
7897bbe2fandrew		tmp = vaesdq_u8(tmp, keysched[i]);
7997bbe2fandrew		tmp = vaesimcq_u8(tmp);
8097bbe2fandrew		tmp = vaesdq_u8(tmp, keysched[i+1]);
8197bbe2fandrew		tmp = vaesimcq_u8(tmp);
8297bbe2fandrew	}
8397bbe2fandrew
8497bbe2fandrew	tmp = vaesdq_u8(tmp, keysched[rounds - 1]);
8597bbe2fandrew	tmp = vaesimcq_u8(tmp);
8697bbe2fandrew	tmp = vaesdq_u8(tmp, keysched[rounds]);
8797bbe2fandrew	tmp = veorq_u8(tmp, keysched[rounds + 1]);
8897bbe2fandrew
8997bbe2fandrew	return (tmp);
9097bbe2fandrew}
9197bbe2fandrew
9297bbe2fandrewvoid
9397bbe2fandrewarmv8_aes_encrypt_cbc(int rounds, const void *key_schedule, size_t len,
9497bbe2fandrew    const uint8_t *from, uint8_t *to, const uint8_t iv[static AES_BLOCK_LEN])
9597bbe2fandrew{
9697bbe2fandrew	uint8x16_t tot, ivreg, tmp;
9797bbe2fandrew	size_t i;
9897bbe2fandrew
9997bbe2fandrew	len /= AES_BLOCK_LEN;
10097bbe2fandrew	ivreg = vld1q_u8(iv);
10197bbe2fandrew	for (i = 0; i < len; i++) {
10297bbe2fandrew		tmp = vld1q_u8(from);
10397bbe2fandrew		tot = armv8_aes_enc(rounds - 1, key_schedule,
10497bbe2fandrew		    veorq_u8(tmp, ivreg));
10597bbe2fandrew		ivreg = tot;
10697bbe2fandrew		vst1q_u8(to, tot);
10797bbe2fandrew		from += AES_BLOCK_LEN;
10897bbe2fandrew		to += AES_BLOCK_LEN;
10997bbe2fandrew	}
11097bbe2fandrew}
11197bbe2fandrew
11297bbe2fandrewvoid
11397bbe2fandrewarmv8_aes_decrypt_cbc(int rounds, const void *key_schedule, size_t len,
11497bbe2fandrew    uint8_t *buf, const uint8_t iv[static AES_BLOCK_LEN])
11597bbe2fandrew{
11697bbe2fandrew	uint8x16_t ivreg, nextiv, tmp;
11797bbe2fandrew	size_t i;
11897bbe2fandrew
11997bbe2fandrew	len /= AES_BLOCK_LEN;
12097bbe2fandrew	ivreg = vld1q_u8(iv);
12197bbe2fandrew	for (i = 0; i < len; i++) {
12297bbe2fandrew		nextiv = vld1q_u8(buf);
12397bbe2fandrew		tmp = armv8_aes_dec(rounds - 1, key_schedule, nextiv);
12497bbe2fandrew		vst1q_u8(buf, veorq_u8(tmp, ivreg));
12597bbe2fandrew		ivreg = nextiv;
12697bbe2fandrew		buf += AES_BLOCK_LEN;
12797bbe2fandrew	}
12897bbe2fandrew}
129