xref: /illumos-gate/usr/src/lib/gss_mechs/mech_dh/backend/mech/crypto.c (revision 694c35fa)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 *	crypto.c
 *
 *	Copyright (c) 1997, by Sun Microsystems, Inc.
 *	All rights reserved.
 *
 */

#include <sys/note.h>
#include "dh_gssapi.h"
#include "crypto.h"

/* Release the storage for a signature */
void
__free_signature(dh_signature_t sig)
{
	Free(sig->dh_signature_val);
	sig->dh_signature_val = NULL;
	sig->dh_signature_len = 0;
}

/* Release the storage for a gss_buffer */
void
__dh_release_buffer(gss_buffer_t b)
{
	Free(b->value);
	b->length = 0;
	b->value = NULL;
}

typedef struct cipher_entry {
	cipher_proc cipher;	/* Routine to en/decrypt with */
	unsigned int pad;	/* Padding need for the routine */
} cipher_entry, *cipher_t;

typedef struct verifer_entry {
	verifier_proc msg;	/* Routine to calculate the check sum */
	unsigned int size;	/* Size of check sum */
	cipher_t signer;	/* Cipher entry to sign the check sum */
} verifier_entry, *verifier_t;

typedef struct QOP_entry {
	int export_level;	/* Not currentlyt used */
	verifier_t verifier;	/* Verifier entry to use for integrity */
} QOP_entry;

/*
 * Return the length produced by using cipher entry c given the supplied len
 */
static unsigned int
cipher_pad(cipher_t c, unsigned int len)
{
	unsigned int pad;

	pad = c ? c->pad : 1;

	return (((len + pad - 1)/pad)*pad);
}


/*
 * Des [en/de]crypt buffer, buf of length, len for each key provided using
 * an CBC initialization vector ivec.
 * If the mode is encrypt we will use the following pattern if the number
 * of keys is odd
 * encrypt(buf, k[0]), decrypt(buf, k[1]), encrypt(buf, k[2])
 *	decrypt(buf, k[4]) ... encrypt(buf, k[keynum - 1])
 * If we have an even number of keys and additional encryption will be
 * done with the first key, i.e., ecrypt(buf, k[0]);
 * In each [en/de]cription above we will used the passed in CBC initialization
 * vector. The new initialization vector will be the vector return from the
 * last encryption.
 *
 * In the decryption case we reverse the proccess. Note in this case
 * the return ivec will be from the first decryption.
 */

static int
__desN_crypt(des_block keys[], int keynum, char *buf, unsigned int len,
    unsigned int mode, char *ivec)
{
	/* Get the direction of ciphering */
	unsigned int m = mode & (DES_ENCRYPT | DES_DECRYPT);
	/* Get the remaining flags from mode */
	unsigned int flags = mode & ~(DES_ENCRYPT | DES_DECRYPT);
	des_block svec, dvec;
	int i, j, stat;

	/* Do we have at least one key */
	if (keynum < 1)
		return (DESERR_BADPARAM);

	/* Save the passed in ivec */
	memcpy(svec.c, ivec, sizeof (des_block));

	/* For  each key do the appropriate cipher */
	for (i = 0; i < keynum; i++) {
		j = (mode & DES_DECRYPT) ? keynum - 1 - i : i;
		stat = cbc_crypt(keys[j].c, buf, len, m | flags, ivec);
		if (mode & DES_DECRYPT && i == 0)
			memcpy(dvec.c, ivec, sizeof (des_block));

		if (DES_FAILED(stat))
			return (stat);

		m = (m == DES_ENCRYPT ? DES_DECRYPT : DES_ENCRYPT);

		if ((mode & DES_DECRYPT) || i != keynum - 1 || i%2)
			memcpy(ivec, svec.c, sizeof (des_block));
	}

	/*
	 * If we have an even number of keys then do an extra round of
	 * [en/de]cryption with the first key.
	 */
	if (keynum % 2 == 0)
		stat = cbc_crypt(keys[0].c, buf, len, mode, ivec);

	/* If were decrypting ivec is set from first decryption */
	if (mode & DES_DECRYPT)
		memcpy(ivec, dvec.c, sizeof (des_block));

	return (stat);
}


/*
 * DesN crypt packaged for use as a cipher entry
 */
static OM_uint32
__dh_desN_crypt(gss_buffer_t buf, dh_key_set_t keys, cipher_mode_t cipher_mode)
{
	int stat = DESERR_BADPARAM;
	int encrypt_flag = (cipher_mode == ENCIPHER);
	unsigned mode = (encrypt_flag ? DES_ENCRYPT : DES_DECRYPT) | DES_HW;
	des_block ivec;

	if (keys->dh_key_set_len < 1)
		return (DH_BADARG_FAILURE);

	/*
	 * We all ways start of with ivec set to zeros. There is no
	 * good way to maintain ivecs since packets could be out of sequence
	 * duplicated or worst of all lost. Under these conditions the
	 * higher level protocol would have to some how resync the ivecs
	 * on both sides and start again. Theres no mechanism for this in
	 * GSS.
	 */
	memset(&ivec, 0, sizeof (ivec));

	/* Do the encryption/decryption */
	stat = __desN_crypt(keys->dh_key_set_val, keys->dh_key_set_len,
			    (char *)buf->value, buf->length, mode, ivec.c);

	if (DES_FAILED(stat))
		return (DH_CIPHER_FAILURE);

	return (DH_SUCCESS);
}

/*
 * Package up plain des cbc crypt for use as a cipher entry.
 */
static OM_uint32
__dh_des_crypt(gss_buffer_t buf, dh_key_set_t keys, cipher_mode_t cipher_mode)
{
	int stat = DESERR_BADPARAM;
	int encrypt_flag = (cipher_mode == ENCIPHER);
	unsigned mode = (encrypt_flag ? DES_ENCRYPT : DES_DECRYPT) | DES_HW;
	des_block ivec;

	if (keys->dh_key_set_len < 1)
		return (DH_BADARG_FAILURE);

	/*  Set the ivec to zeros and then cbc crypt the result */
	memset(&ivec, 0, sizeof (ivec));
	stat = cbc_crypt(keys->dh_key_set_val[0].c, (char *)buf->value,
			buf->length, mode, ivec.c);

	if (DES_FAILED(stat))
		return (DH_CIPHER_FAILURE);

	return (DH_SUCCESS);
}

/*
 * MD5_verifier: This is a verifier routine suitable for use in a
 * verifier entry. It calculates the MD5 check sum over an optional
 * msg and a token. It signs it using the supplied cipher_proc and stores
 * the result in signature.
 *
 * Note signature should already be allocated and be large enough to
 * hold the signature after its been encrypted. If keys is null, then
 * we will just return the unencrypted check sum.
 */
static OM_uint32
MD5_verifier(gss_buffer_t tok, /* The buffer to sign */
	    gss_buffer_t msg, /* Optional buffer to include */
	    cipher_proc signer, /* Routine to encrypt the integrity check */
	    dh_key_set_t keys, /* Optiona keys to be used with the above */
	    dh_signature_t signature /* The resulting MIC */)
{
	MD5_CTX md5_ctx;	/* MD5 context */
	gss_buffer_desc buf;	/* GSS buffer to hold keys for cipher routine */

	/* Initialize the MD5 context */
	MD5Init(&md5_ctx);
	/* If we have a message to digest, digest it */
	if (msg)
	    MD5Update(&md5_ctx, (unsigned char *)msg->value, msg->length);
	/* Digest the supplied token */
	MD5Update(&md5_ctx, (unsigned char *)tok->value, tok->length);
	/* Finalize the sum. The MD5 context contains the digets */
	MD5Final(&md5_ctx);

	/* Copy the digest to the signature */
	memcpy(signature->dh_signature_val, (void *)md5_ctx.digest, 16);

	buf.length = signature->dh_signature_len;
	buf.value = signature->dh_signature_val;

	/* If we have keys encrypt it */
	if (keys != NULL)
		return (signer(&buf, keys, ENCIPHER));

	return (DH_SUCCESS);
}

/* Cipher table */
static
cipher_entry cipher_tab[] = {
	{ NULL, 1},
	{ __dh_desN_crypt, 8},
	{ __dh_des_crypt, 8}
};


#define	__NO_CRYPT	&cipher_tab[0]
#define	__DES_N_CRYPT	&cipher_tab[1]
#define	__DES_CRYPT	&cipher_tab[2]

/* Verifier table */
static
verifier_entry verifier_tab[] = {
	{ MD5_verifier, 16, __DES_N_CRYPT },
	{ MD5_verifier, 16, __DES_CRYPT }
};

/* QOP table */
static
QOP_entry QOP_table[] = {
	{ 0, &verifier_tab[0] },
	{ 0, &verifier_tab[1] }
};

#define	QOP_ENTRIES (sizeof (QOP_table) / sizeof (QOP_entry))

/*
 * __dh_is_valid_QOP: Return true if qop is valid entry into the QOP
 * table, else return false.
 */
bool_t
__dh_is_valid_QOP(dh_qop_t qop)
{
	bool_t is_valid = FALSE;

	is_valid = qop < QOP_ENTRIES;

	return (is_valid);
}

/*
 * __alloc_sig: Allocate a signature for a given QOP. This takes into
 * account the size of the signature after padding for the encryption
 * routine.
 */
OM_uint32
__alloc_sig(dh_qop_t qop, dh_signature_t sig)
{
	OM_uint32 stat = DH_VERIFIER_FAILURE;
	verifier_entry *v;

	/* Check that the QOP is valid */
	if (!__dh_is_valid_QOP(qop))
		return (DH_UNKNOWN_QOP);

	/* Get the verifier entry from the QOP entry */
	v = QOP_table[qop].verifier;

	/* Calulate the length needed for the signature */
	sig->dh_signature_len = cipher_pad(v->signer, v->size);

	/* Allocate the signature */
	sig->dh_signature_val = (void*)New(char, sig->dh_signature_len);
	if (sig->dh_signature_val == NULL) {
		sig->dh_signature_len = 0;
		return (DH_NOMEM_FAILURE);
	}

	stat = DH_SUCCESS;

	return (stat);
}

/*
 * __get_sig_size: Return the total size needed for a signature given a QOP.
 */
OM_uint32
__get_sig_size(dh_qop_t qop, unsigned int *size)
{
	/* Check for valid QOP */
	if (__dh_is_valid_QOP(qop)) {
		/* Get the verifier entry */
		verifier_t v = QOP_table[qop].verifier;

		/* Return the size include the padding needed for encryption */
		*size = v ? cipher_pad(v->signer, v->size) : 0;

		return (DH_SUCCESS);
	}
	*size = 0;

	return (DH_UNKNOWN_QOP);
}

/*
 * __mk_sig: Generate a signature using a given qop over a token of a
 * given length and an optional message. We use the supplied keys to
 * encrypt the check sum if they are available. The output is place
 * in a preallocate signature, that was allocated using __alloc_sig.
 */
OM_uint32
__mk_sig(dh_qop_t qop, /* The QOP to use */
	char *tok, /* The token to sign */
	long len, /* The tokens length */
	gss_buffer_t mesg,	/* An optional message to be included */
	dh_key_set_t keys, /* The optional encryption keys */
	dh_signature_t sig /* The resulting MIC */)
{
	OM_uint32 stat = DH_VERIFIER_FAILURE;


	verifier_entry *v;	/* Verifier entry */
	gss_buffer_desc buf;	/* Buffer to package tok */

	/* Make sure the QOP is valid */
	if (!__dh_is_valid_QOP(qop))
		return (DH_UNKNOWN_QOP);

	/* Grab the verifier entry for the qop */
	v = QOP_table[qop].verifier;

	/* Package the token for use in a verifier_proc */
	buf.length = len;
	buf.value = tok;

	/*
	 * Calculate the signature using the supplied keys. If keys
	 * is null, the the v->signer->cipher routine will not be called
	 * and sig will not be encrypted.
	 */
	stat = (*v->msg)(&buf, mesg, v->signer->cipher, keys, sig);

	return (stat);
}

/*
 * __verify_sig: Verify that the supplied signature, sig, is the same
 * as the token verifier
 */
OM_uint32
__verify_sig(dh_token_t token, /* The token to be verified */
	    dh_qop_t qop, /* The QOP to use */
	    dh_key_set_t keys, /* The context session keys */
	    dh_signature_t sig /* The signature from the serialized token */)
{
	OM_uint32 stat = DH_VERIFIER_FAILURE;

	cipher_proc cipher;	/* cipher routine to use */
	gss_buffer_desc buf;	/* Packaging for sig */

	/* Check the QOP */
	if (!__dh_is_valid_QOP(qop))
		return (DH_UNKNOWN_QOP);

	/* Package up the supplied signature */
	buf.length = sig->dh_signature_len;
	buf.value = sig->dh_signature_val;

	/* Get the cipher proc to use from the verifier entry for qop */
	cipher = QOP_table[qop].verifier->signer->cipher;

	/* Encrypt the check sum using the supplied set of keys */
	if ((stat = (*cipher)(&buf, keys, ENCIPHER)) != DH_SUCCESS)
		return (stat);

	/* Compare the signatures */
	if (__cmpsig(sig, &token->verifier))
		return (DH_SUCCESS);

	stat = DH_VERIFIER_MISMATCH;

	return (stat);
}

/*
 * __cmpsig: Return true if two signatures are the same, else false.
 */
bool_t
__cmpsig(dh_signature_t s1, dh_signature_t s2)
{
	return (s1->dh_signature_len == s2->dh_signature_len &&
	    memcmp(s1->dh_signature_val,
		s2->dh_signature_val, s1->dh_signature_len) == 0);
}

/*
 * wrap_msg_body: Wrap the message pointed to be in into a
 * message pointed to by out that has ben padded out by pad bytes.
 *
 * The output message looks like:
 * out->length = total length of out->value including any padding
 * out->value points to memory as follows:
 * +------------+-------------------------+---------|
 * | in->length | in->value               | XDR PAD |
 * +------------+-------------------------+---------|
 *    4 bytes      in->length bytes         0 - 3
 */
static OM_uint32
wrap_msg_body(gss_buffer_t in, gss_buffer_t out)
{
	XDR xdrs;			/* xdrs to wrap with */
	unsigned int len, out_len;	/* length  */
	size_t size;

	out->length = 0;
	out->value = 0;

	/* Make sure the address of len points to a 32 bit word */
	len = (unsigned int)in->length;
	if (len != in->length)
		return (DH_ENCODE_FAILURE);

	size = ((in->length + sizeof (OM_uint32) + 3)/4) * 4;
	out_len = size;
	if (out_len != size)
		return (DH_ENCODE_FAILURE);

	/* Allocate the output buffer and set the length */
	if ((out->value = (void *)New(char, len)) == NULL)
		return (DH_NOMEM_FAILURE);
	out->length = out_len;


	/* Create xdr stream to wrap into */
	xdrmem_create(&xdrs, out->value, out->length, XDR_ENCODE);

	/* Wrap the bytes in value */
	if (!xdr_bytes(&xdrs, (char **)&in->value, &len, len)) {
		__dh_release_buffer(out);
		return (DH_ENCODE_FAILURE);
	}

	return (DH_SUCCESS);
}

/*
 * __QOPSeal: Wrap the input message placing the output in output given
 * a valid QOP. If confidentialiy is requested it is ignored. We can't
 * support privacy. The return flag will always be zero.
 */
OM_uint32
__QOPSeal(dh_qop_t qop, /* The QOP to use */
	gss_buffer_t input, /* The buffer to wrap */
	int conf_req, /* Do we want privacy ? */
	dh_key_set_t keys, /* The session keys */
	gss_buffer_t output, /* The wraped message */
	int *conf_ret /* Did we encrypt it? */)
{
_NOTE(ARGUNUSED(conf_req,keys))
	OM_uint32 stat = DH_CIPHER_FAILURE;

	*conf_ret = FALSE;	/* No encryption allowed */

	/* Check for valid QOP */
	if (!__dh_is_valid_QOP(qop))
		return (DH_UNKNOWN_QOP);

	/* Wrap the message */
	if ((stat = wrap_msg_body(input, output))
	    != DH_SUCCESS)
		return (stat);

	return (stat);
}

/*
 * unwrap_msg_body: Unwrap the message, that was wrapped from above
 */
static OM_uint32
unwrap_msg_body(gss_buffer_t in, gss_buffer_t out)
{
	XDR xdrs;
	unsigned int len;	/* sizeof (len) == 32bits */

	/* Create an xdr stream to on wrap in */
	xdrmem_create(&xdrs, in->value, in->length, XDR_DECODE);

	/* Unwrap the input into out->value */
	if (!xdr_bytes(&xdrs, (char **)&out->value, &len, in->length))
		return (DH_DECODE_FAILURE);

	/* set the length */
	out->length = len;

	return (DH_SUCCESS);
}

/*
 * __QOPUnSeal: Unwrap the input message into output using the supplied QOP.
 * Note it is the callers responsibility to release the allocated output
 * buffer. If conf_req is true we return DH_CIPHER_FAILURE since we don't
 * support privacy.
 */
OM_uint32
__QOPUnSeal(dh_qop_t qop, /* The QOP to use */
	    gss_buffer_t input, /* The message to unwrap */
	    int conf_req, /* Is the message encrypted */
	    dh_key_set_t keys, /* The session keys to decrypt if conf_req */
	    gss_buffer_t output /* The unwraped message */)
{
_NOTE(ARGUNUSED(keys))
	OM_uint32 stat = DH_CIPHER_FAILURE;

	/* Check that the qop is valid */
	if (!__dh_is_valid_QOP(qop))
		return (DH_UNKNOWN_QOP);

	/* Set output to sane values */
	output->length = 0;
	output->value = NULL;

	/* Fail if this is privacy */
	if (conf_req)
		return (DH_CIPHER_FAILURE);

	/* Unwrap the input into the output, return the status */
	stat = unwrap_msg_body(input, output);

	return (stat);
}