xref: /illumos-gate/usr/src/cmd/sendmail/src/tls.c (revision 553e44ce)
17c478bd9Sstevel@tonic-gate /*
2e9af4bc0SJohn Beck  * Copyright (c) 2000-2006, 2008, 2009 Sendmail, Inc. and its suppliers.
37c478bd9Sstevel@tonic-gate  *	All rights reserved.
470f9559bSTheo Schlossnagle  * Copyright (c) 2012, OmniTI Computer Consulting, Inc. All rights reserved.
5300fdee2SAndy Fiddaman  * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
6*553e44ceSAndrew Stormont  * Copyright 2018 RackTop Systems.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * By using this file, you agree to the terms and conditions set
97c478bd9Sstevel@tonic-gate  * forth in the LICENSE file which can be found at the top level of
107c478bd9Sstevel@tonic-gate  * the sendmail distribution.
117c478bd9Sstevel@tonic-gate  *
127c478bd9Sstevel@tonic-gate  */
137c478bd9Sstevel@tonic-gate 
147c478bd9Sstevel@tonic-gate #include <sendmail.h>
157c478bd9Sstevel@tonic-gate 
16e9af4bc0SJohn Beck SM_RCSID("@(#)$Id: tls.c,v 8.114 2009/08/10 15:11:09 ca Exp $")
177c478bd9Sstevel@tonic-gate 
187c478bd9Sstevel@tonic-gate #if STARTTLS
197c478bd9Sstevel@tonic-gate #  include <openssl/err.h>
207c478bd9Sstevel@tonic-gate #  include <openssl/bio.h>
217c478bd9Sstevel@tonic-gate #  include <openssl/pem.h>
22300fdee2SAndy Fiddaman #  include <openssl/bn.h>
237c478bd9Sstevel@tonic-gate #  ifndef HASURANDOMDEV
247c478bd9Sstevel@tonic-gate #   include <openssl/rand.h>
257c478bd9Sstevel@tonic-gate #  endif /* ! HASURANDOMDEV */
267c478bd9Sstevel@tonic-gate # if !TLS_NO_RSA
27300fdee2SAndy Fiddaman #  include <openssl/rsa.h>
287c478bd9Sstevel@tonic-gate static RSA *rsa_tmp = NULL;	/* temporary RSA key */
297c478bd9Sstevel@tonic-gate # endif /* !TLS_NO_RSA */
307c478bd9Sstevel@tonic-gate static int	tls_verify_cb __P((X509_STORE_CTX *, void *));
317c478bd9Sstevel@tonic-gate static int x509_verify_cb __P((int, X509_STORE_CTX *));
327c478bd9Sstevel@tonic-gate 
337c478bd9Sstevel@tonic-gate #  define CONST097 const
347c478bd9Sstevel@tonic-gate static void	apps_ssl_info_cb __P((CONST097 SSL *, int , int));
357c478bd9Sstevel@tonic-gate static bool	tls_ok_f __P((char *, char *, int));
367c478bd9Sstevel@tonic-gate static bool	tls_safe_f __P((char *, long, bool));
377c478bd9Sstevel@tonic-gate static int	tls_verify_log __P((int, X509_STORE_CTX *, char *));
387c478bd9Sstevel@tonic-gate 
397c478bd9Sstevel@tonic-gate # if !NO_DH
40300fdee2SAndy Fiddaman #  include <openssl/dh.h>
41300fdee2SAndy Fiddaman #  include <openssl/dsa.h>
42300fdee2SAndy Fiddaman 
43*553e44ceSAndrew Stormont #if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
44300fdee2SAndy Fiddaman 
45300fdee2SAndy Fiddaman /*
46300fdee2SAndy Fiddaman  * This compatibility function is taken from
47300fdee2SAndy Fiddaman  * https://wiki.openssl.org/index.php/OpenSSL_1.1.0_Changes
48300fdee2SAndy Fiddaman  *     #Adding_forward-compatible_code_to_older_versions
49300fdee2SAndy Fiddaman  */
50300fdee2SAndy Fiddaman 
51300fdee2SAndy Fiddaman #define DH_set0_pqg __DH_set0_pqg
__DH_set0_pqg(DH * dh,BIGNUM * p,BIGNUM * q,BIGNUM * g)52300fdee2SAndy Fiddaman int __DH_set0_pqg(DH *dh, BIGNUM *p, BIGNUM *q, BIGNUM *g)
53300fdee2SAndy Fiddaman {
54300fdee2SAndy Fiddaman     /* If the fields p and g in d are NULL, the corresponding input
55300fdee2SAndy Fiddaman      * parameters MUST be non-NULL.  q may remain NULL.
56300fdee2SAndy Fiddaman      */
57300fdee2SAndy Fiddaman     if ((dh->p == NULL && p == NULL)
58300fdee2SAndy Fiddaman         || (dh->g == NULL && g == NULL))
59300fdee2SAndy Fiddaman         return 0;
60300fdee2SAndy Fiddaman 
61300fdee2SAndy Fiddaman     if (p != NULL) {
62300fdee2SAndy Fiddaman         BN_free(dh->p);
63300fdee2SAndy Fiddaman         dh->p = p;
64300fdee2SAndy Fiddaman     }
65300fdee2SAndy Fiddaman     if (q != NULL) {
66300fdee2SAndy Fiddaman         BN_free(dh->q);
67300fdee2SAndy Fiddaman         dh->q = q;
68300fdee2SAndy Fiddaman     }
69300fdee2SAndy Fiddaman     if (g != NULL) {
70300fdee2SAndy Fiddaman         BN_free(dh->g);
71300fdee2SAndy Fiddaman         dh->g = g;
72300fdee2SAndy Fiddaman     }
73300fdee2SAndy Fiddaman 
74300fdee2SAndy Fiddaman     if (q != NULL) {
75300fdee2SAndy Fiddaman         dh->length = BN_num_bits(q);
76300fdee2SAndy Fiddaman     }
77300fdee2SAndy Fiddaman 
78300fdee2SAndy Fiddaman     return 1;
79300fdee2SAndy Fiddaman }
80300fdee2SAndy Fiddaman #endif
81300fdee2SAndy Fiddaman 
827c478bd9Sstevel@tonic-gate static DH *get_dh512 __P((void));
837c478bd9Sstevel@tonic-gate 
847c478bd9Sstevel@tonic-gate static unsigned char dh512_p[] =
857c478bd9Sstevel@tonic-gate {
867c478bd9Sstevel@tonic-gate 	0xDA,0x58,0x3C,0x16,0xD9,0x85,0x22,0x89,0xD0,0xE4,0xAF,0x75,
877c478bd9Sstevel@tonic-gate 	0x6F,0x4C,0xCA,0x92,0xDD,0x4B,0xE5,0x33,0xB8,0x04,0xFB,0x0F,
887c478bd9Sstevel@tonic-gate 	0xED,0x94,0xEF,0x9C,0x8A,0x44,0x03,0xED,0x57,0x46,0x50,0xD3,
897c478bd9Sstevel@tonic-gate 	0x69,0x99,0xDB,0x29,0xD7,0x76,0x27,0x6B,0xA2,0xD3,0xD4,0x12,
907c478bd9Sstevel@tonic-gate 	0xE2,0x18,0xF4,0xDD,0x1E,0x08,0x4C,0xF6,0xD8,0x00,0x3E,0x7C,
917c478bd9Sstevel@tonic-gate 	0x47,0x74,0xE8,0x33
927c478bd9Sstevel@tonic-gate };
937c478bd9Sstevel@tonic-gate static unsigned char dh512_g[] =
947c478bd9Sstevel@tonic-gate {
957c478bd9Sstevel@tonic-gate 	0x02
967c478bd9Sstevel@tonic-gate };
977c478bd9Sstevel@tonic-gate 
987c478bd9Sstevel@tonic-gate static DH *
get_dh512()997c478bd9Sstevel@tonic-gate get_dh512()
1007c478bd9Sstevel@tonic-gate {
1017c478bd9Sstevel@tonic-gate 	DH *dh = NULL;
102300fdee2SAndy Fiddaman 	BIGNUM *p, *g;
1037c478bd9Sstevel@tonic-gate 
1047c478bd9Sstevel@tonic-gate 	if ((dh = DH_new()) == NULL)
1057c478bd9Sstevel@tonic-gate 		return NULL;
106300fdee2SAndy Fiddaman 	p = BN_bin2bn(dh512_p, sizeof(dh512_p), NULL);
107300fdee2SAndy Fiddaman 	g = BN_bin2bn(dh512_g, sizeof(dh512_g), NULL);
108300fdee2SAndy Fiddaman 	if ((p == NULL) || (g == NULL)) {
109300fdee2SAndy Fiddaman 		BN_free(p);
110300fdee2SAndy Fiddaman 		BN_free(g);
111300fdee2SAndy Fiddaman 		DH_free(dh);
1127c478bd9Sstevel@tonic-gate 		return NULL;
113300fdee2SAndy Fiddaman 	}
114300fdee2SAndy Fiddaman 
115300fdee2SAndy Fiddaman 	DH_set0_pqg(dh, p, NULL, g);
116300fdee2SAndy Fiddaman 
1177c478bd9Sstevel@tonic-gate 	return dh;
1187c478bd9Sstevel@tonic-gate }
1197c478bd9Sstevel@tonic-gate # endif /* !NO_DH */
1207c478bd9Sstevel@tonic-gate 
1217c478bd9Sstevel@tonic-gate 
1227c478bd9Sstevel@tonic-gate /*
1237c478bd9Sstevel@tonic-gate **  TLS_RAND_INIT -- initialize STARTTLS random generator
1247c478bd9Sstevel@tonic-gate **
1257c478bd9Sstevel@tonic-gate **	Parameters:
1267c478bd9Sstevel@tonic-gate **		randfile -- name of file with random data
1277c478bd9Sstevel@tonic-gate **		logl -- loglevel
1287c478bd9Sstevel@tonic-gate **
1297c478bd9Sstevel@tonic-gate **	Returns:
1307c478bd9Sstevel@tonic-gate **		success/failure
1317c478bd9Sstevel@tonic-gate **
1327c478bd9Sstevel@tonic-gate **	Side Effects:
1337c478bd9Sstevel@tonic-gate **		initializes PRNG for tls library.
1347c478bd9Sstevel@tonic-gate */
1357c478bd9Sstevel@tonic-gate 
1367c478bd9Sstevel@tonic-gate # define MIN_RAND_BYTES	128	/* 1024 bits */
1377c478bd9Sstevel@tonic-gate 
1387c478bd9Sstevel@tonic-gate # define RF_OK		0	/* randfile OK */
1397c478bd9Sstevel@tonic-gate # define RF_MISS	1	/* randfile == NULL || *randfile == '\0' */
1407c478bd9Sstevel@tonic-gate # define RF_UNKNOWN	2	/* unknown prefix for randfile */
1417c478bd9Sstevel@tonic-gate 
1427c478bd9Sstevel@tonic-gate # define RI_NONE	0	/* no init yet */
1437c478bd9Sstevel@tonic-gate # define RI_SUCCESS	1	/* init was successful */
1447c478bd9Sstevel@tonic-gate # define RI_FAIL	2	/* init failed */
1457c478bd9Sstevel@tonic-gate 
1467c478bd9Sstevel@tonic-gate static bool	tls_rand_init __P((char *, int));
1477c478bd9Sstevel@tonic-gate 
1487c478bd9Sstevel@tonic-gate static bool
tls_rand_init(randfile,logl)1497c478bd9Sstevel@tonic-gate tls_rand_init(randfile, logl)
1507c478bd9Sstevel@tonic-gate 	char *randfile;
1517c478bd9Sstevel@tonic-gate 	int logl;
1527c478bd9Sstevel@tonic-gate {
1537c478bd9Sstevel@tonic-gate # ifndef HASURANDOMDEV
1547c478bd9Sstevel@tonic-gate 	/* not required if /dev/urandom exists, OpenSSL does it internally */
1557c478bd9Sstevel@tonic-gate 
1567c478bd9Sstevel@tonic-gate 	bool ok;
1577c478bd9Sstevel@tonic-gate 	int randdef;
1587c478bd9Sstevel@tonic-gate 	static int done = RI_NONE;
1597c478bd9Sstevel@tonic-gate 
1607c478bd9Sstevel@tonic-gate 	/*
1617c478bd9Sstevel@tonic-gate 	**  initialize PRNG
1627c478bd9Sstevel@tonic-gate 	*/
1637c478bd9Sstevel@tonic-gate 
1647c478bd9Sstevel@tonic-gate 	/* did we try this before? if yes: return old value */
1657c478bd9Sstevel@tonic-gate 	if (done != RI_NONE)
1667c478bd9Sstevel@tonic-gate 		return done == RI_SUCCESS;
1677c478bd9Sstevel@tonic-gate 
1687c478bd9Sstevel@tonic-gate 	/* set default values */
1697c478bd9Sstevel@tonic-gate 	ok = false;
1707c478bd9Sstevel@tonic-gate 	done = RI_FAIL;
1717c478bd9Sstevel@tonic-gate 	randdef = (randfile == NULL || *randfile == '\0') ? RF_MISS : RF_OK;
1727c478bd9Sstevel@tonic-gate #   if EGD
1737c478bd9Sstevel@tonic-gate 	if (randdef == RF_OK && sm_strncasecmp(randfile, "egd:", 4) == 0)
1747c478bd9Sstevel@tonic-gate 	{
1757c478bd9Sstevel@tonic-gate 		randfile += 4;
1767c478bd9Sstevel@tonic-gate 		if (RAND_egd(randfile) < 0)
1777c478bd9Sstevel@tonic-gate 		{
1787c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
1797c478bd9Sstevel@tonic-gate 				  "STARTTLS: RAND_egd(%s) failed: random number generator not seeded",
1807c478bd9Sstevel@tonic-gate 				   randfile);
1817c478bd9Sstevel@tonic-gate 		}
1827c478bd9Sstevel@tonic-gate 		else
1837c478bd9Sstevel@tonic-gate 			ok = true;
1847c478bd9Sstevel@tonic-gate 	}
1857c478bd9Sstevel@tonic-gate 	else
1867c478bd9Sstevel@tonic-gate #   endif /* EGD */
1877c478bd9Sstevel@tonic-gate 	if (randdef == RF_OK && sm_strncasecmp(randfile, "file:", 5) == 0)
1887c478bd9Sstevel@tonic-gate 	{
1897c478bd9Sstevel@tonic-gate 		int fd;
1907c478bd9Sstevel@tonic-gate 		long sff;
1917c478bd9Sstevel@tonic-gate 		struct stat st;
1927c478bd9Sstevel@tonic-gate 
1937c478bd9Sstevel@tonic-gate 		randfile += 5;
1947c478bd9Sstevel@tonic-gate 		sff = SFF_SAFEDIRPATH | SFF_NOWLINK
1957c478bd9Sstevel@tonic-gate 		      | SFF_NOGWFILES | SFF_NOWWFILES
1967c478bd9Sstevel@tonic-gate 		      | SFF_NOGRFILES | SFF_NOWRFILES
1977c478bd9Sstevel@tonic-gate 		      | SFF_MUSTOWN | SFF_ROOTOK | SFF_OPENASROOT;
1987c478bd9Sstevel@tonic-gate 		if (DontLockReadFiles)
1997c478bd9Sstevel@tonic-gate 			sff |= SFF_NOLOCK;
2007c478bd9Sstevel@tonic-gate 		if ((fd = safeopen(randfile, O_RDONLY, 0, sff)) >= 0)
2017c478bd9Sstevel@tonic-gate 		{
2027c478bd9Sstevel@tonic-gate 			if (fstat(fd, &st) < 0)
2037c478bd9Sstevel@tonic-gate 			{
2047c478bd9Sstevel@tonic-gate 				if (LogLevel > logl)
2057c478bd9Sstevel@tonic-gate 					sm_syslog(LOG_ERR, NOQID,
2067c478bd9Sstevel@tonic-gate 						  "STARTTLS: can't fstat(%s)",
2077c478bd9Sstevel@tonic-gate 						  randfile);
2087c478bd9Sstevel@tonic-gate 			}
2097c478bd9Sstevel@tonic-gate 			else
2107c478bd9Sstevel@tonic-gate 			{
2117c478bd9Sstevel@tonic-gate 				bool use, problem;
2127c478bd9Sstevel@tonic-gate 
2137c478bd9Sstevel@tonic-gate 				use = true;
2147c478bd9Sstevel@tonic-gate 				problem = false;
2157c478bd9Sstevel@tonic-gate 
2167c478bd9Sstevel@tonic-gate 				/* max. age of file: 10 minutes */
2177c478bd9Sstevel@tonic-gate 				if (st.st_mtime + 600 < curtime())
2187c478bd9Sstevel@tonic-gate 				{
2197c478bd9Sstevel@tonic-gate 					use = bitnset(DBS_INSUFFICIENTENTROPY,
2207c478bd9Sstevel@tonic-gate 						      DontBlameSendmail);
2217c478bd9Sstevel@tonic-gate 					problem = true;
2227c478bd9Sstevel@tonic-gate 					if (LogLevel > logl)
2237c478bd9Sstevel@tonic-gate 						sm_syslog(LOG_ERR, NOQID,
2247c478bd9Sstevel@tonic-gate 							  "STARTTLS: RandFile %s too old: %s",
2257c478bd9Sstevel@tonic-gate 							  randfile,
2267c478bd9Sstevel@tonic-gate 							  use ? "unsafe" :
2277c478bd9Sstevel@tonic-gate 								"unusable");
2287c478bd9Sstevel@tonic-gate 				}
2297c478bd9Sstevel@tonic-gate 				if (use && st.st_size < MIN_RAND_BYTES)
2307c478bd9Sstevel@tonic-gate 				{
2317c478bd9Sstevel@tonic-gate 					use = bitnset(DBS_INSUFFICIENTENTROPY,
2327c478bd9Sstevel@tonic-gate 						      DontBlameSendmail);
2337c478bd9Sstevel@tonic-gate 					problem = true;
2347c478bd9Sstevel@tonic-gate 					if (LogLevel > logl)
2357c478bd9Sstevel@tonic-gate 						sm_syslog(LOG_ERR, NOQID,
2367c478bd9Sstevel@tonic-gate 							  "STARTTLS: size(%s) < %d: %s",
2377c478bd9Sstevel@tonic-gate 							  randfile,
2387c478bd9Sstevel@tonic-gate 							  MIN_RAND_BYTES,
2397c478bd9Sstevel@tonic-gate 							  use ? "unsafe" :
2407c478bd9Sstevel@tonic-gate 								"unusable");
2417c478bd9Sstevel@tonic-gate 				}
2427c478bd9Sstevel@tonic-gate 				if (use)
2437c478bd9Sstevel@tonic-gate 					ok = RAND_load_file(randfile, -1) >=
2447c478bd9Sstevel@tonic-gate 					     MIN_RAND_BYTES;
2457c478bd9Sstevel@tonic-gate 				if (use && !ok)
2467c478bd9Sstevel@tonic-gate 				{
2477c478bd9Sstevel@tonic-gate 					if (LogLevel > logl)
2487c478bd9Sstevel@tonic-gate 						sm_syslog(LOG_WARNING, NOQID,
2497c478bd9Sstevel@tonic-gate 							  "STARTTLS: RAND_load_file(%s) failed: random number generator not seeded",
2507c478bd9Sstevel@tonic-gate 							  randfile);
2517c478bd9Sstevel@tonic-gate 				}
2527c478bd9Sstevel@tonic-gate 				if (problem)
2537c478bd9Sstevel@tonic-gate 					ok = false;
2547c478bd9Sstevel@tonic-gate 			}
2557c478bd9Sstevel@tonic-gate 			if (ok || bitnset(DBS_INSUFFICIENTENTROPY,
2567c478bd9Sstevel@tonic-gate 					  DontBlameSendmail))
2577c478bd9Sstevel@tonic-gate 			{
2587c478bd9Sstevel@tonic-gate 				/* add this even if fstat() failed */
259058561cbSjbeck 				RAND_seed((void *) &st, sizeof(st));
2607c478bd9Sstevel@tonic-gate 			}
2617c478bd9Sstevel@tonic-gate 			(void) close(fd);
2627c478bd9Sstevel@tonic-gate 		}
2637c478bd9Sstevel@tonic-gate 		else
2647c478bd9Sstevel@tonic-gate 		{
2657c478bd9Sstevel@tonic-gate 			if (LogLevel > logl)
2667c478bd9Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
2677c478bd9Sstevel@tonic-gate 					  "STARTTLS: Warning: safeopen(%s) failed",
2687c478bd9Sstevel@tonic-gate 					  randfile);
2697c478bd9Sstevel@tonic-gate 		}
2707c478bd9Sstevel@tonic-gate 	}
2717c478bd9Sstevel@tonic-gate 	else if (randdef == RF_OK)
2727c478bd9Sstevel@tonic-gate 	{
2737c478bd9Sstevel@tonic-gate 		if (LogLevel > logl)
2747c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
2757c478bd9Sstevel@tonic-gate 				  "STARTTLS: Error: no proper random file definition %s",
2767c478bd9Sstevel@tonic-gate 				  randfile);
2777c478bd9Sstevel@tonic-gate 		randdef = RF_UNKNOWN;
2787c478bd9Sstevel@tonic-gate 	}
2797c478bd9Sstevel@tonic-gate 	if (randdef == RF_MISS)
2807c478bd9Sstevel@tonic-gate 	{
2817c478bd9Sstevel@tonic-gate 		if (LogLevel > logl)
2827c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
2837c478bd9Sstevel@tonic-gate 				  "STARTTLS: Error: missing random file definition");
2847c478bd9Sstevel@tonic-gate 	}
2857c478bd9Sstevel@tonic-gate 	if (!ok && bitnset(DBS_INSUFFICIENTENTROPY, DontBlameSendmail))
2867c478bd9Sstevel@tonic-gate 	{
2877c478bd9Sstevel@tonic-gate 		int i;
2887c478bd9Sstevel@tonic-gate 		long r;
2897c478bd9Sstevel@tonic-gate 		unsigned char buf[MIN_RAND_BYTES];
2907c478bd9Sstevel@tonic-gate 
2917c478bd9Sstevel@tonic-gate 		/* assert((MIN_RAND_BYTES % sizeof(long)) == 0); */
2927c478bd9Sstevel@tonic-gate 		for (i = 0; i <= sizeof(buf) - sizeof(long); i += sizeof(long))
2937c478bd9Sstevel@tonic-gate 		{
2947c478bd9Sstevel@tonic-gate 			r = get_random();
2957c478bd9Sstevel@tonic-gate 			(void) memcpy(buf + i, (void *) &r, sizeof(long));
2967c478bd9Sstevel@tonic-gate 		}
297058561cbSjbeck 		RAND_seed(buf, sizeof(buf));
2987c478bd9Sstevel@tonic-gate 		if (LogLevel > logl)
2997c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
3007c478bd9Sstevel@tonic-gate 				  "STARTTLS: Warning: random number generator not properly seeded");
3017c478bd9Sstevel@tonic-gate 		ok = true;
3027c478bd9Sstevel@tonic-gate 	}
3037c478bd9Sstevel@tonic-gate 	done = ok ? RI_SUCCESS : RI_FAIL;
3047c478bd9Sstevel@tonic-gate 	return ok;
3057c478bd9Sstevel@tonic-gate # else /* ! HASURANDOMDEV */
3067c478bd9Sstevel@tonic-gate 	return true;
3077c478bd9Sstevel@tonic-gate # endif /* ! HASURANDOMDEV */
3087c478bd9Sstevel@tonic-gate }
3097c478bd9Sstevel@tonic-gate /*
3107c478bd9Sstevel@tonic-gate **  INIT_TLS_LIBRARY -- Calls functions which setup TLS library for global use.
3117c478bd9Sstevel@tonic-gate **
3127c478bd9Sstevel@tonic-gate **	Parameters:
3137c478bd9Sstevel@tonic-gate **		none.
3147c478bd9Sstevel@tonic-gate **
3157c478bd9Sstevel@tonic-gate **	Returns:
3167c478bd9Sstevel@tonic-gate **		succeeded?
3177c478bd9Sstevel@tonic-gate */
3187c478bd9Sstevel@tonic-gate 
3197c478bd9Sstevel@tonic-gate bool
init_tls_library()3207c478bd9Sstevel@tonic-gate init_tls_library()
3217c478bd9Sstevel@tonic-gate {
3227c478bd9Sstevel@tonic-gate 	/* basic TLS initialization, ignore result for now */
323*553e44ceSAndrew Stormont #if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
324300fdee2SAndy Fiddaman 	/* No longer available (nor necessary) in OpenSSL 1.1 */
3257c478bd9Sstevel@tonic-gate 	SSL_library_init();
3267c478bd9Sstevel@tonic-gate 	SSL_load_error_strings();
327300fdee2SAndy Fiddaman #endif
3287c478bd9Sstevel@tonic-gate # if 0
3297c478bd9Sstevel@tonic-gate 	/* this is currently a macro for SSL_library_init */
3307c478bd9Sstevel@tonic-gate 	SSLeay_add_ssl_algorithms();
3317c478bd9Sstevel@tonic-gate # endif /* 0 */
3327c478bd9Sstevel@tonic-gate 
3337c478bd9Sstevel@tonic-gate 	return tls_rand_init(RandFile, 7);
3347c478bd9Sstevel@tonic-gate }
3357c478bd9Sstevel@tonic-gate /*
3367c478bd9Sstevel@tonic-gate **  TLS_SET_VERIFY -- request client certificate?
3377c478bd9Sstevel@tonic-gate **
3387c478bd9Sstevel@tonic-gate **	Parameters:
3397c478bd9Sstevel@tonic-gate **		ctx -- TLS context
3407c478bd9Sstevel@tonic-gate **		ssl -- TLS structure
3417c478bd9Sstevel@tonic-gate **		vrfy -- require certificate?
3427c478bd9Sstevel@tonic-gate **
3437c478bd9Sstevel@tonic-gate **	Returns:
3447c478bd9Sstevel@tonic-gate **		none.
3457c478bd9Sstevel@tonic-gate **
3467c478bd9Sstevel@tonic-gate **	Side Effects:
3477c478bd9Sstevel@tonic-gate **		Sets verification state for TLS
3487c478bd9Sstevel@tonic-gate **
3497c478bd9Sstevel@tonic-gate # if TLS_VRFY_PER_CTX
3507c478bd9Sstevel@tonic-gate **	Notice:
3517c478bd9Sstevel@tonic-gate **		This is per TLS context, not per TLS structure;
3527c478bd9Sstevel@tonic-gate **		the former is global, the latter per connection.
3537c478bd9Sstevel@tonic-gate **		It would be nice to do this per connection, but this
3547c478bd9Sstevel@tonic-gate **		doesn't work in the current TLS libraries :-(
3557c478bd9Sstevel@tonic-gate # endif * TLS_VRFY_PER_CTX *
3567c478bd9Sstevel@tonic-gate */
3577c478bd9Sstevel@tonic-gate 
3587c478bd9Sstevel@tonic-gate void
tls_set_verify(ctx,ssl,vrfy)3597c478bd9Sstevel@tonic-gate tls_set_verify(ctx, ssl, vrfy)
3607c478bd9Sstevel@tonic-gate 	SSL_CTX *ctx;
3617c478bd9Sstevel@tonic-gate 	SSL *ssl;
3627c478bd9Sstevel@tonic-gate 	bool vrfy;
3637c478bd9Sstevel@tonic-gate {
3647c478bd9Sstevel@tonic-gate # if !TLS_VRFY_PER_CTX
3657c478bd9Sstevel@tonic-gate 	SSL_set_verify(ssl, vrfy ? SSL_VERIFY_PEER : SSL_VERIFY_NONE, NULL);
3667c478bd9Sstevel@tonic-gate # else /* !TLS_VRFY_PER_CTX */
3677c478bd9Sstevel@tonic-gate 	SSL_CTX_set_verify(ctx, vrfy ? SSL_VERIFY_PEER : SSL_VERIFY_NONE,
3687c478bd9Sstevel@tonic-gate 			NULL);
3697c478bd9Sstevel@tonic-gate # endif /* !TLS_VRFY_PER_CTX */
3707c478bd9Sstevel@tonic-gate }
3717c478bd9Sstevel@tonic-gate 
3727c478bd9Sstevel@tonic-gate /*
3737c478bd9Sstevel@tonic-gate **  status in initialization
3747c478bd9Sstevel@tonic-gate **  these flags keep track of the status of the initialization
3757c478bd9Sstevel@tonic-gate **  i.e., whether a file exists (_EX) and whether it can be used (_OK)
3767c478bd9Sstevel@tonic-gate **  [due to permissions]
3777c478bd9Sstevel@tonic-gate */
3787c478bd9Sstevel@tonic-gate 
3797c478bd9Sstevel@tonic-gate # define TLS_S_NONE	0x00000000	/* none yet */
3807c478bd9Sstevel@tonic-gate # define TLS_S_CERT_EX	0x00000001	/* cert file exists */
3817c478bd9Sstevel@tonic-gate # define TLS_S_CERT_OK	0x00000002	/* cert file is ok */
3827c478bd9Sstevel@tonic-gate # define TLS_S_KEY_EX	0x00000004	/* key file exists */
3837c478bd9Sstevel@tonic-gate # define TLS_S_KEY_OK	0x00000008	/* key file is ok */
3847c478bd9Sstevel@tonic-gate # define TLS_S_CERTP_EX	0x00000010	/* CA cert path exists */
3857c478bd9Sstevel@tonic-gate # define TLS_S_CERTP_OK	0x00000020	/* CA cert path is ok */
3867c478bd9Sstevel@tonic-gate # define TLS_S_CERTF_EX	0x00000040	/* CA cert file exists */
3877c478bd9Sstevel@tonic-gate # define TLS_S_CERTF_OK	0x00000080	/* CA cert file is ok */
3887c478bd9Sstevel@tonic-gate # define TLS_S_CRLF_EX	0x00000100	/* CRL file exists */
3897c478bd9Sstevel@tonic-gate # define TLS_S_CRLF_OK	0x00000200	/* CRL file is ok */
3907c478bd9Sstevel@tonic-gate 
3917c478bd9Sstevel@tonic-gate # if _FFR_TLS_1
3927c478bd9Sstevel@tonic-gate #  define TLS_S_CERT2_EX	0x00001000	/* 2nd cert file exists */
3937c478bd9Sstevel@tonic-gate #  define TLS_S_CERT2_OK	0x00002000	/* 2nd cert file is ok */
3947c478bd9Sstevel@tonic-gate #  define TLS_S_KEY2_EX	0x00004000	/* 2nd key file exists */
3957c478bd9Sstevel@tonic-gate #  define TLS_S_KEY2_OK	0x00008000	/* 2nd key file is ok */
3967c478bd9Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
3977c478bd9Sstevel@tonic-gate 
3987c478bd9Sstevel@tonic-gate # define TLS_S_DH_OK	0x00200000	/* DH cert is ok */
3997c478bd9Sstevel@tonic-gate # define TLS_S_DHPAR_EX	0x00400000	/* DH param file exists */
4007c478bd9Sstevel@tonic-gate # define TLS_S_DHPAR_OK	0x00800000	/* DH param file is ok to use */
4017c478bd9Sstevel@tonic-gate 
4027c478bd9Sstevel@tonic-gate /* Type of variable */
4037c478bd9Sstevel@tonic-gate # define TLS_T_OTHER	0
4047c478bd9Sstevel@tonic-gate # define TLS_T_SRV	1
4057c478bd9Sstevel@tonic-gate # define TLS_T_CLT	2
4067c478bd9Sstevel@tonic-gate 
4077c478bd9Sstevel@tonic-gate /*
4087c478bd9Sstevel@tonic-gate **  TLS_OK_F -- can var be an absolute filename?
4097c478bd9Sstevel@tonic-gate **
4107c478bd9Sstevel@tonic-gate **	Parameters:
4117c478bd9Sstevel@tonic-gate **		var -- filename
4127c478bd9Sstevel@tonic-gate **		fn -- what is the filename used for?
4137c478bd9Sstevel@tonic-gate **		type -- type of variable
4147c478bd9Sstevel@tonic-gate **
4157c478bd9Sstevel@tonic-gate **	Returns:
4167c478bd9Sstevel@tonic-gate **		ok?
4177c478bd9Sstevel@tonic-gate */
4187c478bd9Sstevel@tonic-gate 
4197c478bd9Sstevel@tonic-gate static bool
tls_ok_f(var,fn,type)4207c478bd9Sstevel@tonic-gate tls_ok_f(var, fn, type)
4217c478bd9Sstevel@tonic-gate 	char *var;
4227c478bd9Sstevel@tonic-gate 	char *fn;
4237c478bd9Sstevel@tonic-gate 	int type;
4247c478bd9Sstevel@tonic-gate {
4257c478bd9Sstevel@tonic-gate 	/* must be absolute pathname */
4267c478bd9Sstevel@tonic-gate 	if (var != NULL && *var == '/')
4277c478bd9Sstevel@tonic-gate 		return true;
4287c478bd9Sstevel@tonic-gate 	if (LogLevel > 12)
4297c478bd9Sstevel@tonic-gate 		sm_syslog(LOG_WARNING, NOQID, "STARTTLS: %s%s missing",
4307c478bd9Sstevel@tonic-gate 			  type == TLS_T_SRV ? "Server" :
4317c478bd9Sstevel@tonic-gate 			  (type == TLS_T_CLT ? "Client" : ""), fn);
4327c478bd9Sstevel@tonic-gate 	return false;
4337c478bd9Sstevel@tonic-gate }
4347c478bd9Sstevel@tonic-gate /*
4357c478bd9Sstevel@tonic-gate **  TLS_SAFE_F -- is a file safe to use?
4367c478bd9Sstevel@tonic-gate **
4377c478bd9Sstevel@tonic-gate **	Parameters:
4387c478bd9Sstevel@tonic-gate **		var -- filename
4397c478bd9Sstevel@tonic-gate **		sff -- flags for safefile()
4407c478bd9Sstevel@tonic-gate **		srv -- server side?
4417c478bd9Sstevel@tonic-gate **
4427c478bd9Sstevel@tonic-gate **	Returns:
4437c478bd9Sstevel@tonic-gate **		ok?
4447c478bd9Sstevel@tonic-gate */
4457c478bd9Sstevel@tonic-gate 
4467c478bd9Sstevel@tonic-gate static bool
tls_safe_f(var,sff,srv)4477c478bd9Sstevel@tonic-gate tls_safe_f(var, sff, srv)
4487c478bd9Sstevel@tonic-gate 	char *var;
4497c478bd9Sstevel@tonic-gate 	long sff;
4507c478bd9Sstevel@tonic-gate 	bool srv;
4517c478bd9Sstevel@tonic-gate {
4527c478bd9Sstevel@tonic-gate 	int ret;
4537c478bd9Sstevel@tonic-gate 
4547c478bd9Sstevel@tonic-gate 	if ((ret = safefile(var, RunAsUid, RunAsGid, RunAsUserName, sff,
4557c478bd9Sstevel@tonic-gate 			    S_IRUSR, NULL)) == 0)
4567c478bd9Sstevel@tonic-gate 		return true;
4577c478bd9Sstevel@tonic-gate 	if (LogLevel > 7)
4587c478bd9Sstevel@tonic-gate 		sm_syslog(LOG_WARNING, NOQID, "STARTTLS=%s: file %s unsafe: %s",
4597c478bd9Sstevel@tonic-gate 			  srv ? "server" : "client", var, sm_errstring(ret));
4607c478bd9Sstevel@tonic-gate 	return false;
4617c478bd9Sstevel@tonic-gate }
4627c478bd9Sstevel@tonic-gate 
4637c478bd9Sstevel@tonic-gate /*
4647c478bd9Sstevel@tonic-gate **  TLS_OK_F -- macro to simplify calls to tls_ok_f
4657c478bd9Sstevel@tonic-gate **
4667c478bd9Sstevel@tonic-gate **	Parameters:
4677c478bd9Sstevel@tonic-gate **		var -- filename
4687c478bd9Sstevel@tonic-gate **		fn -- what is the filename used for?
4697c478bd9Sstevel@tonic-gate **		req -- is the file required?
4707c478bd9Sstevel@tonic-gate **		st -- status bit to set if ok
4717c478bd9Sstevel@tonic-gate **		type -- type of variable
4727c478bd9Sstevel@tonic-gate **
4737c478bd9Sstevel@tonic-gate **	Side Effects:
4747c478bd9Sstevel@tonic-gate **		uses r, ok; may change ok and status.
4757c478bd9Sstevel@tonic-gate **
4767c478bd9Sstevel@tonic-gate */
4777c478bd9Sstevel@tonic-gate 
4787c478bd9Sstevel@tonic-gate # define TLS_OK_F(var, fn, req, st, type) if (ok) \
4797c478bd9Sstevel@tonic-gate 	{ \
4807c478bd9Sstevel@tonic-gate 		r = tls_ok_f(var, fn, type); \
4817c478bd9Sstevel@tonic-gate 		if (r) \
4827c478bd9Sstevel@tonic-gate 			status |= st; \
4837c478bd9Sstevel@tonic-gate 		else if (req) \
4847c478bd9Sstevel@tonic-gate 			ok = false; \
4857c478bd9Sstevel@tonic-gate 	}
4867c478bd9Sstevel@tonic-gate 
4877c478bd9Sstevel@tonic-gate /*
4887c478bd9Sstevel@tonic-gate **  TLS_UNR -- macro to return whether a file should be unreadable
4897c478bd9Sstevel@tonic-gate **
4907c478bd9Sstevel@tonic-gate **	Parameters:
4917c478bd9Sstevel@tonic-gate **		bit -- flag to test
4927c478bd9Sstevel@tonic-gate **		req -- flags
4937c478bd9Sstevel@tonic-gate **
4947c478bd9Sstevel@tonic-gate **	Returns:
4957c478bd9Sstevel@tonic-gate **		0/SFF_NORFILES
4967c478bd9Sstevel@tonic-gate */
4977c478bd9Sstevel@tonic-gate # define TLS_UNR(bit, req)	(bitset(bit, req) ? SFF_NORFILES : 0)
4987c478bd9Sstevel@tonic-gate # define TLS_OUNR(bit, req)	(bitset(bit, req) ? SFF_NOWRFILES : 0)
4997c478bd9Sstevel@tonic-gate # define TLS_KEYSFF(req)	\
5007c478bd9Sstevel@tonic-gate 	(bitnset(DBS_GROUPREADABLEKEYFILE, DontBlameSendmail) ?	\
5017c478bd9Sstevel@tonic-gate 		TLS_OUNR(TLS_I_KEY_OUNR, req) :			\
5027c478bd9Sstevel@tonic-gate 		TLS_UNR(TLS_I_KEY_UNR, req))
5037c478bd9Sstevel@tonic-gate 
5047c478bd9Sstevel@tonic-gate /*
5057c478bd9Sstevel@tonic-gate **  TLS_SAFE_F -- macro to simplify calls to tls_safe_f
5067c478bd9Sstevel@tonic-gate **
5077c478bd9Sstevel@tonic-gate **	Parameters:
5087c478bd9Sstevel@tonic-gate **		var -- filename
5097c478bd9Sstevel@tonic-gate **		sff -- flags for safefile()
5107c478bd9Sstevel@tonic-gate **		req -- is the file required?
5117c478bd9Sstevel@tonic-gate **		ex -- does the file exist?
5127c478bd9Sstevel@tonic-gate **		st -- status bit to set if ok
5137c478bd9Sstevel@tonic-gate **		srv -- server side?
5147c478bd9Sstevel@tonic-gate **
5157c478bd9Sstevel@tonic-gate **	Side Effects:
5167c478bd9Sstevel@tonic-gate **		uses r, ok, ex; may change ok and status.
5177c478bd9Sstevel@tonic-gate **
5187c478bd9Sstevel@tonic-gate */
5197c478bd9Sstevel@tonic-gate 
5207c478bd9Sstevel@tonic-gate # define TLS_SAFE_F(var, sff, req, ex, st, srv) if (ex && ok) \
5217c478bd9Sstevel@tonic-gate 	{ \
5227c478bd9Sstevel@tonic-gate 		r = tls_safe_f(var, sff, srv); \
5237c478bd9Sstevel@tonic-gate 		if (r) \
5247c478bd9Sstevel@tonic-gate 			status |= st;	\
5257c478bd9Sstevel@tonic-gate 		else if (req) \
5267c478bd9Sstevel@tonic-gate 			ok = false;	\
5277c478bd9Sstevel@tonic-gate 	}
5287c478bd9Sstevel@tonic-gate 
5297c478bd9Sstevel@tonic-gate /*
5307c478bd9Sstevel@tonic-gate **  INITTLS -- initialize TLS
5317c478bd9Sstevel@tonic-gate **
5327c478bd9Sstevel@tonic-gate **	Parameters:
5337c478bd9Sstevel@tonic-gate **		ctx -- pointer to context
5347c478bd9Sstevel@tonic-gate **		req -- requirements for initialization (see sendmail.h)
535e9af4bc0SJohn Beck **		options -- options
5367c478bd9Sstevel@tonic-gate **		srv -- server side?
5377c478bd9Sstevel@tonic-gate **		certfile -- filename of certificate
5387c478bd9Sstevel@tonic-gate **		keyfile -- filename of private key
5397c478bd9Sstevel@tonic-gate **		cacertpath -- path to CAs
5407c478bd9Sstevel@tonic-gate **		cacertfile -- file with CA(s)
5417c478bd9Sstevel@tonic-gate **		dhparam -- parameters for DH
5427c478bd9Sstevel@tonic-gate **
5437c478bd9Sstevel@tonic-gate **	Returns:
5447c478bd9Sstevel@tonic-gate **		succeeded?
5457c478bd9Sstevel@tonic-gate */
5467c478bd9Sstevel@tonic-gate 
547445f2479Sjbeck /*
548445f2479Sjbeck **  The session_id_context identifies the service that created a session.
549445f2479Sjbeck **  This information is used to distinguish between multiple TLS-based
550445f2479Sjbeck **  servers running on the same server. We use the name of the mail system.
551445f2479Sjbeck **  Note: the session cache is not persistent.
552445f2479Sjbeck */
553445f2479Sjbeck 
554445f2479Sjbeck static char server_session_id_context[] = "sendmail8";
555445f2479Sjbeck 
5567c478bd9Sstevel@tonic-gate bool
inittls(ctx,req,options,srv,certfile,keyfile,cacertpath,cacertfile,dhparam)557e9af4bc0SJohn Beck inittls(ctx, req, options, srv, certfile, keyfile, cacertpath, cacertfile, dhparam)
5587c478bd9Sstevel@tonic-gate 	SSL_CTX **ctx;
5597c478bd9Sstevel@tonic-gate 	unsigned long req;
560e9af4bc0SJohn Beck 	long options;
5617c478bd9Sstevel@tonic-gate 	bool srv;
5627c478bd9Sstevel@tonic-gate 	char *certfile, *keyfile, *cacertpath, *cacertfile, *dhparam;
5637c478bd9Sstevel@tonic-gate {
5647c478bd9Sstevel@tonic-gate # if !NO_DH
5657c478bd9Sstevel@tonic-gate 	static DH *dh = NULL;
5667c478bd9Sstevel@tonic-gate # endif /* !NO_DH */
5677c478bd9Sstevel@tonic-gate 	int r;
5687c478bd9Sstevel@tonic-gate 	bool ok;
569e9af4bc0SJohn Beck 	long sff, status;
5707c478bd9Sstevel@tonic-gate 	char *who;
5717c478bd9Sstevel@tonic-gate # if _FFR_TLS_1
5727c478bd9Sstevel@tonic-gate 	char *cf2, *kf2;
5737c478bd9Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
5747c478bd9Sstevel@tonic-gate #  if SM_CONF_SHM
5757c478bd9Sstevel@tonic-gate 	extern int ShmId;
5767c478bd9Sstevel@tonic-gate #  endif /* SM_CONF_SHM */
5777c478bd9Sstevel@tonic-gate 	BIO *crl_file;
5787c478bd9Sstevel@tonic-gate 	X509_CRL *crl;
5797c478bd9Sstevel@tonic-gate 	X509_STORE *store;
5807c478bd9Sstevel@tonic-gate 
5817c478bd9Sstevel@tonic-gate 	status = TLS_S_NONE;
5827c478bd9Sstevel@tonic-gate 	who = srv ? "server" : "client";
5837c478bd9Sstevel@tonic-gate 	if (ctx == NULL)
5843ee0e492Sjbeck 	{
5857c478bd9Sstevel@tonic-gate 		syserr("STARTTLS=%s, inittls: ctx == NULL", who);
5863ee0e492Sjbeck 		/* NOTREACHED */
5873ee0e492Sjbeck 		SM_ASSERT(ctx != NULL);
5883ee0e492Sjbeck 	}
5897c478bd9Sstevel@tonic-gate 
5907c478bd9Sstevel@tonic-gate 	/* already initialized? (we could re-init...) */
5917c478bd9Sstevel@tonic-gate 	if (*ctx != NULL)
5927c478bd9Sstevel@tonic-gate 		return true;
5937c478bd9Sstevel@tonic-gate 	ok = true;
5947c478bd9Sstevel@tonic-gate 
5957c478bd9Sstevel@tonic-gate # if _FFR_TLS_1
5967c478bd9Sstevel@tonic-gate 	/*
5977c478bd9Sstevel@tonic-gate 	**  look for a second filename: it must be separated by a ','
5987c478bd9Sstevel@tonic-gate 	**  no blanks allowed (they won't be skipped).
5997c478bd9Sstevel@tonic-gate 	**  we change a global variable here! this change will be undone
6007c478bd9Sstevel@tonic-gate 	**  before return from the function but only if it returns true.
6017c478bd9Sstevel@tonic-gate 	**  this isn't a problem since in a failure case this function
6027c478bd9Sstevel@tonic-gate 	**  won't be called again with the same (overwritten) values.
6037c478bd9Sstevel@tonic-gate 	**  otherwise each return must be replaced with a goto endinittls.
6047c478bd9Sstevel@tonic-gate 	*/
6057c478bd9Sstevel@tonic-gate 
6067c478bd9Sstevel@tonic-gate 	cf2 = NULL;
6077c478bd9Sstevel@tonic-gate 	kf2 = NULL;
6087c478bd9Sstevel@tonic-gate 	if (certfile != NULL && (cf2 = strchr(certfile, ',')) != NULL)
6097c478bd9Sstevel@tonic-gate 	{
6107c478bd9Sstevel@tonic-gate 		*cf2++ = '\0';
6117c478bd9Sstevel@tonic-gate 		if (keyfile != NULL && (kf2 = strchr(keyfile, ',')) != NULL)
6127c478bd9Sstevel@tonic-gate 			*kf2++ = '\0';
6137c478bd9Sstevel@tonic-gate 	}
6147c478bd9Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
6157c478bd9Sstevel@tonic-gate 
6167c478bd9Sstevel@tonic-gate 	/*
6177c478bd9Sstevel@tonic-gate 	**  Check whether files/paths are defined
6187c478bd9Sstevel@tonic-gate 	*/
6197c478bd9Sstevel@tonic-gate 
6207c478bd9Sstevel@tonic-gate 	TLS_OK_F(certfile, "CertFile", bitset(TLS_I_CERT_EX, req),
6217c478bd9Sstevel@tonic-gate 		 TLS_S_CERT_EX, srv ? TLS_T_SRV : TLS_T_CLT);
6227c478bd9Sstevel@tonic-gate 	TLS_OK_F(keyfile, "KeyFile", bitset(TLS_I_KEY_EX, req),
6237c478bd9Sstevel@tonic-gate 		 TLS_S_KEY_EX, srv ? TLS_T_SRV : TLS_T_CLT);
6247c478bd9Sstevel@tonic-gate 	TLS_OK_F(cacertpath, "CACertPath", bitset(TLS_I_CERTP_EX, req),
6257c478bd9Sstevel@tonic-gate 		 TLS_S_CERTP_EX, TLS_T_OTHER);
6267c478bd9Sstevel@tonic-gate 	TLS_OK_F(cacertfile, "CACertFile", bitset(TLS_I_CERTF_EX, req),
6277c478bd9Sstevel@tonic-gate 		 TLS_S_CERTF_EX, TLS_T_OTHER);
6287c478bd9Sstevel@tonic-gate 	TLS_OK_F(CRLFile, "CRLFile", bitset(TLS_I_CRLF_EX, req),
6297c478bd9Sstevel@tonic-gate 		 TLS_S_CRLF_EX, TLS_T_OTHER);
6307c478bd9Sstevel@tonic-gate 
6317c478bd9Sstevel@tonic-gate # if _FFR_TLS_1
6327c478bd9Sstevel@tonic-gate 	/*
6337c478bd9Sstevel@tonic-gate 	**  if the second file is specified it must exist
6347c478bd9Sstevel@tonic-gate 	**  XXX: it is possible here to define only one of those files
6357c478bd9Sstevel@tonic-gate 	*/
6367c478bd9Sstevel@tonic-gate 
6377c478bd9Sstevel@tonic-gate 	if (cf2 != NULL)
6387c478bd9Sstevel@tonic-gate 	{
6397c478bd9Sstevel@tonic-gate 		TLS_OK_F(cf2, "CertFile", bitset(TLS_I_CERT_EX, req),
6407c478bd9Sstevel@tonic-gate 			 TLS_S_CERT2_EX, srv ? TLS_T_SRV : TLS_T_CLT);
6417c478bd9Sstevel@tonic-gate 	}
6427c478bd9Sstevel@tonic-gate 	if (kf2 != NULL)
6437c478bd9Sstevel@tonic-gate 	{
6447c478bd9Sstevel@tonic-gate 		TLS_OK_F(kf2, "KeyFile", bitset(TLS_I_KEY_EX, req),
6457c478bd9Sstevel@tonic-gate 			 TLS_S_KEY2_EX, srv ? TLS_T_SRV : TLS_T_CLT);
6467c478bd9Sstevel@tonic-gate 	}
6477c478bd9Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
6487c478bd9Sstevel@tonic-gate 
6497c478bd9Sstevel@tonic-gate 	/*
6507c478bd9Sstevel@tonic-gate 	**  valid values for dhparam are (only the first char is checked)
6517c478bd9Sstevel@tonic-gate 	**  none	no parameters: don't use DH
6527c478bd9Sstevel@tonic-gate 	**  512		generate 512 bit parameters (fixed)
6537c478bd9Sstevel@tonic-gate 	**  1024	generate 1024 bit parameters
6547c478bd9Sstevel@tonic-gate 	**  /file/name	read parameters from /file/name
6557c478bd9Sstevel@tonic-gate 	**  default is: 1024 for server, 512 for client (OK? XXX)
6567c478bd9Sstevel@tonic-gate 	*/
6577c478bd9Sstevel@tonic-gate 
6587c478bd9Sstevel@tonic-gate 	if (bitset(TLS_I_TRY_DH, req))
6597c478bd9Sstevel@tonic-gate 	{
6607c478bd9Sstevel@tonic-gate 		if (dhparam != NULL)
6617c478bd9Sstevel@tonic-gate 		{
6627c478bd9Sstevel@tonic-gate 			char c = *dhparam;
6637c478bd9Sstevel@tonic-gate 
6647c478bd9Sstevel@tonic-gate 			if (c == '1')
6657c478bd9Sstevel@tonic-gate 				req |= TLS_I_DH1024;
6667c478bd9Sstevel@tonic-gate 			else if (c == '5')
6677c478bd9Sstevel@tonic-gate 				req |= TLS_I_DH512;
6687c478bd9Sstevel@tonic-gate 			else if (c != 'n' && c != 'N' && c != '/')
6697c478bd9Sstevel@tonic-gate 			{
6707c478bd9Sstevel@tonic-gate 				if (LogLevel > 12)
6717c478bd9Sstevel@tonic-gate 					sm_syslog(LOG_WARNING, NOQID,
6727c478bd9Sstevel@tonic-gate 						  "STARTTLS=%s, error: illegal value '%s' for DHParam",
6737c478bd9Sstevel@tonic-gate 						  who, dhparam);
6747c478bd9Sstevel@tonic-gate 				dhparam = NULL;
6757c478bd9Sstevel@tonic-gate 			}
6767c478bd9Sstevel@tonic-gate 		}
6777c478bd9Sstevel@tonic-gate 		if (dhparam == NULL)
678e9af4bc0SJohn Beck 		{
6797c478bd9Sstevel@tonic-gate 			dhparam = srv ? "1" : "5";
680e9af4bc0SJohn Beck 			req |= (srv ? TLS_I_DH1024 : TLS_I_DH512);
681e9af4bc0SJohn Beck 		}
6827c478bd9Sstevel@tonic-gate 		else if (*dhparam == '/')
6837c478bd9Sstevel@tonic-gate 		{
6847c478bd9Sstevel@tonic-gate 			TLS_OK_F(dhparam, "DHParameters",
6857c478bd9Sstevel@tonic-gate 				 bitset(TLS_I_DHPAR_EX, req),
6867c478bd9Sstevel@tonic-gate 				 TLS_S_DHPAR_EX, TLS_T_OTHER);
6877c478bd9Sstevel@tonic-gate 		}
6887c478bd9Sstevel@tonic-gate 	}
6897c478bd9Sstevel@tonic-gate 	if (!ok)
6907c478bd9Sstevel@tonic-gate 		return ok;
6917c478bd9Sstevel@tonic-gate 
6927c478bd9Sstevel@tonic-gate 	/* certfile etc. must be "safe". */
6937c478bd9Sstevel@tonic-gate 	sff = SFF_REGONLY | SFF_SAFEDIRPATH | SFF_NOWLINK
6947c478bd9Sstevel@tonic-gate 	     | SFF_NOGWFILES | SFF_NOWWFILES
6957c478bd9Sstevel@tonic-gate 	     | SFF_MUSTOWN | SFF_ROOTOK | SFF_OPENASROOT;
6967c478bd9Sstevel@tonic-gate 	if (DontLockReadFiles)
6977c478bd9Sstevel@tonic-gate 		sff |= SFF_NOLOCK;
6987c478bd9Sstevel@tonic-gate 
6997c478bd9Sstevel@tonic-gate 	TLS_SAFE_F(certfile, sff | TLS_UNR(TLS_I_CERT_UNR, req),
7007c478bd9Sstevel@tonic-gate 		   bitset(TLS_I_CERT_EX, req),
7017c478bd9Sstevel@tonic-gate 		   bitset(TLS_S_CERT_EX, status), TLS_S_CERT_OK, srv);
7027c478bd9Sstevel@tonic-gate 	TLS_SAFE_F(keyfile, sff | TLS_KEYSFF(req),
7037c478bd9Sstevel@tonic-gate 		   bitset(TLS_I_KEY_EX, req),
7047c478bd9Sstevel@tonic-gate 		   bitset(TLS_S_KEY_EX, status), TLS_S_KEY_OK, srv);
7057c478bd9Sstevel@tonic-gate 	TLS_SAFE_F(cacertfile, sff | TLS_UNR(TLS_I_CERTF_UNR, req),
7067c478bd9Sstevel@tonic-gate 		   bitset(TLS_I_CERTF_EX, req),
7077c478bd9Sstevel@tonic-gate 		   bitset(TLS_S_CERTF_EX, status), TLS_S_CERTF_OK, srv);
7087c478bd9Sstevel@tonic-gate 	TLS_SAFE_F(dhparam, sff | TLS_UNR(TLS_I_DHPAR_UNR, req),
7097c478bd9Sstevel@tonic-gate 		   bitset(TLS_I_DHPAR_EX, req),
7107c478bd9Sstevel@tonic-gate 		   bitset(TLS_S_DHPAR_EX, status), TLS_S_DHPAR_OK, srv);
7117c478bd9Sstevel@tonic-gate 	TLS_SAFE_F(CRLFile, sff | TLS_UNR(TLS_I_CRLF_UNR, req),
7127c478bd9Sstevel@tonic-gate 		   bitset(TLS_I_CRLF_EX, req),
7137c478bd9Sstevel@tonic-gate 		   bitset(TLS_S_CRLF_EX, status), TLS_S_CRLF_OK, srv);
7147c478bd9Sstevel@tonic-gate 	if (!ok)
7157c478bd9Sstevel@tonic-gate 		return ok;
7167c478bd9Sstevel@tonic-gate # if _FFR_TLS_1
7177c478bd9Sstevel@tonic-gate 	if (cf2 != NULL)
7187c478bd9Sstevel@tonic-gate 	{
7197c478bd9Sstevel@tonic-gate 		TLS_SAFE_F(cf2, sff | TLS_UNR(TLS_I_CERT_UNR, req),
7207c478bd9Sstevel@tonic-gate 			   bitset(TLS_I_CERT_EX, req),
7217c478bd9Sstevel@tonic-gate 			   bitset(TLS_S_CERT2_EX, status), TLS_S_CERT2_OK, srv);
7227c478bd9Sstevel@tonic-gate 	}
7237c478bd9Sstevel@tonic-gate 	if (kf2 != NULL)
7247c478bd9Sstevel@tonic-gate 	{
7257c478bd9Sstevel@tonic-gate 		TLS_SAFE_F(kf2, sff | TLS_KEYSFF(req),
7267c478bd9Sstevel@tonic-gate 			   bitset(TLS_I_KEY_EX, req),
7277c478bd9Sstevel@tonic-gate 			   bitset(TLS_S_KEY2_EX, status), TLS_S_KEY2_OK, srv);
7287c478bd9Sstevel@tonic-gate 	}
7297c478bd9Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
7307c478bd9Sstevel@tonic-gate 
7317c478bd9Sstevel@tonic-gate 	/* create a method and a new context */
7327c478bd9Sstevel@tonic-gate 	if ((*ctx = SSL_CTX_new(srv ? SSLv23_server_method() :
7337c478bd9Sstevel@tonic-gate 				      SSLv23_client_method())) == NULL)
7347c478bd9Sstevel@tonic-gate 	{
7357c478bd9Sstevel@tonic-gate 		if (LogLevel > 7)
7367c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
7377c478bd9Sstevel@tonic-gate 				  "STARTTLS=%s, error: SSL_CTX_new(SSLv23_%s_method()) failed",
7387c478bd9Sstevel@tonic-gate 				  who, who);
7397c478bd9Sstevel@tonic-gate 		if (LogLevel > 9)
7407c478bd9Sstevel@tonic-gate 			tlslogerr(who);
7417c478bd9Sstevel@tonic-gate 		return false;
7427c478bd9Sstevel@tonic-gate 	}
7437c478bd9Sstevel@tonic-gate 
7447c478bd9Sstevel@tonic-gate 	if (CRLFile != NULL)
7457c478bd9Sstevel@tonic-gate 	{
7467c478bd9Sstevel@tonic-gate 		/* get a pointer to the current certificate validation store */
7477c478bd9Sstevel@tonic-gate 		store = SSL_CTX_get_cert_store(*ctx);	/* does not fail */
748300fdee2SAndy Fiddaman 		crl_file = BIO_new(BIO_s_file());
7497c478bd9Sstevel@tonic-gate 		if (crl_file != NULL)
7507c478bd9Sstevel@tonic-gate 		{
7517c478bd9Sstevel@tonic-gate 			if (BIO_read_filename(crl_file, CRLFile) >= 0)
7527c478bd9Sstevel@tonic-gate 			{
7537c478bd9Sstevel@tonic-gate 				crl = PEM_read_bio_X509_CRL(crl_file, NULL,
7547c478bd9Sstevel@tonic-gate 							NULL, NULL);
7557c478bd9Sstevel@tonic-gate 				BIO_free(crl_file);
7567c478bd9Sstevel@tonic-gate 				X509_STORE_add_crl(store, crl);
7577c478bd9Sstevel@tonic-gate 				X509_CRL_free(crl);
7587c478bd9Sstevel@tonic-gate 				X509_STORE_set_flags(store,
7597c478bd9Sstevel@tonic-gate 					X509_V_FLAG_CRL_CHECK|X509_V_FLAG_CRL_CHECK_ALL);
7607c478bd9Sstevel@tonic-gate 				X509_STORE_set_verify_cb_func(store,
7617c478bd9Sstevel@tonic-gate 						x509_verify_cb);
7627c478bd9Sstevel@tonic-gate 			}
7637c478bd9Sstevel@tonic-gate 			else
7647c478bd9Sstevel@tonic-gate 			{
7657c478bd9Sstevel@tonic-gate 				if (LogLevel > 9)
7667c478bd9Sstevel@tonic-gate 				{
7677c478bd9Sstevel@tonic-gate 					sm_syslog(LOG_WARNING, NOQID,
7687c478bd9Sstevel@tonic-gate 						  "STARTTLS=%s, error: PEM_read_bio_X509_CRL(%s)=failed",
7697c478bd9Sstevel@tonic-gate 						  who, CRLFile);
7707c478bd9Sstevel@tonic-gate 				}
7717c478bd9Sstevel@tonic-gate 
7727c478bd9Sstevel@tonic-gate 				/* avoid memory leaks */
7737c478bd9Sstevel@tonic-gate 				BIO_free(crl_file);
7747c478bd9Sstevel@tonic-gate 				return false;
7757c478bd9Sstevel@tonic-gate 			}
7767c478bd9Sstevel@tonic-gate 
7777c478bd9Sstevel@tonic-gate 		}
7787c478bd9Sstevel@tonic-gate 		else if (LogLevel > 9)
7797c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
7807c478bd9Sstevel@tonic-gate 				  "STARTTLS=%s, error: BIO_new=failed", who);
7817c478bd9Sstevel@tonic-gate 	}
782058561cbSjbeck 	else
783058561cbSjbeck 		store = NULL;
7847c478bd9Sstevel@tonic-gate #  if _FFR_CRLPATH
785058561cbSjbeck 	if (CRLPath != NULL && store != NULL)
7867c478bd9Sstevel@tonic-gate 	{
7877c478bd9Sstevel@tonic-gate 		X509_LOOKUP *lookup;
7887c478bd9Sstevel@tonic-gate 
7897c478bd9Sstevel@tonic-gate 		lookup = X509_STORE_add_lookup(store, X509_LOOKUP_hash_dir());
7907c478bd9Sstevel@tonic-gate 		if (lookup == NULL)
7917c478bd9Sstevel@tonic-gate 		{
7927c478bd9Sstevel@tonic-gate 			if (LogLevel > 9)
7937c478bd9Sstevel@tonic-gate 			{
7947c478bd9Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
7957c478bd9Sstevel@tonic-gate 					  "STARTTLS=%s, error: X509_STORE_add_lookup(hash)=failed",
7967c478bd9Sstevel@tonic-gate 					  who, CRLFile);
7977c478bd9Sstevel@tonic-gate 			}
7987c478bd9Sstevel@tonic-gate 			return false;
7997c478bd9Sstevel@tonic-gate 		}
8007c478bd9Sstevel@tonic-gate 		X509_LOOKUP_add_dir(lookup, CRLPath, X509_FILETYPE_PEM);
8017c478bd9Sstevel@tonic-gate 		X509_STORE_set_flags(store,
8027c478bd9Sstevel@tonic-gate 			X509_V_FLAG_CRL_CHECK|X509_V_FLAG_CRL_CHECK_ALL);
8037c478bd9Sstevel@tonic-gate 	}
8047c478bd9Sstevel@tonic-gate #  endif /* _FFR_CRLPATH */
8057c478bd9Sstevel@tonic-gate 
8067c478bd9Sstevel@tonic-gate # if TLS_NO_RSA
8077c478bd9Sstevel@tonic-gate 	/* turn off backward compatibility, required for no-rsa */
8087c478bd9Sstevel@tonic-gate 	SSL_CTX_set_options(*ctx, SSL_OP_NO_SSLv2);
8097c478bd9Sstevel@tonic-gate # endif /* TLS_NO_RSA */
8107c478bd9Sstevel@tonic-gate 
8117c478bd9Sstevel@tonic-gate 
8127c478bd9Sstevel@tonic-gate # if !TLS_NO_RSA
8137c478bd9Sstevel@tonic-gate 	/*
8147c478bd9Sstevel@tonic-gate 	**  Create a temporary RSA key
8157c478bd9Sstevel@tonic-gate 	**  XXX  Maybe we shouldn't create this always (even though it
8167c478bd9Sstevel@tonic-gate 	**  is only at startup).
8177c478bd9Sstevel@tonic-gate 	**  It is a time-consuming operation and it is not always necessary.
8187c478bd9Sstevel@tonic-gate 	**  maybe we should do it only on demand...
8197c478bd9Sstevel@tonic-gate 	*/
8207c478bd9Sstevel@tonic-gate 
8217c478bd9Sstevel@tonic-gate 	if (bitset(TLS_I_RSA_TMP, req)
8227c478bd9Sstevel@tonic-gate #   if SM_CONF_SHM
823300fdee2SAndy Fiddaman 	    && ShmId != SM_SHM_NO_ID
8247c478bd9Sstevel@tonic-gate #   else /* SM_CONF_SHM */
8257c478bd9Sstevel@tonic-gate 	    && 0	/* no shared memory: no need to generate key now */
8267c478bd9Sstevel@tonic-gate #   endif /* SM_CONF_SHM */
827300fdee2SAndy Fiddaman 	    )
8287c478bd9Sstevel@tonic-gate 	{
829300fdee2SAndy Fiddaman 		BIGNUM *e = BN_new();
830300fdee2SAndy Fiddaman 		BN_set_word(e, RSA_F4);
831300fdee2SAndy Fiddaman 
832300fdee2SAndy Fiddaman 		if ((rsa_tmp = RSA_new()) == NULL || RSA_generate_key_ex(
833300fdee2SAndy Fiddaman 		    rsa_tmp, RSA_KEYLENGTH, e, NULL) == 0)
8347c478bd9Sstevel@tonic-gate 		{
835300fdee2SAndy Fiddaman 			BN_free(e);
836300fdee2SAndy Fiddaman 			if (rsa_tmp != NULL)
837300fdee2SAndy Fiddaman 				RSA_free(rsa_tmp);
838300fdee2SAndy Fiddaman 			rsa_tmp = NULL;
839300fdee2SAndy Fiddaman 			if (LogLevel > 7)
840300fdee2SAndy Fiddaman 			{
841300fdee2SAndy Fiddaman 				sm_syslog(LOG_WARNING, NOQID,
842300fdee2SAndy Fiddaman 					  "STARTTLS=%s, error: RSA_generate_key failed",
843300fdee2SAndy Fiddaman 					  who);
844300fdee2SAndy Fiddaman 				if (LogLevel > 9)
845300fdee2SAndy Fiddaman 					tlslogerr(who);
846300fdee2SAndy Fiddaman 			}
847300fdee2SAndy Fiddaman 			return false;
8487c478bd9Sstevel@tonic-gate 		}
849300fdee2SAndy Fiddaman 		BN_free(e);
8507c478bd9Sstevel@tonic-gate 	}
8517c478bd9Sstevel@tonic-gate # endif /* !TLS_NO_RSA */
8527c478bd9Sstevel@tonic-gate 
8537c478bd9Sstevel@tonic-gate 	/*
8547c478bd9Sstevel@tonic-gate 	**  load private key
8557c478bd9Sstevel@tonic-gate 	**  XXX change this for DSA-only version
8567c478bd9Sstevel@tonic-gate 	*/
8577c478bd9Sstevel@tonic-gate 
8587c478bd9Sstevel@tonic-gate 	if (bitset(TLS_S_KEY_OK, status) &&
8597c478bd9Sstevel@tonic-gate 	    SSL_CTX_use_PrivateKey_file(*ctx, keyfile,
8607c478bd9Sstevel@tonic-gate 					 SSL_FILETYPE_PEM) <= 0)
8617c478bd9Sstevel@tonic-gate 	{
8627c478bd9Sstevel@tonic-gate 		if (LogLevel > 7)
8637c478bd9Sstevel@tonic-gate 		{
8647c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
8657c478bd9Sstevel@tonic-gate 				  "STARTTLS=%s, error: SSL_CTX_use_PrivateKey_file(%s) failed",
8667c478bd9Sstevel@tonic-gate 				  who, keyfile);
8677c478bd9Sstevel@tonic-gate 			if (LogLevel > 9)
8687c478bd9Sstevel@tonic-gate 				tlslogerr(who);
8697c478bd9Sstevel@tonic-gate 		}
8707c478bd9Sstevel@tonic-gate 		if (bitset(TLS_I_USE_KEY, req))
8717c478bd9Sstevel@tonic-gate 			return false;
8727c478bd9Sstevel@tonic-gate 	}
8737c478bd9Sstevel@tonic-gate 
8747c478bd9Sstevel@tonic-gate 	/* get the certificate file */
8757c478bd9Sstevel@tonic-gate 	if (bitset(TLS_S_CERT_OK, status) &&
8767c478bd9Sstevel@tonic-gate 	    SSL_CTX_use_certificate_file(*ctx, certfile,
8777c478bd9Sstevel@tonic-gate 					 SSL_FILETYPE_PEM) <= 0)
8787c478bd9Sstevel@tonic-gate 	{
8797c478bd9Sstevel@tonic-gate 		if (LogLevel > 7)
8807c478bd9Sstevel@tonic-gate 		{
8817c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
8827c478bd9Sstevel@tonic-gate 				  "STARTTLS=%s, error: SSL_CTX_use_certificate_file(%s) failed",
8837c478bd9Sstevel@tonic-gate 				  who, certfile);
8847c478bd9Sstevel@tonic-gate 			if (LogLevel > 9)
8857c478bd9Sstevel@tonic-gate 				tlslogerr(who);
8867c478bd9Sstevel@tonic-gate 		}
8877c478bd9Sstevel@tonic-gate 		if (bitset(TLS_I_USE_CERT, req))
8887c478bd9Sstevel@tonic-gate 			return false;
8897c478bd9Sstevel@tonic-gate 	}
8907c478bd9Sstevel@tonic-gate 
8917c478bd9Sstevel@tonic-gate 	/* check the private key */
8927c478bd9Sstevel@tonic-gate 	if (bitset(TLS_S_KEY_OK, status) &&
8937c478bd9Sstevel@tonic-gate 	    (r = SSL_CTX_check_private_key(*ctx)) <= 0)
8947c478bd9Sstevel@tonic-gate 	{
8957c478bd9Sstevel@tonic-gate 		/* Private key does not match the certificate public key */
8967c478bd9Sstevel@tonic-gate 		if (LogLevel > 5)
8977c478bd9Sstevel@tonic-gate 		{
8987c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
8997c478bd9Sstevel@tonic-gate 				  "STARTTLS=%s, error: SSL_CTX_check_private_key failed(%s): %d",
9007c478bd9Sstevel@tonic-gate 				  who, keyfile, r);
9017c478bd9Sstevel@tonic-gate 			if (LogLevel > 9)
9027c478bd9Sstevel@tonic-gate 				tlslogerr(who);
9037c478bd9Sstevel@tonic-gate 		}
9047c478bd9Sstevel@tonic-gate 		if (bitset(TLS_I_USE_KEY, req))
9057c478bd9Sstevel@tonic-gate 			return false;
9067c478bd9Sstevel@tonic-gate 	}
9077c478bd9Sstevel@tonic-gate 
9087c478bd9Sstevel@tonic-gate # if _FFR_TLS_1
9097c478bd9Sstevel@tonic-gate 	/* XXX this code is pretty much duplicated from above! */
9107c478bd9Sstevel@tonic-gate 
9117c478bd9Sstevel@tonic-gate 	/* load private key */
9127c478bd9Sstevel@tonic-gate 	if (bitset(TLS_S_KEY2_OK, status) &&
9137c478bd9Sstevel@tonic-gate 	    SSL_CTX_use_PrivateKey_file(*ctx, kf2, SSL_FILETYPE_PEM) <= 0)
9147c478bd9Sstevel@tonic-gate 	{
9157c478bd9Sstevel@tonic-gate 		if (LogLevel > 7)
9167c478bd9Sstevel@tonic-gate 		{
9177c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
9187c478bd9Sstevel@tonic-gate 				  "STARTTLS=%s, error: SSL_CTX_use_PrivateKey_file(%s) failed",
9197c478bd9Sstevel@tonic-gate 				  who, kf2);
9207c478bd9Sstevel@tonic-gate 			if (LogLevel > 9)
9217c478bd9Sstevel@tonic-gate 				tlslogerr(who);
9227c478bd9Sstevel@tonic-gate 		}
9237c478bd9Sstevel@tonic-gate 	}
9247c478bd9Sstevel@tonic-gate 
9257c478bd9Sstevel@tonic-gate 	/* get the certificate file */
9267c478bd9Sstevel@tonic-gate 	if (bitset(TLS_S_CERT2_OK, status) &&
9277c478bd9Sstevel@tonic-gate 	    SSL_CTX_use_certificate_file(*ctx, cf2, SSL_FILETYPE_PEM) <= 0)
9287c478bd9Sstevel@tonic-gate 	{
9297c478bd9Sstevel@tonic-gate 		if (LogLevel > 7)
9307c478bd9Sstevel@tonic-gate 		{
9317c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
9327c478bd9Sstevel@tonic-gate 				  "STARTTLS=%s, error: SSL_CTX_use_certificate_file(%s) failed",
9337c478bd9Sstevel@tonic-gate 				  who, cf2);
9347c478bd9Sstevel@tonic-gate 			if (LogLevel > 9)
9357c478bd9Sstevel@tonic-gate 				tlslogerr(who);
9367c478bd9Sstevel@tonic-gate 		}
9377c478bd9Sstevel@tonic-gate 	}
9387c478bd9Sstevel@tonic-gate 
9397c478bd9Sstevel@tonic-gate 	/* also check the private key */
9407c478bd9Sstevel@tonic-gate 	if (bitset(TLS_S_KEY2_OK, status) &&
9417c478bd9Sstevel@tonic-gate 	    (r = SSL_CTX_check_private_key(*ctx)) <= 0)
9427c478bd9Sstevel@tonic-gate 	{
9437c478bd9Sstevel@tonic-gate 		/* Private key does not match the certificate public key */
9447c478bd9Sstevel@tonic-gate 		if (LogLevel > 5)
9457c478bd9Sstevel@tonic-gate 		{
9467c478bd9Sstevel@tonic-gate 			sm_syslog(LOG_WARNING, NOQID,
9477c478bd9Sstevel@tonic-gate 				  "STARTTLS=%s, error: SSL_CTX_check_private_key 2 failed: %d",
9487c478bd9Sstevel@tonic-gate 				  who, r);
9497c478bd9Sstevel@tonic-gate 			if (LogLevel > 9)
9507c478bd9Sstevel@tonic-gate 				tlslogerr(who);
9517c478bd9Sstevel@tonic-gate 		}
9527c478bd9Sstevel@tonic-gate 	}
9537c478bd9Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
9547c478bd9Sstevel@tonic-gate 
9557c478bd9Sstevel@tonic-gate 	/* SSL_CTX_set_quiet_shutdown(*ctx, 1); violation of standard? */
9563ee0e492Sjbeck 
9573ee0e492Sjbeck 	SSL_CTX_set_options(*ctx, options);
9587c478bd9Sstevel@tonic-gate 
9597c478bd9Sstevel@tonic-gate # if !NO_DH
9607c478bd9Sstevel@tonic-gate 	/* Diffie-Hellman initialization */
9617c478bd9Sstevel@tonic-gate 	if (bitset(TLS_I_TRY_DH, req))
9627c478bd9Sstevel@tonic-gate 	{
9637c478bd9Sstevel@tonic-gate 		if (bitset(TLS_S_DHPAR_OK, status))
9647c478bd9Sstevel@tonic-gate 		{
9657c478bd9Sstevel@tonic-gate 			BIO *bio;
9667c478bd9Sstevel@tonic-gate 
9677c478bd9Sstevel@tonic-gate 			if ((bio = BIO_new_file(dhparam, "r")) != NULL)
9687c478bd9Sstevel@tonic-gate 			{
9697c478bd9Sstevel@tonic-gate 				dh = PEM_read_bio_DHparams(bio, NULL, NULL, NULL);
9707c478bd9Sstevel@tonic-gate 				BIO_free(bio);
9717c478bd9Sstevel@tonic-gate 				if (dh == NULL && LogLevel > 7)
9727c478bd9Sstevel@tonic-gate 				{
9737c478bd9Sstevel@tonic-gate 					unsigned long err;
9747c478bd9Sstevel@tonic-gate 
9757c478bd9Sstevel@tonic-gate 					err = ERR_get_error();
9767c478bd9Sstevel@tonic-gate 					sm_syslog(LOG_WARNING, NOQID,
9777c478bd9Sstevel@tonic-gate 						  "STARTTLS=%s, error: cannot read DH parameters(%s): %s",
9787c478bd9Sstevel@tonic-gate 						  who, dhparam,
9797c478bd9Sstevel@tonic-gate 						  ERR_error_string(err, NULL));
9807c478bd9Sstevel@tonic-gate 					if (LogLevel > 9)
9817c478bd9Sstevel@tonic-gate 						tlslogerr(who);
9827c478bd9Sstevel@tonic-gate 				}
9837c478bd9Sstevel@tonic-gate 			}
9847c478bd9Sstevel@tonic-gate 			else
9857c478bd9Sstevel@tonic-gate 			{
9867c478bd9Sstevel@tonic-gate 				if (LogLevel > 5)
9877c478bd9Sstevel@tonic-gate 				{
9887c478bd9Sstevel@tonic-gate 					sm_syslog(LOG_WARNING, NOQID,
9897c478bd9Sstevel@tonic-gate 						  "STARTTLS=%s, error: BIO_new_file(%s) failed",
9907c478bd9Sstevel@tonic-gate 						  who, dhparam);
9917c478bd9Sstevel@tonic-gate 					if (LogLevel > 9)
9927c478bd9Sstevel@tonic-gate 						tlslogerr(who);
9937c478bd9Sstevel@tonic-gate 				}
9947c478bd9Sstevel@tonic-gate 			}
9957c478bd9Sstevel@tonic-gate 		}
9967c478bd9Sstevel@tonic-gate 		if (dh == NULL && bitset(TLS_I_DH1024, req))
9977c478bd9Sstevel@tonic-gate 		{
998300fdee2SAndy Fiddaman 			DSA *dsa = NULL;
9997c478bd9Sstevel@tonic-gate 
10007c478bd9Sstevel@tonic-gate 			/* this takes a while! (7-130s on a 450MHz AMD K6-2) */
1001300fdee2SAndy Fiddaman 			if ((dsa = DSA_new()) == NULL ||
1002300fdee2SAndy Fiddaman 			    DSA_generate_parameters_ex(dsa, 1024,
1003300fdee2SAndy Fiddaman 			    NULL, 0, NULL, 0, NULL) == 0)
1004300fdee2SAndy Fiddaman 				dh = NULL;
1005300fdee2SAndy Fiddaman 			else
1006300fdee2SAndy Fiddaman 				dh = DSA_dup_DH(dsa);
1007300fdee2SAndy Fiddaman 			if (dsa != NULL)
1008300fdee2SAndy Fiddaman 				DSA_free(dsa);
10097c478bd9Sstevel@tonic-gate 		}
1010300fdee2SAndy Fiddaman 		else if (dh == NULL && bitset(TLS_I_DH512, req))
10117c478bd9Sstevel@tonic-gate 			dh = get_dh512();
10127c478bd9Sstevel@tonic-gate 
10137c478bd9Sstevel@tonic-gate 		if (dh == NULL)
10147c478bd9Sstevel@tonic-gate 		{
10157c478bd9Sstevel@tonic-gate 			if (LogLevel > 9)
10167c478bd9Sstevel@tonic-gate 			{
10177c478bd9Sstevel@tonic-gate 				unsigned long err;
10187c478bd9Sstevel@tonic-gate 
10197c478bd9Sstevel@tonic-gate 				err = ERR_get_error();
10207c478bd9Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
10217c478bd9Sstevel@tonic-gate 					  "STARTTLS=%s, error: cannot read or set DH parameters(%s): %s",
10227c478bd9Sstevel@tonic-gate 					  who, dhparam,
10237c478bd9Sstevel@tonic-gate 					  ERR_error_string(err, NULL));
10247c478bd9Sstevel@tonic-gate 			}
10257c478bd9Sstevel@tonic-gate 			if (bitset(TLS_I_REQ_DH, req))
10267c478bd9Sstevel@tonic-gate 				return false;
10277c478bd9Sstevel@tonic-gate 		}
10287c478bd9Sstevel@tonic-gate 		else
10297c478bd9Sstevel@tonic-gate 		{
10307c478bd9Sstevel@tonic-gate 			SSL_CTX_set_tmp_dh(*ctx, dh);
10317c478bd9Sstevel@tonic-gate 
10327c478bd9Sstevel@tonic-gate 			/* important to avoid small subgroup attacks */
10337c478bd9Sstevel@tonic-gate 			SSL_CTX_set_options(*ctx, SSL_OP_SINGLE_DH_USE);
10347c478bd9Sstevel@tonic-gate 			if (LogLevel > 13)
10357c478bd9Sstevel@tonic-gate 				sm_syslog(LOG_INFO, NOQID,
10367c478bd9Sstevel@tonic-gate 					  "STARTTLS=%s, Diffie-Hellman init, key=%d bit (%c)",
10377c478bd9Sstevel@tonic-gate 					  who, 8 * DH_size(dh), *dhparam);
10387c478bd9Sstevel@tonic-gate 			DH_free(dh);
10397c478bd9Sstevel@tonic-gate 		}
10407c478bd9Sstevel@tonic-gate 	}
10417c478bd9Sstevel@tonic-gate # endif /* !NO_DH */
10427c478bd9Sstevel@tonic-gate 
10437c478bd9Sstevel@tonic-gate 
10447c478bd9Sstevel@tonic-gate 	/* XXX do we need this cache here? */
10457c478bd9Sstevel@tonic-gate 	if (bitset(TLS_I_CACHE, req))
1046445f2479Sjbeck 	{
1047445f2479Sjbeck 		SSL_CTX_sess_set_cache_size(*ctx, 1);
1048445f2479Sjbeck 		SSL_CTX_set_timeout(*ctx, 1);
1049445f2479Sjbeck 		SSL_CTX_set_session_id_context(*ctx,
1050445f2479Sjbeck 			(void *) &server_session_id_context,
1051445f2479Sjbeck 			sizeof(server_session_id_context));
1052445f2479Sjbeck 		(void) SSL_CTX_set_session_cache_mode(*ctx,
1053445f2479Sjbeck 				SSL_SESS_CACHE_SERVER);
1054445f2479Sjbeck 	}
1055445f2479Sjbeck 	else
1056445f2479Sjbeck 	{
1057445f2479Sjbeck 		(void) SSL_CTX_set_session_cache_mode(*ctx,
1058445f2479Sjbeck 				SSL_SESS_CACHE_OFF);
1059445f2479Sjbeck 	}
10607c478bd9Sstevel@tonic-gate 
10617c478bd9Sstevel@tonic-gate 	/* load certificate locations and default CA paths */
10627c478bd9Sstevel@tonic-gate 	if (bitset(TLS_S_CERTP_EX, status) && bitset(TLS_S_CERTF_EX, status))
10637c478bd9Sstevel@tonic-gate 	{
10647c478bd9Sstevel@tonic-gate 		if ((r = SSL_CTX_load_verify_locations(*ctx, cacertfile,
10657c478bd9Sstevel@tonic-gate 						       cacertpath)) == 1)
10667c478bd9Sstevel@tonic-gate 		{
10677c478bd9Sstevel@tonic-gate 			/*
10687c478bd9Sstevel@tonic-gate 			**  We have to install our own verify callback:
10697c478bd9Sstevel@tonic-gate 			**  SSL_VERIFY_PEER requests a client cert but even
10707c478bd9Sstevel@tonic-gate 			**  though *FAIL_IF* isn't set, the connection
10717c478bd9Sstevel@tonic-gate 			**  will be aborted if the client presents a cert
10727c478bd9Sstevel@tonic-gate 			**  that is not "liked" (can't be verified?) by
10737c478bd9Sstevel@tonic-gate 			**  the TLS library :-(
10747c478bd9Sstevel@tonic-gate 			*/
10757c478bd9Sstevel@tonic-gate 
10767c478bd9Sstevel@tonic-gate 			/*
10777c478bd9Sstevel@tonic-gate 			**  XXX currently we could call tls_set_verify()
10787c478bd9Sstevel@tonic-gate 			**  but we hope that that function will later on
10797c478bd9Sstevel@tonic-gate 			**  only set the mode per connection.
10807c478bd9Sstevel@tonic-gate 			*/
10817c478bd9Sstevel@tonic-gate 			SSL_CTX_set_verify(*ctx,
10827c478bd9Sstevel@tonic-gate 				bitset(TLS_I_NO_VRFY, req) ? SSL_VERIFY_NONE
10837c478bd9Sstevel@tonic-gate 							   : SSL_VERIFY_PEER,
10847c478bd9Sstevel@tonic-gate 				NULL);
10857c478bd9Sstevel@tonic-gate 
10867c478bd9Sstevel@tonic-gate 			/* install verify callback */
10877c478bd9Sstevel@tonic-gate 			SSL_CTX_set_cert_verify_callback(*ctx, tls_verify_cb,
10887c478bd9Sstevel@tonic-gate 							 NULL);
10897c478bd9Sstevel@tonic-gate 			SSL_CTX_set_client_CA_list(*ctx,
10907c478bd9Sstevel@tonic-gate 				SSL_load_client_CA_file(cacertfile));
10917c478bd9Sstevel@tonic-gate 		}
10927c478bd9Sstevel@tonic-gate 		else
10937c478bd9Sstevel@tonic-gate 		{
10947c478bd9Sstevel@tonic-gate 			/*
10957c478bd9Sstevel@tonic-gate 			**  can't load CA data; do we care?
10967c478bd9Sstevel@tonic-gate 			**  the data is necessary to authenticate the client,
10977c478bd9Sstevel@tonic-gate 			**  which in turn would be necessary
10987c478bd9Sstevel@tonic-gate 			**  if we want to allow relaying based on it.
10997c478bd9Sstevel@tonic-gate 			*/
11007c478bd9Sstevel@tonic-gate 			if (LogLevel > 5)
11017c478bd9Sstevel@tonic-gate 			{
11027c478bd9Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
11037c478bd9Sstevel@tonic-gate 					  "STARTTLS=%s, error: load verify locs %s, %s failed: %d",
11047c478bd9Sstevel@tonic-gate 					  who, cacertpath, cacertfile, r);
11057c478bd9Sstevel@tonic-gate 				if (LogLevel > 9)
11067c478bd9Sstevel@tonic-gate 					tlslogerr(who);
11077c478bd9Sstevel@tonic-gate 			}
11087c478bd9Sstevel@tonic-gate 			if (bitset(TLS_I_VRFY_LOC, req))
11097c478bd9Sstevel@tonic-gate 				return false;
11107c478bd9Sstevel@tonic-gate 		}
11117c478bd9Sstevel@tonic-gate 	}
11127c478bd9Sstevel@tonic-gate 
11137c478bd9Sstevel@tonic-gate 	/* XXX: make this dependent on an option? */
11147c478bd9Sstevel@tonic-gate 	if (tTd(96, 9))
11157c478bd9Sstevel@tonic-gate 		SSL_CTX_set_info_callback(*ctx, apps_ssl_info_cb);
11167c478bd9Sstevel@tonic-gate 
11177c478bd9Sstevel@tonic-gate # if _FFR_TLS_1
11187c478bd9Sstevel@tonic-gate 	/* install our own cipher list */
11197c478bd9Sstevel@tonic-gate 	if (CipherList != NULL && *CipherList != '\0')
11207c478bd9Sstevel@tonic-gate 	{
11217c478bd9Sstevel@tonic-gate 		if (SSL_CTX_set_cipher_list(*ctx, CipherList) <= 0)
11227c478bd9Sstevel@tonic-gate 		{
11237c478bd9Sstevel@tonic-gate 			if (LogLevel > 7)
11247c478bd9Sstevel@tonic-gate 			{
11257c478bd9Sstevel@tonic-gate 				sm_syslog(LOG_WARNING, NOQID,
11267c478bd9Sstevel@tonic-gate 					  "STARTTLS=%s, error: SSL_CTX_set_cipher_list(%s) failed, list ignored",
11277c478bd9Sstevel@tonic-gate 					  who, CipherList);
11287c478bd9Sstevel@tonic-gate 
11297c478bd9Sstevel@tonic-gate 				if (LogLevel > 9)
11307c478bd9Sstevel@tonic-gate 					tlslogerr(who);
11317c478bd9Sstevel@tonic-gate 			}
11327c478bd9Sstevel@tonic-gate 			/* failure if setting to this list is required? */
11337c478bd9Sstevel@tonic-gate 		}
11347c478bd9Sstevel@tonic-gate 	}
11357c478bd9Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
11367c478bd9Sstevel@tonic-gate 	if (LogLevel > 12)
11377c478bd9Sstevel@tonic-gate 		sm_syslog(LOG_INFO, NOQID, "STARTTLS=%s, init=%d", who, ok);
11387c478bd9Sstevel@tonic-gate 
11397c478bd9Sstevel@tonic-gate # if _FFR_TLS_1
11407c478bd9Sstevel@tonic-gate #  if 0
11417c478bd9Sstevel@tonic-gate 	/*
11427c478bd9Sstevel@tonic-gate 	**  this label is required if we want to have a "clean" exit
11437c478bd9Sstevel@tonic-gate 	**  see the comments above at the initialization of cf2
11447c478bd9Sstevel@tonic-gate 	*/
11457c478bd9Sstevel@tonic-gate 
11467c478bd9Sstevel@tonic-gate     endinittls:
11477c478bd9Sstevel@tonic-gate #  endif /* 0 */
11487c478bd9Sstevel@tonic-gate 
11497c478bd9Sstevel@tonic-gate 	/* undo damage to global variables */
11507c478bd9Sstevel@tonic-gate 	if (cf2 != NULL)
11517c478bd9Sstevel@tonic-gate 		*--cf2 = ',';
11527c478bd9Sstevel@tonic-gate 	if (kf2 != NULL)
11537c478bd9Sstevel@tonic-gate 		*--kf2 = ',';
11547c478bd9Sstevel@tonic-gate # endif /* _FFR_TLS_1 */
11557c478bd9Sstevel@tonic-gate 
11567c478bd9Sstevel@tonic-gate 	return ok;
11577c478bd9Sstevel@tonic-gate }
11587c478bd9Sstevel@tonic-gate /*
11597c478bd9Sstevel@tonic-gate **  TLS_GET_INFO -- get information about TLS connection
11607c478bd9Sstevel@tonic-gate **
11617c478bd9Sstevel@tonic-gate **	Parameters:
11627c478bd9Sstevel@tonic-gate **		ssl -- TLS connection structure
11637c478bd9Sstevel@tonic-gate **		srv -- server or client
11647c478bd9Sstevel@tonic-gate **		host -- hostname of other side
11657c478bd9Sstevel@tonic-gate **		mac -- macro storage
11667c478bd9Sstevel@tonic-gate **		certreq -- did we ask for a cert?
11677c478bd9Sstevel@tonic-gate **
11687c478bd9Sstevel@tonic-gate **	Returns:
1169