1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 *
25 * Copyright 2017 Nexenta Systems, Inc.  All rights reserved.
26 */
27
28#include <string.h>
29#include <errno.h>
30#include <syslog.h>
31#include <procfs.h>
32#include <unistd.h>
33#include <fcntl.h>
34#include <libintl.h>
35#include <atomic.h>
36#include <pthread.h>
37#include <sys/mman.h>
38#include <time.h>
39#include "solaris-int.h"
40#include "ns_connmgmt.h"
41#include "ns_cache_door.h"
42#include "ns_internal.h"
43
44/*
45 * Access (reference, shutdown, or reload) the current connection
46 * management control structure conn_mgmt_t.
47 */
48#define	NS_CONN_MGMT_OP_REF		1
49#define	NS_CONN_MGMT_OP_SHUTDOWN	2
50#define	NS_CONN_MGMT_OP_RELOAD_CONFIG	3
51#define	NS_CONN_MGMT_OP_NEW_CONFIG	4
52#define	NS_CONN_MGMT_OP_LIB_INIT	5
53
54static ns_conn_mgmt_t *access_conn_mgmt(int);
55static ns_conn_mgmt_t *release_conn_mgmt(ns_conn_mgmt_t *, boolean_t);
56static int close_conn_mt(ns_conn_mt_t *, int, ns_ldap_error_t **,
57	ns_conn_user_t *);
58static int close_conn_mt_when_nouser(ns_conn_mt_t *cm);
59void shutdown_all_conn_mt(ns_conn_mgmt_t *cmg);
60static int conn_signal(ns_conn_mt_t *);
61static int conn_wait(ns_conn_mt_t *, ns_conn_user_t *);
62static void close_conn_mt_by_procchg(ns_conn_mt_t *cm, int rc, char *errmsg);
63static ns_conn_mgmt_t *proc_server_change(ns_server_status_change_t *chg,
64	ns_conn_mgmt_t  *cmg);
65static void get_preferred_servers(boolean_t, boolean_t, ns_conn_mgmt_t *);
66static void start_thread();
67
68static ns_conn_mgmt_t	*ns_connmgmt = NULL;
69static ns_conn_mgmt_t	*ns_connmgmt_parent = NULL;
70static mutex_t		ns_connmgmt_lock = DEFAULTMUTEX;
71static boolean_t	ns_connmgmt_shutting_down = B_FALSE;
72
73#define	NS_CONN_MSG_NO_CONN_MGMT gettext( \
74	"libsldap: unable to allocate the connection management control")
75#define	NS_CONN_MSG_NO_MTC_KEY gettext( \
76	"libsldap: unable to allocate the TSD key for per-thread ldap error")
77#define	NS_CONN_MSG_NO_CMG_KEY gettext( \
78	"libsldap: unable to allocate the TSD key for connection management")
79#define	NS_CONN_MSG_SHUTDOWN gettext("libsldap: library is being unloaded")
80#define	NS_CONN_MSG_RELOADED gettext( \
81	"libsldap: configuration has been reloaded")
82#define	NS_CONN_MSG_SHUTDOWN_RELOADED gettext( \
83	"libsldap: library unloaded or configuration has been reloaded")
84#define	NS_CONN_MSG_BAD_CACHEMGR_DATA gettext( \
85	"libsldap: received incorrect data from ldap_cachemgr")
86#define	NS_CONN_MSG_MEMORY_ERROR gettext( \
87	"libsldap: unable to allocate memory")
88#define	NS_CONN_MSG_NO_PROCCHG_THREAD gettext( \
89	"libsldap: unable to start the server monitor thread (%s)")
90#define	NS_CONN_MSG_DOWN_FROM_CACHEMGR gettext( \
91	"libsldap: server down reported by ldap_cachemgr")
92
93static int ns_conn_free = 1;
94#define	NS_CONN_UNLOCK_AND_FREE(free, cm, cmg)  \
95{ \
96	(void) mutex_unlock(&(cm)->lock);	\
97	if (free == 1)	\
98		cmg = free_conn_mt((cm), 1); \
99	if (cmg != NULL) \
100		(void) mutex_unlock(&(cmg)->lock); \
101}
102
103#define	NS_CONN_CHECK_ABORT_AND_LOCK(cmg, cu, errp) \
104{ \
105	char *msg = NULL; \
106	(void) mutex_lock(&(cmg)->lock); \
107	if ((cmg)->shutting_down == B_TRUE) \
108		msg = NS_CONN_MSG_SHUTDOWN; \
109	else if ((cmg)->cfg_reloaded == B_TRUE)  \
110		msg = NS_CONN_MSG_RELOADED; \
111	if (msg != NULL) { \
112		(*errp) = __s_api_make_error(NS_LDAP_OP_FAILED, msg); \
113		(void) mutex_unlock(&(cmg)->lock); \
114		return (NS_LDAP_OP_FAILED); \
115	} \
116}
117
118/*
119 * TSD keys ns_mtckey and ns_cmgkey are for sharing ldap connections
120 * and their associated connection management structure among
121 * multiple threads. The pointers to the per-thread ldap error
122 * information and the connection management structure are
123 * saved in ns_mtckey and ns_cmgkey.
124 */
125thread_key_t ns_mtckey = THR_ONCE_KEY;
126thread_key_t ns_cmgkey = THR_ONCE_KEY;
127
128/* Per thread LDAP error resides in thread-specific data (ns_mtckey) */
129struct ldap_error {
130	int	le_errno;
131	char	*le_matched;
132	char	*le_errmsg;
133};
134
135/* NULL struct ldap_error */
136static struct ldap_error ldap_error_NULL = { LDAP_SUCCESS, NULL, NULL};
137
138/* destructor: free the ldap error data in the thread specific area */
139static void
140ns_mtckey_cleanup(void *key)
141{
142	struct ldap_error *le = (struct ldap_error *)key;
143
144	if (le == NULL)
145		return;
146	if (le->le_matched != NULL) {
147		ldap_memfree(le->le_matched);
148	}
149	if (le->le_errmsg != NULL) {
150		ldap_memfree(le->le_errmsg);
151	}
152	free(le);
153}
154
155/* Free/detach the thread specific data structures */
156static void
157conn_tsd_free(void)
158{
159	void	*tsd = NULL;
160	int	rc;
161
162	/* free the per-thread ldap error info */
163	rc = thr_getspecific(ns_mtckey, &tsd);
164	if (rc == 0 && tsd != NULL)
165		ns_mtckey_cleanup(tsd);
166	(void) thr_setspecific(ns_mtckey, NULL);
167
168	/* detach the connection management control */
169	(void) thr_setspecific(ns_cmgkey, NULL);
170}
171
172/* per-thread callback function for allocating a mutex */
173static void *
174ns_mutex_alloc(void)
175{
176	mutex_t *mutexp = NULL;
177
178	if ((mutexp = malloc(sizeof (mutex_t))) != NULL) {
179		if (mutex_init(mutexp, USYNC_THREAD, NULL) != 0) {
180			free(mutexp);
181			mutexp = NULL;
182		}
183	}
184	return (mutexp);
185}
186
187/* per-thread callback function for freeing a mutex */
188static void
189ns_mutex_free(void *mutexp)
190{
191	(void) mutex_destroy((mutex_t *)mutexp);
192	free(mutexp);
193}
194
195/*
196 * Function for setting up thread-specific data
197 * where per thread LDAP error and the pointer
198 * to the active connection management control
199 * are stored.
200 */
201static int
202conn_tsd_setup(ns_conn_mgmt_t *cmg)
203{
204	void	*tsd;
205	int	rc;
206
207	rc = thr_setspecific(ns_cmgkey, cmg);
208	if (rc != 0) /* must be ENOMEM */
209		return (-1);
210
211	/* return success if the ns_mtckey TSD is already set */
212	rc = thr_getspecific(ns_mtckey, &tsd);
213	if (rc == 0 && tsd != NULL)
214		return (0);
215
216	/* allocate and set the ns_mtckey TSD */
217	tsd = (void *) calloc(1, sizeof (struct ldap_error));
218	if (tsd == NULL)
219		return (-1);
220	rc = thr_setspecific(ns_mtckey, tsd);
221	if (rc != 0) { /* must be ENOMEM */
222		free(tsd);
223		return (-1);
224	}
225	return (0);
226}
227
228/* Callback function for setting the per thread LDAP error */
229/*ARGSUSED*/
230static void
231set_ld_error(int err, char *matched, char *errmsg, void *dummy)
232{
233	struct ldap_error	*le;
234	int			eno;
235
236	if ((eno = thr_getspecific(ns_mtckey, (void **)&le)) != 0) {
237		syslog(LOG_ERR, gettext(
238		    "libsldap: set_ld_error: thr_getspecific failed (%s)."),
239		    strerror(eno));
240		return;
241	}
242
243	/* play safe, do nothing if TSD pointer is NULL */
244	if (le == NULL) {
245		syslog(LOG_INFO, gettext(
246		    "libsldap: set_ld_error: TSD pointer is NULL."));
247		return;
248	}
249
250	le->le_errno = err;
251
252	if (le->le_matched != NULL) {
253		ldap_memfree(le->le_matched);
254		le->le_matched = NULL;
255	}
256	le->le_matched = matched;
257
258	if (le->le_errmsg != NULL) {
259		ldap_memfree(le->le_errmsg);
260		le->le_errmsg = NULL;
261	}
262	le->le_errmsg = errmsg;
263}
264
265/* check and allocate the thread-specific data for using a MT connection */
266static int
267conn_tsd_check(ns_conn_mgmt_t *cmg)
268{
269	if (conn_tsd_setup(cmg) != 0)
270		return (NS_LDAP_MEMORY);
271
272	return (NS_LDAP_SUCCESS);
273}
274
275/* Callback function for getting the per thread LDAP error */
276/*ARGSUSED*/
277static int
278get_ld_error(char **matched, char **errmsg, void *dummy)
279{
280	struct ldap_error	*le;
281	int			eno;
282
283	if ((eno = thr_getspecific(ns_mtckey, (void **)&le)) != 0) {
284		syslog(LOG_ERR, gettext(
285		    "libsldap: get_ld_error: thr_getspecific failed (%s)"),
286		    strerror(eno));
287		return (eno);
288	}
289
290	/* play safe, return NULL error data, if TSD pointer is NULL */
291	if (le == NULL)
292		le = &ldap_error_NULL;
293
294	if (matched != NULL) {
295		*matched = le->le_matched;
296	}
297	if (errmsg != NULL) {
298		*errmsg = le->le_errmsg;
299	}
300	return (le->le_errno);
301}
302
303/* Callback function for setting per thread errno */
304static void
305set_errno(int err)
306{
307	errno = err;
308}
309
310/* Callback function for getting per thread errno */
311static int
312get_errno(void)
313{
314	return (errno);
315}
316
317static void *
318ltf_threadid(void)
319{
320	return ((void *)(uintptr_t)thr_self());
321}
322
323/* set up an ldap session 'ld' for sharing among multiple threads */
324static int
325setup_mt_conn(LDAP *ld)
326{
327
328	struct ldap_thread_fns		tfns;
329	struct ldap_extra_thread_fns	extrafns;
330	int				rc;
331
332	/*
333	 * Set the function pointers for dealing with mutexes
334	 * and error information
335	 */
336	(void) memset(&tfns, '\0', sizeof (struct ldap_thread_fns));
337	tfns.ltf_mutex_alloc = (void *(*)(void)) ns_mutex_alloc;
338	tfns.ltf_mutex_free = (void (*)(void *)) ns_mutex_free;
339	tfns.ltf_mutex_lock = (int (*)(void *)) mutex_lock;
340	tfns.ltf_mutex_unlock = (int (*)(void *)) mutex_unlock;
341	tfns.ltf_get_errno = get_errno;
342	tfns.ltf_set_errno = set_errno;
343	tfns.ltf_get_lderrno = get_ld_error;
344	tfns.ltf_set_lderrno = set_ld_error;
345	tfns.ltf_lderrno_arg = NULL;
346
347	/*
348	 * Set up the ld to use those function pointers
349	 */
350	rc = ldap_set_option(ld, LDAP_OPT_THREAD_FN_PTRS,
351	    (void *) &tfns);
352	if (rc < 0) {
353		syslog(LOG_INFO, gettext("libsldap: ldap_set_option "
354		"(LDAP_OPT_THREAD_FN_PTRS)"));
355		return (0);
356	}
357
358	/*
359	 * Set the function pointers for working with semaphores
360	 */
361	(void) memset(&extrafns, '\0',
362	    sizeof (struct ldap_extra_thread_fns));
363	extrafns.ltf_threadid_fn = ltf_threadid;
364	extrafns.ltf_mutex_trylock = NULL;
365	extrafns.ltf_sema_alloc = NULL;
366	extrafns.ltf_sema_free = NULL;
367	extrafns.ltf_sema_wait = NULL;
368	extrafns.ltf_sema_post = NULL;
369
370	/* Set up the ld to use those function pointers */
371	rc = ldap_set_option(ld, LDAP_OPT_EXTRA_THREAD_FN_PTRS,
372	    (void *) &extrafns);
373	if (rc < 0) {
374		syslog(LOG_INFO, gettext("libsldap: ldap_set_option "
375		"(LDAP_OPT_EXTRA_THREAD_FN_PTRS)"));
376		return (0);
377	}
378
379	return (1);
380}
381
382/* set up an MT connection for sharing among multiple threads */
383static int
384setup_mt_ld(LDAP *ld, ns_conn_mgmt_t *cmg)
385{
386	thread_t	t = thr_self();
387
388	/* set up the per-thread data for using the MT connection */
389	if (conn_tsd_setup(cmg) == -1) {
390		syslog(LOG_WARNING,
391		    gettext("libsldap: tid= %d: unable to set up TSD\n"), t);
392		return (-1);
393	}
394
395	if (setup_mt_conn(ld) == 0) {
396		/* multiple threads per connection not supported */
397		syslog(LOG_WARNING, gettext("libsldap: tid= %d: multiple "
398		    "threads per connection not supported\n"), t);
399		conn_tsd_free();
400		return (-1);
401	}
402	return (0);
403}
404
405/*
406 * Check name and UID of process, if it is nscd.
407 *
408 * Input:
409 *   pid	: PID of checked process
410 *   check_uid	: check if UID == 0
411 * Output:
412 *   B_TRUE	: nscd detected
413 *   B_FALSE	: nscd not confirmed
414 */
415static boolean_t
416check_nscd_proc(pid_t pid, boolean_t check_uid)
417{
418	psinfo_t	pinfo;
419	char		fname[MAXPATHLEN];
420	ssize_t		ret;
421	int		fd;
422
423	if (snprintf(fname, MAXPATHLEN, "/proc/%d/psinfo", pid) > 0) {
424		if ((fd = open(fname,  O_RDONLY)) >= 0) {
425			ret = read(fd, &pinfo, sizeof (psinfo_t));
426			(void) close(fd);
427			if ((ret == sizeof (psinfo_t)) &&
428			    (strcmp(pinfo.pr_fname, "nscd") == 0)) {
429				if (check_uid && (pinfo.pr_uid != 0))
430					return (B_FALSE);
431				return (B_TRUE);
432			}
433		}
434	}
435	return (B_FALSE);
436}
437
438/*
439 * Check if this process is peruser nscd.
440 */
441boolean_t
442__s_api_peruser_proc(void)
443{
444	pid_t		my_ppid;
445	static mutex_t	nscdLock = DEFAULTMUTEX;
446	static pid_t	checkedPpid = (pid_t)-1;
447	static boolean_t isPeruserNscd = B_FALSE;
448
449	my_ppid = getppid();
450
451	/*
452	 * Already checked before for this process? If yes, return cached
453	 * response.
454	 */
455	if (my_ppid == checkedPpid) {
456		return (isPeruserNscd);
457	}
458
459	(void) mutex_lock(&nscdLock);
460
461	/* Check once more incase another thread has just complete this. */
462	if (my_ppid == checkedPpid) {
463		(void) mutex_unlock(&nscdLock);
464		return (isPeruserNscd);
465	}
466
467	/* Reinitialize to be sure there is no residue after fork. */
468	isPeruserNscd = B_FALSE;
469
470	/* Am I the nscd process? */
471	if (check_nscd_proc(getpid(), B_FALSE)) {
472		/* Is my parent the nscd process with UID == 0. */
473		isPeruserNscd = check_nscd_proc(my_ppid, B_TRUE);
474	}
475
476	/* Remember for whom isPeruserNscd is. */
477	checkedPpid = my_ppid;
478
479	(void) mutex_unlock(&nscdLock);
480	return (isPeruserNscd);
481}
482
483/*
484 * Check if this process is main nscd.
485 */
486boolean_t
487__s_api_nscd_proc(void)
488{
489	pid_t		my_pid;
490	static mutex_t	nscdLock = DEFAULTMUTEX;
491	static pid_t	checkedPid = (pid_t)-1;
492	static boolean_t isMainNscd = B_FALSE;
493
494	/*
495	 * Don't bother checking if this process isn't root, this cannot
496	 * be main nscd.
497	 */
498	if (getuid() != 0)
499		return (B_FALSE);
500
501	my_pid = getpid();
502
503	/*
504	 * Already checked before for this process? If yes, return cached
505	 * response.
506	 */
507	if (my_pid == checkedPid) {
508		return (isMainNscd);
509	}
510
511	(void) mutex_lock(&nscdLock);
512
513	/* Check once more incase another thread has just done this. */
514	if (my_pid == checkedPid) {
515		(void) mutex_unlock(&nscdLock);
516		return (isMainNscd);
517	}
518
519	/*
520	 * Am I the nscd process? UID is already checked, not needed from
521	 * psinfo.
522	 */
523	isMainNscd = check_nscd_proc(my_pid, B_FALSE);
524
525	/* Remember for whom isMainNscd is. */
526	checkedPid = my_pid;
527
528	(void) mutex_unlock(&nscdLock);
529	return (isMainNscd);
530}
531
532/*
533 * initialize a connection management control structure conn_mgmt_t
534 */
535ns_conn_mgmt_t *
536init_conn_mgmt()
537{
538	ns_conn_mgmt_t	*cmg;
539
540	cmg = (ns_conn_mgmt_t *)calloc(1, sizeof (*cmg));
541	if (cmg == NULL) {
542		syslog(LOG_ERR, NS_CONN_MSG_NO_CONN_MGMT);
543		return (NULL);
544	}
545
546	/* is this process nscd or peruser nscd ? */
547	cmg->is_nscd = __s_api_nscd_proc();
548	cmg->is_peruser_nscd = __s_api_peruser_proc();
549
550	/*
551	 * assume the underlying libldap allows multiple threads sharing
552	 * the same ldap connection (MT connection)
553	 */
554	cmg->ldap_mt = B_TRUE;
555	/* state is inactive until MT connection is required/requested */
556	cmg->state = NS_CONN_MGMT_INACTIVE;
557
558	(void) mutex_init(&cmg->lock, USYNC_THREAD, NULL);
559	(void) mutex_init(&cmg->cfg_lock, USYNC_THREAD, NULL);
560	cmg->pid = getpid();
561
562	/* for nscd or peruser nscd, MT connection is required */
563	if (cmg->is_nscd == B_TRUE || cmg->is_peruser_nscd == B_TRUE)
564		cmg->state = NS_CONN_MGMT_ACTIVE;
565
566	/*
567	 * reference (or initialize) the current Native LDAP configuration and
568	 * if in nscd process, make it never refreshed
569	 */
570	cmg->config = __s_api_get_default_config_global();
571	if (cmg->config == NULL)
572		cmg->config = __s_api_loadrefresh_config_global();
573	if (cmg->config != NULL) {
574		/*
575		 * main nscd get config change notice from ldap_cachemgr
576		 * so won't times out and refresh the config
577		 */
578		if (cmg->is_nscd  == B_TRUE)
579			(cmg->config)->paramList[NS_LDAP_EXP_P].ns_tm = 0;
580		cmg->cfg_cookie = cmg->config->config_cookie;
581	}
582
583	return (cmg);
584}
585
586static void
587mark_shutdown_or_reloaded(int op)
588{
589	ns_conn_mgmt_t	*cmg = ns_connmgmt;
590
591	(void) mutex_lock(&cmg->lock);
592	if (op == NS_CONN_MGMT_OP_SHUTDOWN)
593		cmg->shutting_down = B_TRUE;
594	else
595		cmg->cfg_reloaded = B_TRUE;
596	atomic_inc_uint(&cmg->ref_cnt);
597	cmg->state = NS_CONN_MGMT_DETACHED;
598
599	if (op == NS_CONN_MGMT_OP_RELOAD_CONFIG)
600		__s_api_init_config_global(NULL);
601
602	(void) mutex_unlock(&cmg->lock);
603}
604
605/*
606 * Return a pointer to the current connection management. If
607 * it has not been created, or is requested to recreate, then
608 * create and return the pointer. It is possible, the current
609 * one is created by the parent before fork, create a new
610 * one too in such a case.
611 */
612static ns_conn_mgmt_t *
613get_current_conn_mgmt(int op)
614{
615	ns_conn_mgmt_t	*cmg = ns_connmgmt;
616	static pid_t	checked_pid = (pid_t)-1;
617	pid_t		mypid;
618
619	mypid = getpid();
620	if (cmg == NULL || checked_pid != mypid) {
621		checked_pid = mypid;
622
623		/*
624		 * if current conn_mgmt not created yet or is from parent
625		 * or is requested to recreate, create it
626		 */
627		if (cmg == NULL || cmg->pid != mypid) {
628			if (cmg != NULL) {
629				/*
630				 * We don't want to free the conn_mgmt
631				 * allocated by the parent, since
632				 * there may be ldap connections
633				 * still being used. So leave it
634				 * alone but keep it referenced,
635				 * so that it will not be flagged
636				 * as a piece of leaked memory.
637				 */
638				ns_connmgmt_parent = cmg;
639				/*
640				 * avoid lint warning; does not
641				 * change the conn_mgmt in parent
642				 */
643				ns_connmgmt_parent->state =
644				    NS_CONN_MGMT_DETACHED;
645			}
646			ns_connmgmt = init_conn_mgmt();
647			cmg = ns_connmgmt;
648			/*
649			 * ensure it will not be destroyed until explicitly
650			 * shut down or reloaded
651			 */
652			if (op == NS_CONN_MGMT_OP_REF)
653				atomic_inc_uint(&cmg->ref_cnt);
654		}
655	}
656
657	return (cmg);
658}
659
660static ns_conn_mgmt_t *
661access_conn_mgmt(int op)
662{
663	ns_conn_mgmt_t	*cmg = NULL;
664	ns_conn_mgmt_t	*cmg_prev;
665
666	(void) mutex_lock(&ns_connmgmt_lock);
667
668	/*
669	 * connection management is not available when the libsldap is being
670	 * unloaded or shut down
671	 */
672	if (ns_connmgmt_shutting_down == B_TRUE) {
673		(void) mutex_unlock(&ns_connmgmt_lock);
674		return (NULL);
675	}
676
677	if (op == NS_CONN_MGMT_OP_SHUTDOWN) {
678		ns_connmgmt_shutting_down = B_TRUE;
679		if (ns_connmgmt != NULL) {
680			cmg = ns_connmgmt;
681			mark_shutdown_or_reloaded(op);
682			ns_connmgmt = NULL;
683		}
684		(void) mutex_unlock(&ns_connmgmt_lock);
685		return (cmg);
686	}
687
688	if (op == NS_CONN_MGMT_OP_RELOAD_CONFIG ||
689	    op == NS_CONN_MGMT_OP_NEW_CONFIG) {
690		cmg_prev = ns_connmgmt;
691		mark_shutdown_or_reloaded(op);
692		/*
693		 * the previous cmg (cmg_prev) will be freed later
694		 * when its ref count reaches zero
695		 */
696		ns_connmgmt = NULL;
697	}
698
699	cmg = get_current_conn_mgmt(op);
700	if (cmg == NULL) {
701		(void) mutex_unlock(&ns_connmgmt_lock);
702		return (NULL);
703	}
704
705	atomic_inc_uint(&cmg->ref_cnt);
706	if (op == NS_CONN_MGMT_OP_RELOAD_CONFIG ||
707	    op == NS_CONN_MGMT_OP_NEW_CONFIG)
708		cmg = cmg_prev;
709	else { /* op is  NS_CONN_MGMT_OP_REF  or NS_CONN_MGMT_OP_LIB_INIT */
710		if (cmg->config == NULL)
711			cmg->config = __s_api_get_default_config();
712	}
713
714	(void) mutex_unlock(&ns_connmgmt_lock);
715	return (cmg);
716}
717
718/*
719 * free a connection management control
720 */
721static void
722free_conn_mgmt(ns_conn_mgmt_t *cmg)
723{
724	union {
725		ldap_data_t	s_d;
726		char		s_b[1024];
727	} space;
728	ldap_data_t	*sptr;
729	int		ndata;
730	int		adata;
731	int		rc;
732	ldap_get_chg_cookie_t cookie;
733
734	if (cmg == NULL)
735		return;
736	cookie = cmg->cfg_cookie;
737
738	__s_api_free2dArray(cmg->pservers);
739	/* destroy the previous config or release the current one */
740	if (cmg->config != NULL) {
741		if (cmg->state == NS_CONN_MGMT_DETACHED)
742			__s_api_destroy_config(cmg->config);
743		else
744			__s_api_release_config(cmg->config);
745	}
746
747	/* stop the server status/config-change monitor thread */
748	if (cmg->procchg_started == B_TRUE) {
749		if (cmg->procchg_tid != thr_self()) {
750			if (cmg->procchg_door_call == B_TRUE) {
751				adata = sizeof (ldap_call_t) + 1;
752				ndata = sizeof (space);
753				space.s_d.ldap_call.ldap_callnumber =
754				    GETSTATUSCHANGE;
755				space.s_d.ldap_call.ldap_u.get_change.op =
756				    NS_STATUS_CHANGE_OP_STOP;
757				space.s_d.ldap_call.ldap_u.get_change.cookie =
758				    cookie;
759				sptr = &space.s_d;
760				rc = __ns_ldap_trydoorcall(&sptr, &ndata,
761				    &adata);
762				if (rc != NS_CACHE_SUCCESS)
763					syslog(LOG_INFO,
764					    gettext("libsldap: "
765					    "free_conn_mgmt():"
766					    " stopping door call "
767					    " GETSTATUSCHANGE failed "
768					    " (rc = %d)"), rc);
769			}
770			(void) pthread_cancel(cmg->procchg_tid);
771			cmg->procchg_started = B_FALSE;
772		}
773	}
774
775	free(cmg);
776}
777
778static ns_conn_mgmt_t *
779release_conn_mgmt(ns_conn_mgmt_t *cmg, boolean_t unlock_cmg)
780{
781	if (cmg == NULL)
782		return (NULL);
783	if (atomic_dec_uint_nv(&cmg->ref_cnt) == 0) {
784		if (cmg->state == NS_CONN_MGMT_DETACHED) {
785			if (unlock_cmg == B_TRUE)
786				(void) mutex_unlock(&cmg->lock);
787			free_conn_mgmt(cmg);
788			__s_api_free_sessionPool();
789			return (NULL);
790		} else {
791			syslog(LOG_WARNING,
792			    gettext("libsldap: connection management "
793			    " has a refcount of zero but the state "
794			    " is not DETACHED (%d)"), cmg->state);
795			cmg = NULL;
796		}
797	}
798	return (cmg);
799}
800
801/*
802 * exposed function for initializing a connection management control structure
803 */
804ns_conn_mgmt_t *
805__s_api_conn_mgmt_init()
806{
807	if (thr_keycreate_once(&ns_mtckey, ns_mtckey_cleanup) != 0) {
808		syslog(LOG_WARNING, NS_CONN_MSG_NO_MTC_KEY);
809		return (NULL);
810	}
811
812	if (thr_keycreate_once(&ns_cmgkey, NULL) != 0) {
813		syslog(LOG_WARNING, NS_CONN_MSG_NO_CMG_KEY);
814		return (NULL);
815	}
816
817	return (access_conn_mgmt(NS_CONN_MGMT_OP_LIB_INIT));
818}
819
820/* initialize a connection user */
821ns_conn_user_t *
822__s_api_conn_user_init(int type, void *userinfo, boolean_t referral)
823{
824	ns_conn_user_t	*cu;
825	ns_conn_mgmt_t	*cmg;
826
827	/* delete the reference to the previously used conn_mgmt */
828	(void) thr_setspecific(ns_cmgkey, NULL);
829
830	cmg = access_conn_mgmt(NS_CONN_MGMT_OP_REF);
831	if (cmg == NULL)
832		return (NULL);
833
834	if (cmg->state != NS_CONN_MGMT_ACTIVE &&
835	    cmg->state != NS_CONN_MGMT_INACTIVE) {
836		atomic_dec_uint(&cmg->ref_cnt);
837		return (NULL);
838	}
839
840	cu = (ns_conn_user_t *)calloc(1, sizeof (*cu));
841	if (cu == NULL) {
842		atomic_dec_uint(&cmg->ref_cnt);
843		return (NULL);
844	}
845
846	cu->type = type;
847	cu->state = NS_CONN_USER_ALLOCATED;
848	cu->tid = thr_self();
849	cu->userinfo = userinfo;
850	cu->referral = referral;
851	cu->ns_rc = NS_LDAP_SUCCESS;
852	cu->conn_mgmt = cmg;
853
854	(void) conn_tsd_setup(cmg);
855
856	return (cu);
857}
858
859/*
860 * Free the resources used by a connection user.
861 * The caller should ensure this conn_user is
862 * not associated with any conn_mt, i.e.,
863 * not in any conn_mt's linked list of conn_users.
864 * The caller needs to free the userinfo member
865 * as well.
866 */
867void
868__s_api_conn_user_free(ns_conn_user_t *cu)
869{
870	ns_conn_mgmt_t	*cmg;
871
872	if (cu == NULL)
873		return;
874
875	cu->state = NS_CONN_USER_FREED;
876	if (cu->ns_error != NULL)
877		(void) __ns_ldap_freeError(&cu->ns_error);
878
879	cmg = cu->conn_mgmt;
880	conn_tsd_free();
881	(void) release_conn_mgmt(cmg, B_FALSE);
882	(void) free(cu);
883}
884
885/*
886 * Initialize an MT connection control structure
887 * that will be used to represent an ldap connection
888 * to be shared among multiple threads and to hold
889 * and manage all the conn_users using the ldap
890 * connection.
891 */
892static ns_conn_mt_t *
893init_conn_mt(ns_conn_mgmt_t *cmg, ns_ldap_error_t **ep)
894{
895	ns_conn_mt_t	*cm;
896	ns_conn_mgmt_t	*cmg_a;
897
898	cm = (ns_conn_mt_t *)calloc(1, sizeof (*cm));
899	if (cm == NULL) {
900		if (ep != NULL)
901			*ep = __s_api_make_error(NS_LDAP_MEMORY, NULL);
902		return (NULL);
903	}
904
905	cmg_a = access_conn_mgmt(NS_CONN_MGMT_OP_REF);
906	if (cmg_a != cmg) {
907		if (cmg_a != NULL) {
908			(void) release_conn_mgmt(cmg_a, B_FALSE);
909			if (ep != NULL)
910				*ep = __s_api_make_error(NS_LDAP_OP_FAILED,
911				    NS_CONN_MSG_SHUTDOWN_RELOADED);
912		}
913		return (NULL);
914	}
915
916	(void) mutex_init(&cm->lock, USYNC_THREAD, NULL);
917	cm->state = NS_CONN_MT_CONNECTING;
918	cm->tid = thr_self();
919	cm->pid = getpid();
920	cm->next = NULL;
921	cm->cu_head = NULL;
922	cm->cu_tail = NULL;
923	cm->conn = NULL;
924	cm->conn_mgmt = cmg;
925
926	return (cm);
927}
928
929/*
930 * Free an MT connection control structure, assume conn_mgmt is locked.
931 * 'unlock_cmg' is passed to release_conn_mgmt() to indicate the
932 * cmg needs to be unlocked or not.
933 */
934static ns_conn_mgmt_t *
935free_conn_mt(ns_conn_mt_t *cm, int unlock_cmg)
936{
937	ns_conn_mgmt_t	*cmg;
938
939	if (cm == NULL)
940		return (NULL);
941	if (cm->ns_error != NULL)
942		(void) __ns_ldap_freeError(&cm->ns_error);
943	if (cm->conn != NULL) {
944		if (cm->conn->ld != NULL)
945			(void) ldap_unbind(cm->conn->ld);
946		__s_api_freeConnection(cm->conn);
947	}
948	cmg = cm->conn_mgmt;
949	free(cm);
950	return (release_conn_mgmt(cmg, unlock_cmg));
951}
952
953/* add a connection user to an MT connection */
954static void
955add_cu2cm(ns_conn_user_t *cu, ns_conn_mt_t *cm)
956{
957
958	if (cm->cu_head == NULL) {
959		cm->cu_head = cu;
960		cm->cu_tail = cu;
961	} else {
962		cm->cu_tail->next = cu;
963		cm->cu_tail = cu;
964	}
965	cm->cu_cnt++;
966}
967
968/* add an MT connection to the connection management */
969static void
970add_cm2cmg(ns_conn_mt_t *cm, ns_conn_mgmt_t *cmg)
971{
972	/*
973	 * add connection opened for WRITE to top of list
974	 * for garbage collection purpose. This is to
975	 * ensure the connection will be closed after a
976	 * certain amount of time (60 seconds).
977	 */
978	if (cmg->cm_head == NULL) {
979		cmg->cm_head = cm;
980		cmg->cm_tail = cm;
981	} else {
982		if (cm->opened_for == NS_CONN_USER_WRITE) {
983			cm->next = cmg->cm_head;
984			cmg->cm_head = cm;
985		} else {
986			cmg->cm_tail->next = cm;
987			cmg->cm_tail = cm;
988		}
989	}
990	cmg->cm_cnt++;
991}
992
993/* delete a connection user from an MT connection */
994static void
995del_cu4cm(ns_conn_user_t *cu, ns_conn_mt_t *cm)
996{
997	ns_conn_user_t *pu, *u;
998
999	if (cu == NULL || cm->cu_head == NULL || cm->cu_cnt == 0)
1000		return;
1001
1002	/* only one conn_user on list */
1003	if (cm->cu_head == cm->cu_tail) {
1004		if (cu == cm->cu_head) {
1005			cm->cu_head = cm->cu_tail = NULL;
1006			cm->cu_cnt = 0;
1007			cu->next = NULL;
1008		}
1009		return;
1010	}
1011
1012	/* more than one and cu is the first one */
1013	if (cu == cm->cu_head) {
1014		cm->cu_head = cu->next;
1015		cm->cu_cnt--;
1016		cu->next = NULL;
1017		return;
1018	}
1019
1020	pu = cm->cu_head;
1021	for (u = cm->cu_head->next; u; u = u->next) {
1022		if (cu == u)
1023			break;
1024		pu = u;
1025	}
1026	if (pu != cm->cu_tail) {
1027		pu->next = cu->next;
1028		if (pu->next == NULL)
1029			cm->cu_tail = pu;
1030		cm->cu_cnt--;
1031		cu->next = NULL;
1032	} else {
1033		syslog(LOG_INFO, gettext(
1034		    "libsldap: del_cu4cm(): connection user not found"));
1035	}
1036}
1037
1038/* delete an MT connection from the connection management control structure */
1039static void
1040del_cm4cmg(ns_conn_mt_t *cm, ns_conn_mgmt_t *cmg)
1041{
1042	ns_conn_mt_t *pm, *m;
1043
1044	if (cm == NULL || cmg->cm_head == NULL || cmg->cm_cnt == 0)
1045		return;
1046
1047	/* only one conn_mt on list */
1048	if (cmg->cm_head == cmg->cm_tail) {
1049		if (cm == cmg->cm_head) {
1050			cmg->cm_head = cmg->cm_tail = NULL;
1051			cmg->cm_cnt = 0;
1052			cm->next = NULL;
1053		}
1054		return;
1055	}
1056
1057	/* more than one and cm is the first one */
1058	if (cm == cmg->cm_head) {
1059		cmg->cm_head = cm->next;
1060		cmg->cm_cnt--;
1061		cm->next = NULL;
1062		return;
1063	}
1064
1065	pm = cmg->cm_head;
1066	for (m = cmg->cm_head->next; m; m = m->next) {
1067		if (cm == m)
1068			break;
1069		pm = m;
1070	}
1071	if (pm != cmg->cm_tail) {
1072		pm->next = cm->next;
1073		if (pm->next == NULL)
1074			cmg->cm_tail = pm;
1075		cmg->cm_cnt--;
1076		cm->next = NULL;
1077	} else {
1078		syslog(LOG_INFO, gettext(
1079		    "libsldap: del_cm4cmg(): MT connection not found"));
1080	}
1081}
1082
1083/*
1084 * compare to see if the server and credential for authentication match
1085 * those used by an MT connection
1086 */
1087static boolean_t
1088is_server_cred_matched(const char *server, const ns_cred_t *cred,
1089    ns_conn_mt_t *cm)
1090{
1091	Connection	*cp = cm->conn;
1092
1093	/* check server first */
1094	if (server != NULL && *server != 0) {
1095		if (strcasecmp(server, cp->serverAddr) != 0)
1096			return (B_FALSE);
1097	}
1098
1099	if (cred == NULL)
1100		return (B_TRUE);
1101
1102	/* then check cred */
1103	return (__s_api_is_auth_matched(cp->auth, cred));
1104}
1105
1106/*
1107 * Wait until a pending MT connection becomes available.
1108 * Return 1 if so, 0 if error.
1109 *
1110 * Assume the current conn_mgmt and the input conn_mt
1111 * are locked.
1112 */
1113static int
1114wait_for_conn_mt(ns_conn_user_t *cu, ns_conn_mt_t *cm)
1115{
1116
1117	cu->state = NS_CONN_USER_WAITING;
1118	add_cu2cm(cu, cm);
1119	cu->conn_mt = cm;
1120
1121	(void) mutex_unlock(&cm->lock);
1122	/*
1123	 * It could take some time so we don't want to hold
1124	 * cm->conn_mgmt across the wait
1125	 */
1126	(void) mutex_unlock(&(cm->conn_mgmt)->lock);
1127
1128	(void) mutex_lock(&cm->lock);
1129	/* check one more time see if need to wait */
1130	if (cm->state == NS_CONN_MT_CONNECTING) {
1131		(void) conn_wait(cm, cu);
1132
1133		/* cm->lock is locked again at this point */
1134
1135		cu->state = NS_CONN_USER_WOKEUP;
1136	}
1137
1138	if (cm->state == NS_CONN_MT_CONNECTED)
1139		return (1);
1140	else {
1141		del_cu4cm(cu, cm);
1142		cu->conn_mt = NULL;
1143		cu->bad_mt_conn = B_FALSE;
1144		return (0);
1145	}
1146}
1147
1148/*
1149 * Check and see if the input MT connection '*cm' should be closed.
1150 * In two cases, it should be closed. If a preferred server is
1151 * found to be up when ldap_cachemgr is queried and reported back.
1152 * Or when the server being used for the connection is found to
1153 * be down. Return B_FALSE if the connection is not closed (or not marked
1154 * to be closed), otherwise unlock mutex (*cm)->lock and return B_TRUE.
1155 * This function assumes conn_mgmt cmg and conn_mt *cm are locked.
1156 */
1157static boolean_t
1158check_and_close_conn(ns_conn_mgmt_t *cmg, ns_conn_mt_t **cm,
1159    ns_conn_user_t *cu)
1160{
1161
1162	int rc;
1163	int j;
1164	int svridx = -1;
1165	int upidx = -1;
1166	int free_cm;
1167	ns_server_info_t sinfo;
1168	ns_ldap_error_t *errorp = NULL;
1169
1170	/*
1171	 * check only if preferred servers are defined
1172	 */
1173	if (cmg->pservers_loaded == B_FALSE)
1174		get_preferred_servers(B_FALSE, B_FALSE, cmg);
1175	if (cmg->pservers == NULL)
1176		return (B_FALSE);
1177
1178	/*
1179	 * ask ldap_cachemgr for the first available server
1180	 */
1181	rc = __s_api_requestServer(NS_CACHE_NEW, NULL,
1182	    &sinfo, &errorp, NS_CACHE_ADDR_IP);
1183	if (rc != NS_LDAP_SUCCESS || sinfo.server == NULL) {
1184		(void) __ns_ldap_freeError(&errorp);
1185		return (B_FALSE);
1186	}
1187
1188	/*
1189	 * Did ldap_cachemgr return a preferred server ?
1190	 */
1191	for (j = 0; cmg->pservers[j] != NULL; j++) {
1192		if (strcasecmp(sinfo.server, cmg->pservers[j]) != 0)
1193			continue;
1194		upidx = j;
1195		break;
1196	}
1197
1198	/*
1199	 * Is the server being used a preferred one ?
1200	 */
1201	for (j = 0; cmg->pservers[j] != NULL; j++) {
1202		if (strcasecmp(cmg->pservers[j], (*cm)->conn->serverAddr) != 0)
1203			continue;
1204		svridx = j;
1205		break;
1206	}
1207
1208	/*
1209	 * Need to fall back to a down-but-now-up preferred server ?
1210	 * A preferred server falls back to a more preferred one.
1211	 * A regular one falls back to any preferred ones. So if
1212	 * both are preferred ones and same index, or both
1213	 * are not preferred ones, then no need to close the
1214	 * connection.
1215	 */
1216	if ((upidx == -1 && svridx == -1) ||
1217	    (upidx != -1 && svridx != -1 && upidx == svridx)) {
1218		__s_api_free_server_info(&sinfo);
1219		return (B_FALSE);
1220	}
1221
1222	/*
1223	 * otherwise, 4 cases, all may need to close the connection:
1224	 * For case 1 and 2, both servers are preferred ones:
1225	 * 1. ldap_cachemgr returned a better one to use (upidx < svridx)
1226	 * 2. the server being used is down (upidx > svridx)
1227	 * 3. ldap_cachemgr returned a preferred one, but the server
1228	 *    being used is not, so need to fall back to the preferred server
1229	 * 4. ldap_cachemgr returned a non-preferred one, but the server
1230	 *    being used is a preferred one, so it must be down (since
1231	 *    ldap_cachemgr always returns a preferred one when possible).
1232	 * For case 1 & 3, close the READ connection when no user uses it.
1233	 * For 2 and 4, close the connection with error rc, LDAP_SERVER_DOWN.
1234	 */
1235	if (upidx != -1 && (svridx == -1 || upidx < svridx)) { /* case 1 & 3 */
1236		/* fallback does not make sense for WRITE/referred connection */
1237		if ((*cm)->opened_for == NS_CONN_USER_WRITE ||
1238		    (*cm)->referral == B_TRUE) {
1239			__s_api_free_server_info(&sinfo);
1240			return (B_FALSE);
1241		}
1242		free_cm = close_conn_mt_when_nouser(*cm);
1243		if (cmg->shutting_down == B_FALSE)
1244			cu->retry = B_TRUE;
1245	} else {
1246		ns_ldap_error_t *ep;
1247		ep = __s_api_make_error(LDAP_SERVER_DOWN,
1248		    NS_CONN_MSG_DOWN_FROM_CACHEMGR);
1249		/* cu has not been attached to cm yet, use NULL as cu pointer */
1250		free_cm = close_conn_mt(*cm, LDAP_SERVER_DOWN, &ep, NULL);
1251		if (cmg->shutting_down == B_FALSE)
1252			cu->retry = B_TRUE;
1253		(void) __ns_ldap_freeError(&ep);
1254	}
1255
1256	(void) mutex_unlock(&(*cm)->lock);
1257	if (free_cm == 1) {
1258		(void) free_conn_mt(*cm, 0);
1259		*cm = NULL;
1260	}
1261
1262	__s_api_free_server_info(&sinfo);
1263
1264	return (B_TRUE);
1265}
1266
1267/*
1268 * Check to see if a conn_mt matches the connection criteria from
1269 * a conn_user. Return B_TRUE if yes, B_FALSE, otherwise. The input
1270 * conn_mt pointer (*cmt) may be freed and *cmt will be set to NULL
1271 * to indicate so.
1272 * conn_mt *cmt and conn_mgmt cm->conn_mgmt are assumed locked.
1273 * cm->lock is unlocked at exit if rc is B_FALSE.
1274 */
1275static boolean_t
1276match_conn_mt(ns_conn_user_t *cu, ns_conn_mt_t **cmt,
1277    ns_conn_mt_state_t st, const char *server,
1278    const ns_cred_t *cred)
1279{
1280	boolean_t	matched = B_FALSE;
1281	boolean_t	drop_conn;
1282	int		free_cm = 0;
1283	ns_conn_mt_t	*cm = *cmt;
1284	ns_conn_mgmt_t	*cmg = cm->conn_mgmt;
1285
1286	if (cm->state != st || cm->close_when_nouser  == B_TRUE ||
1287	    cm->detached == B_TRUE || cm->pid != getpid() ||
1288	    cm->referral != cu->referral) {
1289		(void) mutex_unlock(&cm->lock);
1290		return (B_FALSE);
1291	}
1292
1293	/*
1294	 * if a conn_mt opened for WRITE is idle
1295	 * long enough, then close it. To improve
1296	 * the performance of applications, such
1297	 * as ldapaddent, a WRITE connection is
1298	 * given a short time to live in the
1299	 * connection pool, expecting the write
1300	 * requests to come in a quick succession.
1301	 * To save resource, the connection will
1302	 * be closed if idle more than 60 seconds.
1303	 */
1304	if (cm->opened_for == NS_CONN_USER_WRITE &&
1305	    cu->type != NS_CONN_USER_WRITE && cm->cu_cnt == 0 &&
1306	    ((time(NULL) - cm->access_time) > 60)) {
1307		/*
1308		 * NS_LDAP_INTERNAL is irrelevant here. There no
1309		 * conn_user to consume the rc
1310		 */
1311		free_cm = close_conn_mt(cm, NS_LDAP_INTERNAL, NULL, NULL);
1312		(void) mutex_unlock(&cm->lock);
1313		if (free_cm == 1) {
1314			(void) free_conn_mt(cm, 0);
1315			*cmt = NULL;
1316		}
1317		return (B_FALSE);
1318	}
1319
1320	switch (cu->type) {
1321	case NS_CONN_USER_SEARCH:
1322	case NS_CONN_USER_GETENT:
1323		if (cm->opened_for == NS_CONN_USER_SEARCH ||
1324		    cm->opened_for == NS_CONN_USER_GETENT)
1325			matched = B_TRUE;
1326		break;
1327
1328	case NS_CONN_USER_WRITE:
1329		if (cm->opened_for == NS_CONN_USER_WRITE)
1330			matched = B_TRUE;
1331		break;
1332
1333	default:
1334		matched = B_FALSE;
1335		break;
1336	}
1337
1338	if (matched == B_TRUE && ((server != NULL || cred != NULL) &&
1339	    is_server_cred_matched(server, cred, cm) == B_FALSE))
1340		matched = B_FALSE;
1341
1342	if (matched != B_FALSE) {
1343		/*
1344		 * Check and drop the 'connected' connection if
1345		 * necessary. Main nscd gets status changes from
1346		 * the ldap_cachemgr daemon directly via the
1347		 * GETSTATUSCHANGE door call, the standalone
1348		 * function works in a no ldap_cachemgr environment,
1349		 * so no need to check and drop connections.
1350		 */
1351		if (cm->state == NS_CONN_MT_CONNECTED &&
1352		    cmg->is_nscd == B_FALSE && !__s_api_isStandalone()) {
1353			drop_conn = check_and_close_conn(cmg, &cm, cu);
1354			if (drop_conn == B_TRUE) {
1355				if (cm == NULL)
1356					*cmt = NULL;
1357				return (B_FALSE);
1358			}
1359		}
1360
1361		/* check if max. users using or waiting for the connection */
1362		if ((cm->state == NS_CONN_MT_CONNECTED &&
1363		    cm->cu_max != NS_CONN_MT_USER_NO_MAX &&
1364		    cm->cu_cnt >= cm->cu_max) ||
1365		    (cm->state == NS_CONN_MT_CONNECTING &&
1366		    cm->cu_max != NS_CONN_MT_USER_NO_MAX &&
1367		    cm->waiter_cnt >= cm->cu_max - 1))
1368			matched = B_FALSE;
1369	}
1370
1371	if (matched == B_FALSE)
1372		(void) mutex_unlock(&cm->lock);
1373
1374	return (matched);
1375}
1376
1377/*
1378 * obtain an MT connection from the connection management for a conn_user
1379 *
1380 * Input:
1381 *   server	: server name or IP address
1382 *   flags	: libsldap API flags
1383 *   cred	: pointer to the user credential
1384 *   cu		: pointer to the conn_user structure
1385 * Output:
1386 *   session	: hold pointer to the Connection structure
1387 *   errorp	: hold pointer to error info (ns_ldap_error_t)
1388 */
1389int
1390__s_api_conn_mt_get(const char *server, const int flags, const ns_cred_t *cred,
1391    Connection **session, ns_ldap_error_t **errorp, ns_conn_user_t *cu)
1392{
1393	int		rc;
1394	int		i;
1395	ns_conn_mt_t	*cn;
1396	ns_conn_mt_state_t st;
1397	ns_conn_mgmt_t	*cmg;
1398
1399	if (errorp == NULL || cu == NULL || session == NULL)
1400		return (NS_LDAP_INVALID_PARAM);
1401
1402	*session = NULL;
1403	cmg = cu->conn_mgmt;
1404
1405	/*
1406	 * for pam_ldap, always try opening a new connection
1407	 */
1408	if (cu->type == NS_CONN_USER_AUTH)
1409		return (NS_LDAP_NOTFOUND);
1410
1411	/* if need a new conn, then don't reuse */
1412	if (flags & NS_LDAP_NEW_CONN)
1413		return (NS_LDAP_NOTFOUND);
1414
1415	if (flags & NS_LDAP_KEEP_CONN)
1416		cu->keep_conn = B_TRUE;
1417
1418	/*
1419	 * We want to use MT connection only if keep-connection flag is
1420	 * set or if MT was requested (or active)
1421	 */
1422	if (!((cmg->state == NS_CONN_MGMT_INACTIVE &&
1423	    cu->keep_conn == B_TRUE) || cmg->state == NS_CONN_MGMT_ACTIVE))
1424		return (NS_LDAP_NOTFOUND);
1425
1426	/* MT connection will be used now (if possible/available) */
1427	cu->use_mt_conn = B_TRUE;
1428
1429	NS_CONN_CHECK_ABORT_AND_LOCK(cmg, cu, errorp);
1430
1431	/* first look for a connection already open */
1432	st = NS_CONN_MT_CONNECTED;
1433	cu->state = NS_CONN_USER_FINDING;
1434	for (i = 0; i < 2; i++) {
1435		for (cn = cmg->cm_head; cn; cn = cn->next) {
1436			(void) mutex_lock(&cn->lock);
1437			rc = match_conn_mt(cu, &cn, st, server, cred);
1438			if (rc == B_FALSE && cn != NULL) /* not found */
1439				continue;
1440			if (cn == NULL) { /* not found and cn freed */
1441				/*
1442				 * as the conn_mt list could
1443				 * be different due to cn's
1444				 * deletion, scan the entire
1445				 * conn_mt list again
1446				 */
1447				st = NS_CONN_MT_CONNECTED;
1448				i = -1;
1449				break;
1450			}
1451
1452			/* return a connected one if found */
1453			if (cn->state == NS_CONN_MT_CONNECTED) {
1454				*session = cn->conn;
1455				add_cu2cm(cu, cn);
1456				cu->conn_mt = cn;
1457				cu->state = NS_CONN_USER_CONNECTED;
1458				(void) mutex_unlock(&cn->lock);
1459				(void) mutex_unlock(&cmg->lock);
1460				return (NS_LDAP_SUCCESS);
1461			}
1462
1463			/*
1464			 * if cn is not connecting, or allow only
1465			 * one user, skip it
1466			 */
1467			if (cn->state != NS_CONN_MT_CONNECTING ||
1468			    cn->cu_max == 1) {
1469				(void) mutex_unlock(&cn->lock);
1470				continue;
1471			}
1472
1473			/* wait for the connecting conn_mt */
1474			if (wait_for_conn_mt(cu, cn) != 1) {
1475				/*
1476				 * NS_LDAP_NOTFOUND signals that the function
1477				 * __s_api_check_libldap_MT_conn_support()
1478				 * detected that the lower libldap library
1479				 * does not support MT connection, so return
1480				 * NS_LDAP_NOTFOUND to let the caller to
1481				 * open a non-MT conneciton. Otherwise,
1482				 * connect error occurred, return
1483				 * NS_CONN_USER_CONNECT_ERROR
1484				 */
1485				if (cn->ns_rc != NS_LDAP_NOTFOUND)
1486					cu->state = NS_CONN_USER_CONNECT_ERROR;
1487				else {
1488					cu->state = NS_CONN_USER_FINDING;
1489					cu->use_mt_conn = B_FALSE;
1490				}
1491				(void) mutex_unlock(&cn->lock);
1492
1493				/* cmg->lock unlocked by wait_for_conn_mt() */
1494
1495				return (cn->ns_rc);
1496			}
1497
1498			/* return the newly available conn_mt */
1499			*session = cn->conn;
1500			cu->state = NS_CONN_USER_CONNECTED;
1501			(void) mutex_unlock(&cn->lock);
1502
1503			/* cmg->lock unlocked by wait_for_conn_mt() */
1504
1505			return (NS_LDAP_SUCCESS);
1506		}
1507
1508		/* next, look for a connecting conn_mt */
1509		if (i == 0)
1510			st = NS_CONN_MT_CONNECTING;
1511	}
1512
1513	/* no connection found, start opening one */
1514	cn = init_conn_mt(cmg, errorp);
1515	if (cn == NULL) {
1516		(void) mutex_unlock(&cmg->lock);
1517		return ((*errorp)->status);
1518	}
1519	cu->conn_mt = cn;
1520	cn->opened_for = cu->type;
1521	cn->referral = cu->referral;
1522	if (cmg->ldap_mt == B_TRUE)
1523		cn->cu_max = NS_CONN_MT_USER_MAX;
1524	else
1525		cn->cu_max = 1;
1526	add_cm2cmg(cn, cmg);
1527	(void) mutex_unlock(&cmg->lock);
1528
1529	return (NS_LDAP_NOTFOUND);
1530}
1531
1532
1533/*
1534 * add an MT connection to the connection management
1535 *
1536 * Input:
1537 *   con	: pointer to the Connection info
1538 *   cu		: pointer to the conn_user structure
1539 * Output:
1540 *   ep		: hold pointer to error info (ns_ldap_error_t)
1541 */
1542int
1543__s_api_conn_mt_add(Connection *con, ns_conn_user_t *cu, ns_ldap_error_t **ep)
1544{
1545	ns_conn_mgmt_t	*cmg = cu->conn_mgmt;
1546	ns_conn_mt_t	*cm = cu->conn_mt;
1547
1548	/* if the conn_mgmt is being shut down, return error */
1549	NS_CONN_CHECK_ABORT_AND_LOCK(cmg, cu, ep);
1550
1551	/*
1552	 * start the change monitor thread only if it
1553	 * hasn't been started and the process is the
1554	 * main nscd (not peruser nscd)
1555	 */
1556	if (cmg->procchg_started == B_FALSE && cmg->is_nscd == B_TRUE) {
1557		start_thread(cmg);
1558		cmg->procchg_started = B_TRUE;
1559	}
1560	(void) mutex_lock(&cm->lock);
1561	cm->conn = con;
1562	cm->state = NS_CONN_MT_CONNECTED;
1563	cm->pid = getpid();
1564	cm->create_time = time(NULL);
1565	cm->access_time = cm->create_time;
1566	cm->opened_for = cu->type;
1567	add_cu2cm(cu, cm);
1568	cu->conn_mt = cm;
1569	cu->state = NS_CONN_USER_CONNECTED;
1570	if (cmg->ldap_mt == B_TRUE)
1571		cm->cu_max = NS_CONN_MT_USER_MAX;
1572	else
1573		cm->cu_max = 1;
1574
1575	/* wake up the waiters if any */
1576	(void) conn_signal(cm);
1577
1578	(void) mutex_unlock(&cm->lock);
1579	(void) mutex_unlock(&cmg->lock);
1580
1581	return (NS_LDAP_SUCCESS);
1582}
1583
1584/*
1585 * return an MT connection to the pool when a conn user is done using it
1586 *
1587 * Input:
1588 *   cu		: pointer to the conn_user structure
1589 * Output:	NONE
1590 */
1591void
1592__s_api_conn_mt_return(ns_conn_user_t *cu)
1593{
1594	ns_conn_mt_t	*cm;
1595	ns_conn_mgmt_t	*cmg;
1596
1597	if (cu == NULL || cu->use_mt_conn == B_FALSE)
1598		return;
1599	cm = cu->conn_mt;
1600	if (cm == NULL)
1601		return;
1602	cmg = cu->conn_mgmt;
1603
1604	(void) mutex_lock(&cm->lock);
1605	del_cu4cm(cu, cm);
1606	cu->state = NS_CONN_USER_DISCONNECTED;
1607	cu->conn_mt = NULL;
1608	cu->bad_mt_conn = B_FALSE;
1609
1610	/*
1611	 *  if this MT connection is no longer needed, or not usable, and
1612	 * no more conn_user uses it, then close it.
1613	 */
1614
1615	if ((cm->close_when_nouser == B_TRUE ||
1616	    cm->state != NS_CONN_MT_CONNECTED) && cm->cu_cnt == 0) {
1617		(void) mutex_unlock(&cm->lock);
1618		(void) mutex_lock(&cmg->lock);
1619		(void) mutex_lock(&cm->lock);
1620		del_cm4cmg(cm, cmg);
1621		/* use ns_conn_free (instead of 1) to avoid lint warning */
1622		NS_CONN_UNLOCK_AND_FREE(ns_conn_free, cm, cmg);
1623	} else {
1624		if (cm->state == NS_CONN_MT_CONNECTED && cm->cu_cnt == 0 &&
1625		    cm->conn != NULL && cm->conn->ld != NULL) {
1626			struct timeval	zerotime;
1627			LDAPMessage	*res;
1628
1629			zerotime.tv_sec = zerotime.tv_usec = 0L;
1630			/* clean up remaining results just in case */
1631			while (ldap_result(cm->conn->ld, LDAP_RES_ANY,
1632			    LDAP_MSG_ALL, &zerotime, &res) > 0) {
1633				if (res != NULL)
1634					(void) ldap_msgfree(res);
1635			}
1636		}
1637		(void) mutex_unlock(&cm->lock);
1638	}
1639}
1640
1641/* save error info (rc and ns_ldap_error_t) in the conn_mt */
1642static void
1643err2cm(ns_conn_mt_t *cm, int rc, ns_ldap_error_t **errorp)
1644{
1645	ns_ldap_error_t	*ep;
1646
1647	cm->ns_rc = rc;
1648	cm->ns_error = NULL;
1649	if (errorp != NULL && *errorp != NULL) {
1650		ep = __s_api_copy_error(*errorp);
1651		if (ep == NULL)
1652			cm->ns_rc = NS_LDAP_MEMORY;
1653		else
1654			cm->ns_error = ep;
1655	}
1656}
1657
1658/* copy error info (rc and ns_ldap_error_t) from conn_mt to conn_user */
1659static void
1660err_from_cm(ns_conn_user_t *cu, ns_conn_mt_t *cm)
1661{
1662	ns_ldap_error_t	*ep;
1663
1664	cu->ns_rc = cm->ns_rc;
1665	if (cu->ns_error != NULL)
1666		(void) __ns_ldap_freeError(&cu->ns_error);
1667	cu->ns_error = NULL;
1668	if (cm->ns_rc != NS_LDAP_SUCCESS && cm->ns_error != NULL) {
1669		ep = __s_api_copy_error(cm->ns_error);
1670		if (ep == NULL)
1671			cu->ns_rc = NS_LDAP_MEMORY;
1672		else
1673			cu->ns_error = ep;
1674	}
1675}
1676
1677/* copy error info (rc and ns_ldap_error_t) from caller to conn_user */
1678static void
1679err_from_caller(ns_conn_user_t *cu, int rc, ns_ldap_error_t **errorp)
1680{
1681
1682	cu->ns_rc = rc;
1683	if (errorp != NULL) {
1684		if (cu->ns_error != NULL)
1685			(void) __ns_ldap_freeError(&cu->ns_error);
1686		cu->ns_error = *errorp;
1687		*errorp = NULL;
1688	} else
1689		cu->ns_error = NULL;
1690}
1691
1692/*
1693 * remove an MT connection from the connection management when failed to open
1694 *
1695 * Input:
1696 *   cu		: pointer to the conn_user structure
1697 *   rc		: error code
1698 *   errorp	: pointer to pointer to error info (ns_ldap_error_t)
1699 * Output:
1700 *   errorp	: set to NULL, if none NULL cm, callers do not need to free it
1701 */
1702void
1703__s_api_conn_mt_remove(ns_conn_user_t *cu, int rc, ns_ldap_error_t **errorp)
1704{
1705	ns_conn_mgmt_t	*cmg;
1706	ns_conn_mt_t	*cm;
1707	int		free_cm = 0;
1708
1709	if (cu == NULL || cu->use_mt_conn == B_FALSE)
1710		return;
1711	if ((cm = cu->conn_mt) == NULL)
1712		return;
1713	cmg = cu->conn_mgmt;
1714
1715	(void) mutex_lock(&cmg->lock);
1716	(void) mutex_lock(&cm->lock);
1717	if (cm->state != NS_CONN_MT_CONNECT_ERROR) {
1718		cm->state = NS_CONN_MT_CONNECT_ERROR;
1719		cm->ns_rc = rc;
1720		if (errorp != NULL) {
1721			cm->ns_error = *errorp;
1722			*errorp = NULL;
1723		}
1724	}
1725
1726	/* all the conn_users share the same error rc and ns_ldap_error_t */
1727	err_from_cm(cu, cm);
1728	/* wake up the waiters if any */
1729	(void) conn_signal(cm);
1730
1731	del_cu4cm(cu, cm);
1732	cu->conn_mt = NULL;
1733	cu->bad_mt_conn = B_FALSE;
1734	if (cm->cu_cnt == 0) {
1735		del_cm4cmg(cm, cmg);
1736		free_cm = 1;
1737	}
1738
1739	NS_CONN_UNLOCK_AND_FREE(free_cm, cm, cmg);
1740}
1741
1742/*
1743 * check to see if the underlying libldap supports multi-threaded client
1744 * (MT connections)
1745 */
1746int
1747__s_api_check_libldap_MT_conn_support(ns_conn_user_t *cu, LDAP *ld,
1748    ns_ldap_error_t **ep)
1749{
1750	int		rc;
1751	ns_conn_mgmt_t	*cmg;
1752
1753	/* if no need to check, just return success */
1754	if (cu->conn_mt == NULL || cu->use_mt_conn == B_FALSE)
1755		return (NS_LDAP_SUCCESS);
1756
1757	cmg = cu->conn_mgmt;
1758	rc = setup_mt_ld(ld, cmg);
1759
1760	if (cmg->do_mt_conn == B_FALSE) {
1761		/*
1762		 * If the conn_mgmt is being shut down, return error.
1763		 * if cmg is usable, cmg->lock will be locked. Otherwise,
1764		 * this function will return with rc NS_LDAP_OP_FAILED.
1765		 */
1766		NS_CONN_CHECK_ABORT_AND_LOCK(cmg, cu, ep);
1767		if (cmg->do_mt_conn == B_FALSE) {
1768			if (rc < 0)
1769				cmg->ldap_mt = B_FALSE;
1770			else {
1771				cmg->ldap_mt = B_TRUE;
1772				if (cmg->is_nscd  == B_TRUE ||
1773				    cmg->is_peruser_nscd == B_TRUE) {
1774					cmg->do_mt_conn = B_TRUE;
1775					cmg->state = NS_CONN_MGMT_ACTIVE;
1776				}
1777			}
1778		}
1779		(void) mutex_unlock(&cmg->lock);
1780	}
1781
1782	if (rc < 0)
1783		__s_api_conn_mt_remove(cu, NS_LDAP_NOTFOUND, NULL);
1784	return (NS_LDAP_SUCCESS);
1785}
1786
1787/*
1788 * Close an MT connection.
1789 * Assume cm not null and locked, assume conn_mgmt is also locked.
1790 * Return -1 if error, 1 if the cm should be freed, otherwise 0.
1791 */
1792static int
1793close_conn_mt(ns_conn_mt_t *cm, int rc, ns_ldap_error_t **errorp,
1794    ns_conn_user_t *cu)
1795{
1796	ns_conn_mgmt_t	*cmg = cm->conn_mgmt;
1797	ns_conn_mt_t	*m;
1798	ns_conn_user_t	*u;
1799
1800	if ((cm->state != NS_CONN_MT_CONNECTED && cm->state !=
1801	    NS_CONN_MT_CLOSING) || cmg->cm_head == NULL || cmg->cm_cnt == 0)
1802		return (-1);
1803
1804	/* if the conn_mt is not in the MT connection pool, nothing to do */
1805	for (m = cmg->cm_head; m; m = m->next) {
1806		if (cm == m)
1807			break;
1808	}
1809	if (m == NULL)
1810		return (-1);
1811
1812	if (cm->state == NS_CONN_MT_CONNECTED) { /* first time in here */
1813		cm->state = NS_CONN_MT_CLOSING;
1814		/*
1815		 * If more cu exist to consume the error info, copy
1816		 * it to the cm. If the caller calls on behalf of
1817		 * a cu, cu won't be NULL. Check to see if there's
1818		 * more cu that needs the error info. If caller does
1819		 * not have a specific cu attached to it (e.g.,
1820		 * shutdown_all_conn_mt()), cu is NULL, check if at
1821		 * least one cu exists.
1822		 */
1823		if ((cu != NULL && cm->cu_cnt > 1) ||
1824		    (cu == NULL && cm->cu_cnt > 0)) {
1825			err2cm(cm, rc, errorp);
1826			/* wake up waiter (conn_user) if any */
1827			(void) conn_signal(cm);
1828		}
1829
1830		/* for each conn_user using the conn_mt, set bad_mt_conn flag */
1831		if (cm->cu_head != NULL) {
1832			for (u = cm->cu_head; u; u = u->next) {
1833				u->bad_mt_conn = B_TRUE;
1834				if (cmg->shutting_down == B_FALSE)
1835					u->retry = B_TRUE;
1836			}
1837		}
1838	}
1839
1840	/* detach the conn_mt if no more conn_user left */
1841	if ((cu != NULL && cm->cu_cnt == 1) ||
1842	    (cu == NULL && cm->cu_cnt ==  0)) {
1843		del_cm4cmg(cm, cmg);
1844		cm->detached = B_TRUE;
1845		return (1);
1846	}
1847
1848	return (0);
1849}
1850
1851/*
1852 * An MT connection becomes bad, close it and free resources.
1853 * This function is called with a ns_conn_user_t representing
1854 * a user of the MT connection.
1855 *
1856 * Input:
1857 *   cu		: pointer to the conn_user structure
1858 *   rc		: error code
1859 *   errorp	: pointer to pointer to error info (ns_ldap_error_t)
1860 * Output:
1861 *   errorp	: set to NULL (if no error), callers do not need to free it
1862 */
1863void
1864__s_api_conn_mt_close(ns_conn_user_t *cu, int rc, ns_ldap_error_t **errorp)
1865{
1866	ns_conn_mgmt_t	*cmg;
1867	ns_conn_mt_t	*cm;
1868	int		free_cm = 0;
1869
1870	if (cu == NULL || cu->use_mt_conn == B_FALSE)
1871		return;
1872
1873	if (cu->state != NS_CONN_USER_CONNECTED || (cm = cu->conn_mt) == NULL)
1874		return;
1875	cmg = cu->conn_mgmt;
1876
1877	(void) mutex_lock(&cmg->lock);
1878	(void) mutex_lock(&cm->lock);
1879
1880	/* close the MT connection if possible */
1881	free_cm = close_conn_mt(cm, rc, errorp, cu);
1882	if (free_cm == -1) { /* error case */
1883		(void) mutex_unlock(&cm->lock);
1884		(void) mutex_unlock(&cmg->lock);
1885		return;
1886	}
1887
1888	if (rc != NS_LDAP_SUCCESS) { /* error info passed in, use it */
1889		err_from_caller(cu, rc, errorp);
1890	} else { /* error not passed in, use those saved in the conn_mt */
1891		err_from_cm(cu, cm);
1892	}
1893
1894	/* detach the conn_user from the conn_mt */
1895	del_cu4cm(cu, cm);
1896	cu->conn_mt = NULL;
1897	cu->bad_mt_conn = B_FALSE;
1898	if (cmg->shutting_down == B_FALSE)
1899		cu->retry = B_TRUE;
1900	NS_CONN_UNLOCK_AND_FREE(free_cm, cm, cmg);
1901}
1902
1903/*
1904 * Close an MT connection when the associated server is known to be
1905 * down. This function is called with a ns_conn_mt_t representing
1906 * the MT connection. That is, the caller is not a conn_user
1907 * thread but rather the procchg thread.
1908 */
1909static void
1910close_conn_mt_by_procchg(ns_conn_mt_t *cm, int rc, char *errmsg)
1911{
1912	ns_conn_mgmt_t	*cmg;
1913	int		free_cm = 0;
1914	ns_ldap_error_t	*ep;
1915
1916	if (cm == NULL)
1917		return;
1918	cmg = cm->conn_mgmt;
1919
1920	ep = (ns_ldap_error_t *)calloc(1, sizeof (*ep));
1921	if (ep != NULL) {
1922		ep->status = rc;
1923		if (errmsg != NULL)
1924			ep->message =  strdup(errmsg); /* OK if returns NULL */
1925	}
1926
1927	(void) mutex_lock(&cmg->lock);
1928	(void) mutex_lock(&cm->lock);
1929
1930	/* close the MT connection if possible */
1931	free_cm = close_conn_mt(cm, LDAP_SERVER_DOWN, &ep, NULL);
1932	if (free_cm == -1) { /* error case */
1933		(void) mutex_unlock(&cm->lock);
1934		(void) mutex_unlock(&cmg->lock);
1935		return;
1936	}
1937	(void) __ns_ldap_freeError(&ep);
1938
1939	NS_CONN_UNLOCK_AND_FREE(free_cm, cm, cmg);
1940}
1941
1942/*
1943 * Close an MT connection when there is a better server to connect to.
1944 * Mark the connection as to-be-closed-when-no-one-using so that
1945 * any outstanding ldap operations can run to completion.
1946 * Assume that both the conn_mt and conn_mgmt are locked.
1947 * Return 1 if the conn_mt should be freed.
1948 */
1949static int
1950close_conn_mt_when_nouser(ns_conn_mt_t *cm)
1951{
1952	int		free_cm = 0;
1953
1954	if (cm->cu_cnt == 0) {
1955		del_cm4cmg(cm, cm->conn_mgmt);
1956		free_cm = 1;
1957	} else {
1958		cm->close_when_nouser = B_TRUE;
1959	}
1960
1961	return (free_cm);
1962}
1963
1964/*
1965 * Retrieve the configured preferred server list.
1966 * This function locked the conn_mgmt and does not
1967 * unlock at exit.
1968 */
1969static void
1970get_preferred_servers(boolean_t lock, boolean_t reload, ns_conn_mgmt_t *cmg)
1971{
1972	ns_ldap_error_t *errorp = NULL;
1973	void		**pservers = NULL;
1974
1975	if (lock == B_TRUE)
1976		(void) mutex_lock(&cmg->lock);
1977
1978	/* if already done, and no reload, then return */
1979	if (cmg->pservers_loaded == B_TRUE && reload == B_FALSE)
1980		return;
1981
1982	if (cmg->pservers != NULL) {
1983		(void) __ns_ldap_freeParam((void ***)&cmg->pservers);
1984		cmg->pservers = NULL;
1985	}
1986
1987	if (__ns_ldap_getParam(NS_LDAP_SERVER_PREF_P,
1988	    &pservers, &errorp) == NS_LDAP_SUCCESS) {
1989		cmg->pservers = (char **)pservers;
1990		cmg->pservers_loaded = B_TRUE;
1991	} else {
1992		(void) __ns_ldap_freeError(&errorp);
1993		(void) __ns_ldap_freeParam(&pservers);
1994	}
1995}
1996
1997/*
1998 * This function handles the config or server status change notification
1999 * from the ldap_cachemgr.
2000 */
2001static ns_conn_mgmt_t *
2002proc_server_change(ns_server_status_change_t *chg, ns_conn_mgmt_t  *cmg)
2003{
2004	int		cnt, i, j, k, n;
2005	boolean_t	loop = B_TRUE;
2006	boolean_t	cmg_locked = B_FALSE;
2007	char		*s;
2008	ns_conn_mt_t	*cm;
2009	ns_conn_mgmt_t	*ocmg;
2010
2011	/* if config changed, reload the configuration */
2012	if (chg->config_changed == B_TRUE) {
2013		/* reload the conn_mgmt and Native LDAP config */
2014		ocmg = access_conn_mgmt(NS_CONN_MGMT_OP_RELOAD_CONFIG);
2015		shutdown_all_conn_mt(ocmg);
2016		/* release the one obtained from access_conn_mgmt(RELOAD) */
2017		(void) release_conn_mgmt(ocmg, B_FALSE);
2018		/* release the one obtained when ocmg was created */
2019		(void) release_conn_mgmt(ocmg, B_FALSE);
2020		return (ocmg);
2021	}
2022
2023	if ((cnt = chg->num_server) == 0)
2024		return (cmg);
2025
2026	/* handle down servers first */
2027	for (i = 0; i < cnt; i++) {
2028
2029		if (chg->changes[i] != NS_SERVER_DOWN)
2030			continue;
2031		s = chg->servers[i];
2032
2033		/*
2034		 * look for a CONNECTED MT connection using
2035		 * the same server s, and close it
2036		 */
2037		while (loop) {
2038			if (cmg_locked == B_FALSE) {
2039				(void) mutex_lock(&cmg->lock);
2040				cmg_locked = B_TRUE;
2041			}
2042			for (cm = cmg->cm_head; cm; cm = cm->next) {
2043				(void) mutex_lock(&cm->lock);
2044
2045				if (cm->state == NS_CONN_MT_CONNECTED &&
2046				    cm->conn != NULL &&
2047				    strcasecmp(cm->conn->serverAddr, s) == 0) {
2048					(void) mutex_unlock(&cm->lock);
2049					break;
2050				}
2051
2052				(void) mutex_unlock(&cm->lock);
2053			}
2054			if (cm != NULL) {
2055				(void) mutex_unlock(&cmg->lock);
2056				cmg_locked = B_FALSE;
2057				close_conn_mt_by_procchg(cm, LDAP_SERVER_DOWN,
2058				    NS_CONN_MSG_DOWN_FROM_CACHEMGR);
2059				/*
2060				 * Process the next cm using server s.
2061				 * Start from the head of the cm linked
2062				 * list again, as the cm list may change
2063				 * after close_conn_mt_by_procchg() is done.
2064				 */
2065				continue;
2066			}
2067
2068			/*
2069			 * No (more) MT connection using the down server s.
2070			 * Process the next server on the list.
2071			 */
2072			break;
2073		} /* while loop */
2074	}
2075
2076	/*
2077	 * Next handle servers whose status changed to up.
2078	 * Get the preferred server list first if not done yet.
2079	 * get_preferred_servers() leaves conn_mgmt locked.
2080	 */
2081	get_preferred_servers(cmg_locked == B_FALSE ? B_TRUE : B_FALSE,
2082	    B_FALSE, cmg);
2083	cmg_locked = B_TRUE;
2084	/*
2085	 * if no preferred server configured, we don't switch MT connection
2086	 * to a more preferred server (i.e., fallback), so just return
2087	 */
2088	if (cmg->pservers == NULL) {
2089		(void) mutex_unlock(&cmg->lock);
2090		return (cmg);
2091	}
2092
2093	/* for each server that is up now */
2094	for (i = 0; i < cnt; i++) {
2095		if (chg->changes[i] != NS_SERVER_UP)
2096			continue;
2097		s = chg->servers[i];
2098
2099		/*
2100		 * look for a CONNECTED MT connection which uses
2101		 * a server less preferred than s, and treat it
2102		 * as 'fallback needed' by calling
2103		 * close_conn_mt_when_nouser()
2104		 */
2105		k = -1;
2106		loop = B_TRUE;
2107		while (loop) {
2108			if (cmg_locked == B_FALSE) {
2109				(void) mutex_lock(&cmg->lock);
2110				cmg_locked = B_TRUE;
2111			}
2112
2113			/* Is s a preferred server ? */
2114			if (k == -1) {
2115				for (j = 0; cmg->pservers[j] != NULL; j++) {
2116					if (strcasecmp(cmg->pservers[j],
2117					    s) == 0) {
2118						k = j;
2119						break;
2120					}
2121				}
2122			}
2123			/* skip s if not a preferred server */
2124			if (k == -1) {
2125				break;
2126			}
2127
2128			/* check each MT connection */
2129			for (cm = cmg->cm_head; cm; cm = cm->next) {
2130				(void) mutex_lock(&cm->lock);
2131				/*
2132				 * Find an MT connection that is connected and
2133				 * not marked, but leave WRITE or REFERRAL
2134				 * connections alone, since fallback does not
2135				 * make sense for them.
2136				 */
2137				if (cm->state == NS_CONN_MT_CONNECTED &&
2138				    cm->close_when_nouser == B_FALSE &&
2139				    cm->conn != NULL && cm->opened_for !=
2140				    NS_CONN_USER_WRITE &&
2141				    cm->referral == B_FALSE) {
2142					n = -1;
2143					/*
2144					 * j < k ??? should we close
2145					 * an active MT that is using s ?
2146					 * ie could s went down and up
2147					 * again, but cm is bound prior to
2148					 * the down ? Play safe here,
2149					 * and check j <= k.
2150					 */
2151					for (j = 0; j <= k; j++) {
2152						if (strcasecmp(
2153						    cm->conn->serverAddr,
2154						    cmg->pservers[j]) == 0) {
2155							n = j;
2156							break;
2157						}
2158					}
2159					/*
2160					 * s is preferred, if its location
2161					 * in the preferred server list is
2162					 * ahead of that of the server
2163					 * used by the cm (i.e., no match
2164					 * found before s)
2165					 */
2166					if (n == -1) { /* s is preferred */
2167						int fr = 0;
2168						fr = close_conn_mt_when_nouser(
2169						    cm);
2170						NS_CONN_UNLOCK_AND_FREE(fr,
2171						    cm, cmg);
2172						cmg_locked = B_FALSE;
2173						/*
2174						 * break, not continue,
2175						 * because we need to
2176						 * check the entire cm
2177						 * list again. The call
2178						 * above may change the
2179						 * cm list.
2180						 */
2181						break;
2182					}
2183				}
2184				(void) mutex_unlock(&cm->lock);
2185			}
2186			/* if no (more) cm using s, check next server */
2187			if (cm == NULL)
2188				loop = B_FALSE;
2189		} /* while loop */
2190	}
2191	if (cmg_locked == B_TRUE)
2192		(void) mutex_unlock(&cmg->lock);
2193	return (cmg);
2194}
2195
2196/* Shut down all MT connection managed by the connection management */
2197void
2198shutdown_all_conn_mt(ns_conn_mgmt_t  *cmg)
2199{
2200	ns_ldap_error_t	*ep;
2201	ns_conn_mt_t	*cm;
2202	int		free_cm = 0;
2203	boolean_t	done = B_FALSE;
2204
2205	ep = (ns_ldap_error_t *)calloc(1, sizeof (*ep));
2206	if (ep != NULL) { /* if NULL, not a problem */
2207		/* OK if returns NULL */
2208		ep->message = strdup(NS_CONN_MSG_SHUTDOWN_RELOADED);
2209	}
2210
2211	(void) mutex_lock(&cmg->lock);
2212	while (cmg->cm_head != NULL && done == B_FALSE) {
2213		for (cm = cmg->cm_head; cm; cm = cm->next) {
2214			(void) mutex_lock(&cm->lock);
2215			if (cm->next == NULL)
2216				done = B_TRUE;
2217			/* shut down each conn_mt, ignore errors */
2218			free_cm = close_conn_mt(cm, LDAP_OTHER, &ep, NULL);
2219			(void) mutex_unlock(&cm->lock);
2220			if (free_cm == 1) {
2221				(void) free_conn_mt(cm, 0);
2222				/*
2223				 * conn_mt may change, so start from
2224				 * top of list again
2225				 */
2226				break;
2227			}
2228		}
2229	}
2230	(void) mutex_unlock(&cmg->lock);
2231	(void) __ns_ldap_freeError(&ep);
2232}
2233
2234/* free all the resources used by the connection management */
2235void
2236__s_api_shutdown_conn_mgmt()
2237{
2238	ns_conn_mgmt_t	*cmg;
2239
2240	cmg = access_conn_mgmt(NS_CONN_MGMT_OP_SHUTDOWN);
2241	if (cmg == NULL) /* already being SHUT done */
2242		return;
2243
2244	(void) shutdown_all_conn_mt(cmg);
2245	(void) release_conn_mgmt(cmg, B_FALSE);
2246
2247	/* then destroy the conn_mgmt */
2248	(void) release_conn_mgmt(cmg, B_FALSE);
2249}
2250
2251
2252/*
2253 * Reinitialize the libsldap connection management after
2254 * a new native LDAP configuration is received.
2255 */
2256void
2257__s_api_reinit_conn_mgmt_new_config(ns_config_t *new_cfg)
2258{
2259	ns_conn_mgmt_t	*cmg;
2260	ns_conn_mgmt_t	*ocmg;
2261
2262	cmg = access_conn_mgmt(NS_CONN_MGMT_OP_REF);
2263	if (cmg == NULL)
2264		return;
2265	if (cmg->config == new_cfg || cmg->state == NS_CONN_MGMT_DETACHED) {
2266		(void) release_conn_mgmt(cmg, B_FALSE);
2267		return;
2268	}
2269
2270	/* reload the conn_mgmt and native LDAP config */
2271	ocmg = access_conn_mgmt(NS_CONN_MGMT_OP_NEW_CONFIG);
2272	if (ocmg == cmg)
2273		shutdown_all_conn_mt(ocmg);
2274	/* release the one obtained from access_conn_mgmt(RELOAD) */
2275	(void) release_conn_mgmt(ocmg, B_FALSE);
2276	/* release the one obtained when ocmg was created */
2277	(void) release_conn_mgmt(ocmg, B_FALSE);
2278	/* release the one obtained when this function is entered */
2279	(void) release_conn_mgmt(cmg, B_FALSE);
2280}
2281
2282/*
2283 * Prepare to retry ldap search operation if needed.
2284 * Return 1 if retry is needed, otherwise 0.
2285 * If first time in, return 1. If not, return 1 if:
2286 * - not a NS_CONN_USER_GETENT conn_user AND
2287 * - have not retried 3 times yet AND
2288 * - previous search failed AND
2289 * - the retry flag is set in the ns_conn_user_t or config was reloaded
2290 */
2291int
2292__s_api_setup_retry_search(ns_conn_user_t **conn_user,
2293    ns_conn_user_type_t type, int *try_cnt, int *rc,
2294    ns_ldap_error_t **errorp)
2295{
2296	boolean_t	retry;
2297	ns_conn_user_t	*cu = *conn_user;
2298	ns_conn_mgmt_t	*cmg;
2299
2300	if (*try_cnt > 0 && cu != NULL) {
2301		/*
2302		 * if called from firstEntry(), keep conn_mt for
2303		 * the subsequent getnext requests
2304		 */
2305		if (cu->type == NS_CONN_USER_GETENT && *rc == NS_LDAP_SUCCESS)
2306			return (0);
2307		cmg = cu->conn_mgmt;
2308		retry = cu->retry;
2309		if (cu->conn_mt != NULL)
2310			__s_api_conn_mt_return(cu);
2311		if (cmg != NULL && cmg->cfg_reloaded == B_TRUE)
2312			retry = B_TRUE;
2313		__s_api_conn_user_free(cu);
2314		*conn_user = NULL;
2315
2316		if (*rc == NS_LDAP_SUCCESS || retry != B_TRUE)
2317			return (0);
2318	}
2319
2320	*try_cnt = *try_cnt + 1;
2321	if (*try_cnt > NS_LIST_TRY_MAX)
2322		return (0);
2323
2324	*conn_user = __s_api_conn_user_init(type, NULL, B_FALSE);
2325	if (*conn_user == NULL) {
2326		if (*try_cnt == 1) { /* first call before any retry */
2327			*rc = NS_LDAP_MEMORY;
2328			*errorp = NULL;
2329		}
2330		/* for 1+ try, use previous rc and errorp */
2331		return (0);
2332	}
2333
2334	/* free ldap_error_t from previous search */
2335	if (*try_cnt > 1 && rc != NS_LDAP_SUCCESS && *errorp != NULL)
2336		(void) __ns_ldap_freeError(errorp);
2337
2338	return (1);
2339}
2340
2341/* prepare to get the next entry for an enumeration */
2342int
2343__s_api_setup_getnext(ns_conn_user_t *cu, int *ns_err,
2344    ns_ldap_error_t **errorp)
2345{
2346	int rc;
2347	ns_conn_mgmt_t	*cmg;
2348
2349	/*
2350	 * if using an MT connection, ensure the thread-specific data are set,
2351	 * but if the MT connection is no longer good, return the error saved.
2352	 */
2353	if (cu->conn_mt != NULL && (cmg = cu->conn_mgmt) != NULL) {
2354
2355		if (cu->bad_mt_conn ==  B_TRUE) {
2356			__s_api_conn_mt_close(cu, 0, NULL);
2357			*ns_err = cu->ns_rc;
2358			*errorp = cu->ns_error;
2359			cu->ns_error = NULL;
2360			return (*ns_err);
2361		}
2362
2363		rc = conn_tsd_check(cmg);
2364		if (rc != NS_LDAP_SUCCESS) {
2365			*errorp = NULL;
2366			return (rc);
2367		}
2368	}
2369
2370	return (NS_LDAP_SUCCESS);
2371}
2372
2373/* wait for an MT connection to become available */
2374static int
2375conn_wait(ns_conn_mt_t *conn_mt, ns_conn_user_t *conn_user)
2376{
2377	ns_conn_waiter_t	mywait;
2378	ns_conn_waiter_t	*head = &conn_mt->waiter;
2379
2380	(void) cond_init(&(mywait.waitcv), USYNC_THREAD, 0);
2381	mywait.key = conn_user;
2382	mywait.signaled = 0;
2383	mywait.next = head->next;
2384	mywait.prev = head;
2385	if (mywait.next)
2386		mywait.next->prev = &mywait;
2387	head->next = &mywait;
2388	atomic_inc_uint(&conn_mt->waiter_cnt);
2389
2390	while (!mywait.signaled)
2391		(void) cond_wait(&(mywait.waitcv), &conn_mt->lock);
2392	if (mywait.prev)
2393		mywait.prev->next = mywait.next;
2394	if (mywait.next)
2395		mywait.next->prev = mywait.prev;
2396	return (0);
2397}
2398
2399/* signal that an MT connection is now available */
2400static int
2401conn_signal(ns_conn_mt_t *conn_mt)
2402{
2403	int			c = 0;
2404	ns_conn_waiter_t	*head = &conn_mt->waiter;
2405	ns_conn_waiter_t	*tmp = head->next;
2406
2407	while (tmp) {
2408		(void) cond_signal(&(tmp->waitcv));
2409		tmp->signaled = 1;
2410		atomic_dec_uint(&conn_mt->waiter_cnt);
2411		c++;
2412		tmp = tmp->next;
2413	}
2414
2415	return (c);
2416}
2417
2418/*
2419 * wait and process the server status and/or config change notification
2420 * from ldap_cachemgr
2421 */
2422static void *
2423get_server_change(void *arg)
2424{
2425	union {
2426		ldap_data_t	s_d;
2427		char		s_b[DOORBUFFERSIZE];
2428	} space;
2429	ldap_data_t	*sptr = &space.s_d;
2430	int		ndata;
2431	int		adata;
2432	char		*ptr;
2433	int		ds_cnt;
2434	int		door_rc;
2435	int		which;
2436	int		retry = 0;
2437	boolean_t	loop = B_TRUE;
2438	char		*c, *oc;
2439	int		dslen = strlen(DOORLINESEP);
2440	char		dsep = DOORLINESEP_CHR;
2441	char		chg_data[DOORBUFFERSIZE];
2442	char		**servers = NULL;
2443	boolean_t	getchg_not_supported = B_FALSE;
2444	ns_conn_mgmt_t	*ocmg = (ns_conn_mgmt_t *)arg;
2445	ns_conn_mgmt_t	*cmg;
2446	ns_server_status_t *status = NULL;
2447	ns_server_status_change_t chg = { 0 };
2448	ldap_get_change_out_t *get_chg;
2449	ldap_get_chg_cookie_t cookie;
2450	ldap_get_chg_cookie_t new_cookie;
2451
2452	cmg = access_conn_mgmt(NS_CONN_MGMT_OP_REF);
2453	if (cmg != ocmg)
2454		thr_exit(NULL);
2455	/* cmg is locked before called */
2456	cmg->procchg_tid = thr_self();
2457
2458	/* make sure the thread specific data are set */
2459	(void) conn_tsd_setup(cmg);
2460	cookie = cmg->cfg_cookie;
2461
2462	while (loop) {
2463
2464		if (chg.servers != NULL)
2465			free(chg.servers);
2466		if (chg.changes != NULL)
2467			free(chg.changes);
2468		if (sptr != &space.s_d)
2469			(void) munmap((char *)sptr, sizeof (space));
2470
2471		/*
2472		 * If the attached conn_mgmt has been deleted,
2473		 * then exit. The new conn_mgmt will starts it
2474		 * own monitor thread later. If libsldap is being
2475		 * unloaded or configuration reloaded, OR
2476		 * ldap_cachemgr rejected the GETSTATUSCHANGE door
2477		 * call, then exit as well.
2478		 */
2479		if (cmg == NULL || cmg->state == NS_CONN_MGMT_DETACHED ||
2480		    getchg_not_supported == B_TRUE) {
2481
2482			if (cmg != NULL) {
2483				cmg->procchg_started = B_FALSE;
2484				(void) release_conn_mgmt(cmg, B_FALSE);
2485			}
2486
2487			conn_tsd_free();
2488			thr_exit(NULL);
2489		}
2490
2491		(void) memset(space.s_b, 0, DOORBUFFERSIZE);
2492		(void) memset(&chg, 0, sizeof (chg));
2493		adata = sizeof (ldap_call_t) + 1;
2494		ndata = sizeof (space);
2495		space.s_d.ldap_call.ldap_callnumber = GETSTATUSCHANGE;
2496		space.s_d.ldap_call.ldap_u.get_change.op =
2497		    NS_STATUS_CHANGE_OP_START;
2498		space.s_d.ldap_call.ldap_u.get_change.cookie = cookie;
2499		sptr = &space.s_d;
2500		door_rc = __ns_ldap_trydoorcall_getfd();
2501		cmg->procchg_door_call = B_TRUE;
2502		if (release_conn_mgmt(cmg, B_FALSE) == NULL) {
2503			conn_tsd_free();
2504			thr_exit(NULL);
2505		}
2506
2507		if (door_rc == NS_CACHE_SUCCESS)
2508			door_rc = __ns_ldap_trydoorcall_send(&sptr, &ndata,
2509			    &adata);
2510
2511		/*
2512		 * Check and see if the conn_mgmt is still current.
2513		 * If not, no need to continue.
2514		 */
2515		cmg = access_conn_mgmt(NS_CONN_MGMT_OP_REF);
2516		if (cmg != NULL)
2517			cmg->procchg_door_call = B_FALSE;
2518		if (cmg != ocmg) {
2519			if (cmg != NULL) {
2520				cmg->procchg_started = B_FALSE;
2521				(void) release_conn_mgmt(cmg, B_FALSE);
2522			}
2523			conn_tsd_free();
2524			thr_exit(NULL);
2525		}
2526
2527		if (door_rc != NS_CACHE_SUCCESS) {
2528			if (door_rc == NS_CACHE_NOSERVER) {
2529				if (retry++ > 10)
2530					getchg_not_supported = B_TRUE;
2531				else {
2532					/*
2533					 * ldap_cachemgr may be down, give
2534					 * it time to restart
2535					 */
2536					(void) sleep(2);
2537				}
2538			} else if (door_rc == NS_CACHE_NOTFOUND)
2539				getchg_not_supported = B_TRUE;
2540			continue;
2541		} else
2542			retry = 0;
2543
2544		/* copy info from door call return structure */
2545		get_chg =  &sptr->ldap_ret.ldap_u.changes;
2546		ptr = get_chg->data;
2547		/* configuration change ? */
2548		if (get_chg->type == NS_STATUS_CHANGE_TYPE_CONFIG) {
2549			chg.config_changed = B_TRUE;
2550			cmg = proc_server_change(&chg, cmg);
2551			continue;
2552		}
2553
2554		/* server status changes ? */
2555		if (get_chg->type == NS_STATUS_CHANGE_TYPE_SERVER) {
2556			/*
2557			 * first check cookies, if don't match, config
2558			 * has changed
2559			 */
2560			new_cookie = get_chg->cookie;
2561			if (new_cookie.mgr_pid != cookie.mgr_pid ||
2562			    new_cookie.seq_num != cookie.seq_num) {
2563				chg.config_changed = B_TRUE;
2564				cmg = proc_server_change(&chg, cmg);
2565				continue;
2566			}
2567
2568			(void) strlcpy(chg_data, ptr, sizeof (chg_data));
2569			chg.num_server = get_chg->server_count;
2570
2571			servers = (char **)calloc(chg.num_server,
2572			    sizeof (char *));
2573			if (servers == NULL) {
2574				syslog(LOG_INFO, NS_CONN_MSG_MEMORY_ERROR);
2575				continue;
2576			}
2577			status = (ns_server_status_t *)calloc(chg.num_server,
2578			    sizeof (int));
2579			if (status == NULL) {
2580				syslog(LOG_INFO, NS_CONN_MSG_MEMORY_ERROR);
2581				free(servers);
2582				continue;
2583			}
2584			ds_cnt = 0;
2585			which = 0;
2586			oc = ptr;
2587			for (c = ptr; which != 2; c++) {
2588				/* look for DOORLINESEP or end of string */
2589				if (*c != dsep && *c != '\0')
2590					continue;
2591				if (*c == dsep) { /* DOORLINESEP */
2592					*c = '\0'; /* current value */
2593					c += dslen; /* skip to next value */
2594				}
2595				if (which == 0) { /* get server info */
2596					servers[ds_cnt] = oc;
2597					oc = c;
2598					which = 1; /* get status next */
2599					continue;
2600				}
2601				/* which == 1, get up/down status */
2602				if (strcmp(NS_SERVER_CHANGE_UP, oc) == 0) {
2603					status[ds_cnt] = NS_SERVER_UP;
2604				} else if (strcmp(NS_SERVER_CHANGE_DOWN,
2605				    oc) == 0)
2606					status[ds_cnt] = NS_SERVER_DOWN;
2607				else {
2608					syslog(LOG_INFO,
2609					    NS_CONN_MSG_BAD_CACHEMGR_DATA);
2610					continue;
2611				}
2612				oc = c;
2613				ds_cnt++;
2614				if (*c == '\0')
2615					which = 2; /* exit the loop */
2616				else
2617					which = 0; /* get server info next */
2618			}
2619			chg.servers = servers;
2620			chg.changes = status;
2621			cmg = proc_server_change(&chg, cmg);
2622			continue;
2623		}
2624	}
2625
2626	return (NULL);
2627}
2628
2629/* start the thread handling the change notification from ldap_cachemgr */
2630static void
2631start_thread(ns_conn_mgmt_t *cmg)
2632{
2633
2634	int		errnum;
2635
2636	/*
2637	 * start a thread to get and process config and server status changes
2638	 */
2639	if (thr_create(NULL, 0, get_server_change,
2640	    (void *)cmg, THR_DETACHED, NULL) != 0) {
2641		errnum = errno;
2642		syslog(LOG_WARNING, NS_CONN_MSG_NO_PROCCHG_THREAD,
2643		    strerror(errnum));
2644	}
2645}
2646