1dbed73cbSSangeeta Misra /*
2dbed73cbSSangeeta Misra  * CDDL HEADER START
3dbed73cbSSangeeta Misra  *
4dbed73cbSSangeeta Misra  * The contents of this file are subject to the terms of the
5dbed73cbSSangeeta Misra  * Common Development and Distribution License (the "License").
6dbed73cbSSangeeta Misra  * You may not use this file except in compliance with the License.
7dbed73cbSSangeeta Misra  *
8dbed73cbSSangeeta Misra  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9dbed73cbSSangeeta Misra  * or http://www.opensolaris.org/os/licensing.
10dbed73cbSSangeeta Misra  * See the License for the specific language governing permissions
11dbed73cbSSangeeta Misra  * and limitations under the License.
12dbed73cbSSangeeta Misra  *
13dbed73cbSSangeeta Misra  * When distributing Covered Code, include this CDDL HEADER in each
14dbed73cbSSangeeta Misra  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15dbed73cbSSangeeta Misra  * If applicable, add the following below this CDDL HEADER, with the
16dbed73cbSSangeeta Misra  * fields enclosed by brackets "[]" replaced with your own identifying
17dbed73cbSSangeeta Misra  * information: Portions Copyright [yyyy] [name of copyright owner]
18dbed73cbSSangeeta Misra  *
19dbed73cbSSangeeta Misra  * CDDL HEADER END
20dbed73cbSSangeeta Misra  */
21dbed73cbSSangeeta Misra 
22dbed73cbSSangeeta Misra /*
23dbed73cbSSangeeta Misra  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24dbed73cbSSangeeta Misra  * Use is subject to license terms.
2533f5ff17SMilan Jurik  * Copyright 2012 Milan Jurik. All rights reserved.
26dbed73cbSSangeeta Misra  */
27dbed73cbSSangeeta Misra 
28dbed73cbSSangeeta Misra #include <sys/types.h>
29dbed73cbSSangeeta Misra #include <sys/socket.h>
30dbed73cbSSangeeta Misra #include <sys/list.h>
31dbed73cbSSangeeta Misra #include <sys/stropts.h>
32dbed73cbSSangeeta Misra #include <sys/siginfo.h>
33dbed73cbSSangeeta Misra #include <sys/wait.h>
34dbed73cbSSangeeta Misra #include <arpa/inet.h>
35dbed73cbSSangeeta Misra #include <netinet/in.h>
36dbed73cbSSangeeta Misra #include <stdlib.h>
37dbed73cbSSangeeta Misra #include <stdio.h>
38dbed73cbSSangeeta Misra #include <strings.h>
39dbed73cbSSangeeta Misra #include <stddef.h>
40dbed73cbSSangeeta Misra #include <unistd.h>
41dbed73cbSSangeeta Misra #include <libilb.h>
42dbed73cbSSangeeta Misra #include <port.h>
43dbed73cbSSangeeta Misra #include <time.h>
44dbed73cbSSangeeta Misra #include <signal.h>
45dbed73cbSSangeeta Misra #include <assert.h>
46dbed73cbSSangeeta Misra #include <errno.h>
47dbed73cbSSangeeta Misra #include <spawn.h>
48dbed73cbSSangeeta Misra #include <fcntl.h>
49dbed73cbSSangeeta Misra #include <limits.h>
50dbed73cbSSangeeta Misra #include "libilb_impl.h"
51dbed73cbSSangeeta Misra #include "ilbd.h"
52dbed73cbSSangeeta Misra 
53dbed73cbSSangeeta Misra /* Global list of HC objects */
54dbed73cbSSangeeta Misra list_t ilbd_hc_list;
55dbed73cbSSangeeta Misra 
56dbed73cbSSangeeta Misra /* Timer queue for all hc related timers. */
57dbed73cbSSangeeta Misra static iu_tq_t *ilbd_hc_timer_q;
58dbed73cbSSangeeta Misra 
59dbed73cbSSangeeta Misra /* Indicate whether the timer needs to be updated */
60dbed73cbSSangeeta Misra static boolean_t hc_timer_restarted;
61dbed73cbSSangeeta Misra 
62dbed73cbSSangeeta Misra static void ilbd_hc_probe_timer(iu_tq_t *, void *);
63dbed73cbSSangeeta Misra static ilb_status_t ilbd_hc_restart_timer(ilbd_hc_t *, ilbd_hc_srv_t *);
64dbed73cbSSangeeta Misra static boolean_t ilbd_run_probe(ilbd_hc_srv_t *);
65dbed73cbSSangeeta Misra 
66dbed73cbSSangeeta Misra #define	MAX(a, b)	((a) > (b) ? (a) : (b))
67dbed73cbSSangeeta Misra 
68dbed73cbSSangeeta Misra /*
69dbed73cbSSangeeta Misra  * Number of arguments passed to a probe.  argc[0] is the path name of
70dbed73cbSSangeeta Misra  * the probe.
71dbed73cbSSangeeta Misra  */
72dbed73cbSSangeeta Misra #define	HC_PROBE_ARGC	8
73dbed73cbSSangeeta Misra 
74dbed73cbSSangeeta Misra /*
75dbed73cbSSangeeta Misra  * Max number of characters to be read from the output of a probe.  It
76dbed73cbSSangeeta Misra  * is long enough to read in a 64 bit integer.
77dbed73cbSSangeeta Misra  */
78dbed73cbSSangeeta Misra #define	HC_MAX_PROBE_OUTPUT	24
79dbed73cbSSangeeta Misra 
80dbed73cbSSangeeta Misra void
i_ilbd_setup_hc_list(void)81dbed73cbSSangeeta Misra i_ilbd_setup_hc_list(void)
82dbed73cbSSangeeta Misra {
83dbed73cbSSangeeta Misra 	list_create(&ilbd_hc_list, sizeof (ilbd_hc_t),
84dbed73cbSSangeeta Misra 	    offsetof(ilbd_hc_t, ihc_link));
85dbed73cbSSangeeta Misra }
86dbed73cbSSangeeta Misra 
87dbed73cbSSangeeta Misra /*
88dbed73cbSSangeeta Misra  * Given a hc object name, return a pointer to hc object if found.
89dbed73cbSSangeeta Misra  */
90dbed73cbSSangeeta Misra ilbd_hc_t *
ilbd_get_hc(const char * name)91dbed73cbSSangeeta Misra ilbd_get_hc(const char *name)
92dbed73cbSSangeeta Misra {
93dbed73cbSSangeeta Misra 	ilbd_hc_t *hc;
94dbed73cbSSangeeta Misra 
95dbed73cbSSangeeta Misra 	for (hc = list_head(&ilbd_hc_list); hc != NULL;
96dbed73cbSSangeeta Misra 	    hc = list_next(&ilbd_hc_list, hc)) {
97dbed73cbSSangeeta Misra 		if (strcasecmp(hc->ihc_name, name) == 0)
98dbed73cbSSangeeta Misra 			return (hc);
99dbed73cbSSangeeta Misra 	}
100dbed73cbSSangeeta Misra 	return (NULL);
101dbed73cbSSangeeta Misra }
102dbed73cbSSangeeta Misra 
103dbed73cbSSangeeta Misra /*
104dbed73cbSSangeeta Misra  * Generates an audit record for create-healthcheck,
105dbed73cbSSangeeta Misra  * delete-healtcheck subcommands.
106dbed73cbSSangeeta Misra  */
107dbed73cbSSangeeta Misra static void
ilbd_audit_hc_event(const char * audit_hcname,const ilb_hc_info_t * audit_hcinfo,ilbd_cmd_t cmd,ilb_status_t rc,ucred_t * ucredp)108dbed73cbSSangeeta Misra ilbd_audit_hc_event(const char *audit_hcname,
109dbed73cbSSangeeta Misra     const ilb_hc_info_t *audit_hcinfo, ilbd_cmd_t cmd,
110dbed73cbSSangeeta Misra     ilb_status_t rc, ucred_t *ucredp)
111dbed73cbSSangeeta Misra {
112dbed73cbSSangeeta Misra 	adt_session_data_t	*ah;
113dbed73cbSSangeeta Misra 	adt_event_data_t	*event;
114dbed73cbSSangeeta Misra 	au_event_t	flag;
115dbed73cbSSangeeta Misra 	int	audit_error;
116dbed73cbSSangeeta Misra 
117dbed73cbSSangeeta Misra 	if ((ucredp == NULL) && (cmd == ILBD_CREATE_HC))  {
118dbed73cbSSangeeta Misra 		/*
119dbed73cbSSangeeta Misra 		 * we came here from the path where ilbd incorporates
120dbed73cbSSangeeta Misra 		 * the configuration that is listed in SCF:
121dbed73cbSSangeeta Misra 		 * i_ilbd_read_config->ilbd_walk_hc_pgs->
122dbed73cbSSangeeta Misra 		 *   ->ilbd_scf_instance_walk_pg->ilbd_create_hc
123dbed73cbSSangeeta Misra 		 * We skip auditing in that case
124dbed73cbSSangeeta Misra 		 */
125dbed73cbSSangeeta Misra 		logdebug("ilbd_audit_hc_event: skipping auditing");
126dbed73cbSSangeeta Misra 		return;
127dbed73cbSSangeeta Misra 	}
128dbed73cbSSangeeta Misra 
129dbed73cbSSangeeta Misra 	if (adt_start_session(&ah, NULL, 0) != 0) {
130dbed73cbSSangeeta Misra 		logerr("ilbd_audit_hc_event: adt_start_session failed");
131dbed73cbSSangeeta Misra 		exit(EXIT_FAILURE);
132dbed73cbSSangeeta Misra 	}
133dbed73cbSSangeeta Misra 	if (adt_set_from_ucred(ah, ucredp, ADT_NEW) != 0) {
134dbed73cbSSangeeta Misra 		(void) adt_end_session(ah);
135dbed73cbSSangeeta Misra 		logerr("ilbd_audit_rule_event: adt_set_from_ucred failed");
136dbed73cbSSangeeta Misra 		exit(EXIT_FAILURE);
137dbed73cbSSangeeta Misra 	}
138dbed73cbSSangeeta Misra 	if (cmd == ILBD_CREATE_HC)
139dbed73cbSSangeeta Misra 		flag = ADT_ilb_create_healthcheck;
140dbed73cbSSangeeta Misra 	else if (cmd == ILBD_DESTROY_HC)
141dbed73cbSSangeeta Misra 		flag = ADT_ilb_delete_healthcheck;
142dbed73cbSSangeeta Misra 
143dbed73cbSSangeeta Misra 	if ((event = adt_alloc_event(ah, flag)) == NULL) {
144dbed73cbSSangeeta Misra 		logerr("ilbd_audit_hc_event: adt_alloc_event failed");
145dbed73cbSSangeeta Misra 		exit(EXIT_FAILURE);
146dbed73cbSSangeeta Misra 	}
147dbed73cbSSangeeta Misra 	(void) memset((char *)event, 0, sizeof (adt_event_data_t));
148dbed73cbSSangeeta Misra 
149dbed73cbSSangeeta Misra 	switch (cmd) {
150dbed73cbSSangeeta Misra 	case ILBD_CREATE_HC:
151dbed73cbSSangeeta Misra 		event->adt_ilb_create_healthcheck.auth_used =
152dbed73cbSSangeeta Misra 		    NET_ILB_CONFIG_AUTH;
153dbed73cbSSangeeta Misra 		event->adt_ilb_create_healthcheck.hc_test =
154dbed73cbSSangeeta Misra 		    (char *)audit_hcinfo->hci_test;
155dbed73cbSSangeeta Misra 		event->adt_ilb_create_healthcheck.hc_name =
156dbed73cbSSangeeta Misra 		    (char *)audit_hcinfo->hci_name;
157dbed73cbSSangeeta Misra 
158dbed73cbSSangeeta Misra 		/*
159dbed73cbSSangeeta Misra 		 * If the value 0 is stored, the default values are
160dbed73cbSSangeeta Misra 		 * set in the kernel. User land does not know about them
161dbed73cbSSangeeta Misra 		 * So if the user does not specify them, audit record
162dbed73cbSSangeeta Misra 		 * will show them as 0
163dbed73cbSSangeeta Misra 		 */
164dbed73cbSSangeeta Misra 		event->adt_ilb_create_healthcheck.hc_timeout =
165dbed73cbSSangeeta Misra 		    audit_hcinfo->hci_timeout;
166dbed73cbSSangeeta Misra 		event->adt_ilb_create_healthcheck.hc_count =
167dbed73cbSSangeeta Misra 		    audit_hcinfo->hci_count;
168dbed73cbSSangeeta Misra 		event->adt_ilb_create_healthcheck.hc_interval =
169dbed73cbSSangeeta Misra 		    audit_hcinfo->hci_interval;
170dbed73cbSSangeeta Misra 		break;
171dbed73cbSSangeeta Misra 	case ILBD_DESTROY_HC:
172dbed73cbSSangeeta Misra 		event->adt_ilb_delete_healthcheck.auth_used =
173dbed73cbSSangeeta Misra 		    NET_ILB_CONFIG_AUTH;
174dbed73cbSSangeeta Misra 		event->adt_ilb_delete_healthcheck.hc_name =
175dbed73cbSSangeeta Misra 		    (char *)audit_hcname;
176dbed73cbSSangeeta Misra 		break;
177dbed73cbSSangeeta Misra 	}
178dbed73cbSSangeeta Misra 
179dbed73cbSSangeeta Misra 	/* Fill in success/failure */
180dbed73cbSSangeeta Misra 	if (rc == ILB_STATUS_OK) {
181dbed73cbSSangeeta Misra 		if (adt_put_event(event, ADT_SUCCESS, ADT_SUCCESS) != 0) {
182dbed73cbSSangeeta Misra 			logerr("ilbd_audit_hc_event: adt_put_event failed");
183dbed73cbSSangeeta Misra 			exit(EXIT_FAILURE);
184dbed73cbSSangeeta Misra 		}
185dbed73cbSSangeeta Misra 	} else {
186dbed73cbSSangeeta Misra 		audit_error = ilberror2auditerror(rc);
187dbed73cbSSangeeta Misra 		if (adt_put_event(event, ADT_FAILURE, audit_error) != 0) {
188dbed73cbSSangeeta Misra 			logerr("ilbd_audit_hc_event: adt_put_event failed");
189dbed73cbSSangeeta Misra 			exit(EXIT_FAILURE);
190dbed73cbSSangeeta Misra 		}
191dbed73cbSSangeeta Misra 	}
192dbed73cbSSangeeta Misra 	adt_free_event(event);
193dbed73cbSSangeeta Misra 	(void) adt_end_session(ah);
194dbed73cbSSangeeta Misra }
195dbed73cbSSangeeta Misra 
196dbed73cbSSangeeta Misra /*
197dbed73cbSSangeeta Misra  * Given the ilb_hc_info_t passed in (from the libilb), create a hc object
198dbed73cbSSangeeta Misra  * in ilbd.  The parameter ev_port is not used, refer to comments of
199dbed73cbSSangeeta Misra  * ilbd_create_sg() in ilbd_sg.c
200dbed73cbSSangeeta Misra  */
201dbed73cbSSangeeta Misra /* ARGSUSED */
202dbed73cbSSangeeta Misra ilb_status_t
ilbd_create_hc(const ilb_hc_info_t * hc_info,int ev_port,const struct passwd * ps,ucred_t * ucredp)203dbed73cbSSangeeta Misra ilbd_create_hc(const ilb_hc_info_t *hc_info, int ev_port,
204dbed73cbSSangeeta Misra     const struct passwd *ps, ucred_t *ucredp)
205dbed73cbSSangeeta Misra {
206dbed73cbSSangeeta Misra 	ilbd_hc_t *hc;
207dbed73cbSSangeeta Misra 	ilb_status_t ret = ILB_STATUS_OK;
208dbed73cbSSangeeta Misra 
209dbed73cbSSangeeta Misra 	/*
210dbed73cbSSangeeta Misra 	 * ps == NULL is from the daemon when it starts and load configuration
211dbed73cbSSangeeta Misra 	 * ps != NULL is from client.
212dbed73cbSSangeeta Misra 	 */
213dbed73cbSSangeeta Misra 	if (ps != NULL) {
214dbed73cbSSangeeta Misra 		ret = ilbd_check_client_config_auth(ps);
215dbed73cbSSangeeta Misra 		if (ret != ILB_STATUS_OK) {
216dbed73cbSSangeeta Misra 			ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
217dbed73cbSSangeeta Misra 			    ret, ucredp);
218dbed73cbSSangeeta Misra 			return (ret);
219dbed73cbSSangeeta Misra 		}
220dbed73cbSSangeeta Misra 	}
221dbed73cbSSangeeta Misra 
222dbed73cbSSangeeta Misra 	if (hc_info->hci_name[0] == '\0') {
223dbed73cbSSangeeta Misra 		logdebug("ilbd_create_hc: missing healthcheck info");
224dbed73cbSSangeeta Misra 		ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
225dbed73cbSSangeeta Misra 		    ILB_STATUS_ENOHCINFO, ucredp);
226dbed73cbSSangeeta Misra 		return (ILB_STATUS_ENOHCINFO);
227dbed73cbSSangeeta Misra 	}
228dbed73cbSSangeeta Misra 
229dbed73cbSSangeeta Misra 	hc = ilbd_get_hc(hc_info->hci_name);
230dbed73cbSSangeeta Misra 	if (hc != NULL) {
231dbed73cbSSangeeta Misra 		logdebug("ilbd_create_hc: healthcheck name %s already"
232dbed73cbSSangeeta Misra 		    " exists", hc_info->hci_name);
233dbed73cbSSangeeta Misra 		ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
234dbed73cbSSangeeta Misra 		    ILB_STATUS_EEXIST, ucredp);
235dbed73cbSSangeeta Misra 		return (ILB_STATUS_EEXIST);
236dbed73cbSSangeeta Misra 	}
237dbed73cbSSangeeta Misra 
238dbed73cbSSangeeta Misra 	/*
239dbed73cbSSangeeta Misra 	 * Sanity check on user supplied probe.  The given path name
240dbed73cbSSangeeta Misra 	 * must be a full path name (starts with '/') and is
241dbed73cbSSangeeta Misra 	 * executable.
242dbed73cbSSangeeta Misra 	 */
243dbed73cbSSangeeta Misra 	if (strcasecmp(hc_info->hci_test, ILB_HC_STR_TCP) != 0 &&
244dbed73cbSSangeeta Misra 	    strcasecmp(hc_info->hci_test, ILB_HC_STR_UDP) != 0 &&
245dbed73cbSSangeeta Misra 	    strcasecmp(hc_info->hci_test, ILB_HC_STR_PING) != 0 &&
246dbed73cbSSangeeta Misra 	    (hc_info->hci_test[0] != '/' ||
247dbed73cbSSangeeta Misra 	    access(hc_info->hci_test, X_OK) == -1)) {
248dbed73cbSSangeeta Misra 		if (errno == ENOENT) {
249dbed73cbSSangeeta Misra 			logdebug("ilbd_create_hc: user script %s doesn't "
250dbed73cbSSangeeta Misra 			    "exist", hc_info->hci_test);
251dbed73cbSSangeeta Misra 			ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
252dbed73cbSSangeeta Misra 			    ILB_STATUS_ENOENT, ucredp);
253dbed73cbSSangeeta Misra 			return (ILB_STATUS_ENOENT);
254dbed73cbSSangeeta Misra 		} else {
255dbed73cbSSangeeta Misra 			logdebug("ilbd_create_hc: user script %s is "
256dbed73cbSSangeeta Misra 			    "invalid", hc_info->hci_test);
257dbed73cbSSangeeta Misra 			ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
258dbed73cbSSangeeta Misra 			    ILB_STATUS_EINVAL, ucredp);
259dbed73cbSSangeeta Misra 			return (ILB_STATUS_EINVAL);
260dbed73cbSSangeeta Misra 		}
261dbed73cbSSangeeta Misra 	}
262dbed73cbSSangeeta Misra 
263dbed73cbSSangeeta Misra 	/* Create and add the hc object */
264dbed73cbSSangeeta Misra 	hc = calloc(1, sizeof (ilbd_hc_t));
265dbed73cbSSangeeta Misra 	if (hc == NULL) {
266dbed73cbSSangeeta Misra 		ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
267dbed73cbSSangeeta Misra 		    ILB_STATUS_ENOMEM, ucredp);
268dbed73cbSSangeeta Misra 		return (ILB_STATUS_ENOMEM);
269dbed73cbSSangeeta Misra 	}
270dbed73cbSSangeeta Misra 	(void) memcpy(&hc->ihc_info, hc_info, sizeof (ilb_hc_info_t));
271dbed73cbSSangeeta Misra 	if (strcasecmp(hc->ihc_test, ILB_HC_STR_TCP) == 0)
272dbed73cbSSangeeta Misra 		hc->ihc_test_type = ILBD_HC_TCP;
273dbed73cbSSangeeta Misra 	else if (strcasecmp(hc->ihc_test, ILB_HC_STR_UDP) == 0)
274dbed73cbSSangeeta Misra 		hc->ihc_test_type = ILBD_HC_UDP;
275dbed73cbSSangeeta Misra 	else if (strcasecmp(hc->ihc_test, ILB_HC_STR_PING) == 0)
276dbed73cbSSangeeta Misra 		hc->ihc_test_type = ILBD_HC_PING;
277dbed73cbSSangeeta Misra 	else
278dbed73cbSSangeeta Misra 		hc->ihc_test_type = ILBD_HC_USER;
279dbed73cbSSangeeta Misra 	list_create(&hc->ihc_rules, sizeof (ilbd_hc_rule_t),
280dbed73cbSSangeeta Misra 	    offsetof(ilbd_hc_rule_t, hcr_link));
281dbed73cbSSangeeta Misra 
282dbed73cbSSangeeta Misra 	/* Update SCF */
283dbed73cbSSangeeta Misra 	if (ps != NULL) {
284dbed73cbSSangeeta Misra 		if ((ret = ilbd_create_pg(ILBD_SCF_HC, (void *)hc)) !=
285dbed73cbSSangeeta Misra 		    ILB_STATUS_OK) {
286dbed73cbSSangeeta Misra 			ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC,
287dbed73cbSSangeeta Misra 			    ret, ucredp);
28856b8f71eSSerghei Samsi 			list_destroy(&hc->ihc_rules);
289dbed73cbSSangeeta Misra 			free(hc);
290dbed73cbSSangeeta Misra 			return (ret);
291dbed73cbSSangeeta Misra 		}
292dbed73cbSSangeeta Misra 	}
293dbed73cbSSangeeta Misra 
294dbed73cbSSangeeta Misra 	/* Everything is fine, now add it to the global list. */
295dbed73cbSSangeeta Misra 	list_insert_tail(&ilbd_hc_list, hc);
296dbed73cbSSangeeta Misra 	ilbd_audit_hc_event(NULL, hc_info, ILBD_CREATE_HC, ret, ucredp);
297dbed73cbSSangeeta Misra 	return (ret);
298dbed73cbSSangeeta Misra }
299dbed73cbSSangeeta Misra 
300dbed73cbSSangeeta Misra /*
301dbed73cbSSangeeta Misra  * Given a name of a hc object, destroy it.
302dbed73cbSSangeeta Misra  */
303dbed73cbSSangeeta Misra ilb_status_t
ilbd_destroy_hc(const char * hc_name,const struct passwd * ps,ucred_t * ucredp)304dbed73cbSSangeeta Misra ilbd_destroy_hc(const char *hc_name, const struct passwd *ps,
305dbed73cbSSangeeta Misra     ucred_t *ucredp)
306dbed73cbSSangeeta Misra {
307dbed73cbSSangeeta Misra 	ilb_status_t ret;
308dbed73cbSSangeeta Misra 	ilbd_hc_t *hc;
309dbed73cbSSangeeta Misra 
310dbed73cbSSangeeta Misra 	/*
311dbed73cbSSangeeta Misra 	 * No need to check ps == NULL, daemon won't call any destroy func
312dbed73cbSSangeeta Misra 	 * at start up.
313dbed73cbSSangeeta Misra 	 */
314dbed73cbSSangeeta Misra 	ret = ilbd_check_client_config_auth(ps);
315dbed73cbSSangeeta Misra 	if (ret != ILB_STATUS_OK) {
316dbed73cbSSangeeta Misra 		ilbd_audit_hc_event(hc_name, NULL, ILBD_DESTROY_HC,
317dbed73cbSSangeeta Misra 		    ret, ucredp);
318dbed73cbSSangeeta Misra 		return (ret);
319dbed73cbSSangeeta Misra 	}
320dbed73cbSSangeeta Misra 
321dbed73cbSSangeeta Misra 	hc = ilbd_get_hc(hc_name);
322dbed73cbSSangeeta Misra 	if (hc == NULL) {
323dbed73cbSSangeeta Misra 		logdebug("ilbd_destroy_hc: healthcheck %s does not exist",
324dbed73cbSSangeeta Misra 		    hc_name);
325dbed73cbSSangeeta Misra 		ilbd_audit_hc_event(hc_name, NULL, ILBD_DESTROY_HC,
326dbed73cbSSangeeta Misra 		    ILB_STATUS_ENOENT, ucredp);
327dbed73cbSSangeeta Misra 		return (ILB_STATUS_ENOENT);
328dbed73cbSSangeeta Misra 	}
329dbed73cbSSangeeta Misra 
330dbed73cbSSangeeta Misra 	/* If hc is in use, cannot delete it */
331dbed73cbSSangeeta Misra 	if (hc->ihc_rule_cnt > 0) {
332dbed73cbSSangeeta Misra 		logdebug("ilbd_destroy_hc: healthcheck %s is associated"
333dbed73cbSSangeeta Misra 		    " with a rule - cannot remove", hc_name);
334dbed73cbSSangeeta Misra 		ilbd_audit_hc_event(hc_name, NULL, ILBD_DESTROY_HC,
335dbed73cbSSangeeta Misra 		    ILB_STATUS_INUSE, ucredp);
336dbed73cbSSangeeta Misra 		return (ILB_STATUS_INUSE);
337dbed73cbSSangeeta Misra 	}
338dbed73cbSSangeeta Misra 
339dbed73cbSSangeeta Misra 	if ((ret = ilbd_destroy_pg(ILBD_SCF_HC, hc_name)) !=
340dbed73cbSSangeeta Misra 	    ILB_STATUS_OK) {
341dbed73cbSSangeeta Misra 		logdebug("ilbd_destroy_hc: cannot destroy healthcheck %s "
342dbed73cbSSangeeta Misra 		    "property group", hc_name);
343dbed73cbSSangeeta Misra 		ilbd_audit_hc_event(hc_name, NULL, ILBD_DESTROY_HC,
344dbed73cbSSangeeta Misra 		    ret, ucredp);
345dbed73cbSSangeeta Misra 		return (ret);
346dbed73cbSSangeeta Misra 	}
347dbed73cbSSangeeta Misra 
348dbed73cbSSangeeta Misra 	list_remove(&ilbd_hc_list, hc);
34956b8f71eSSerghei Samsi 	list_destroy(&hc->ihc_rules);
350dbed73cbSSangeeta Misra 	free(hc);
351dbed73cbSSangeeta Misra 	ilbd_audit_hc_event(hc_name, NULL, ILBD_DESTROY_HC, ret, ucredp);
352dbed73cbSSangeeta Misra 	return (ret);
353dbed73cbSSangeeta Misra }
354dbed73cbSSangeeta Misra 
355dbed73cbSSangeeta Misra /*
356dbed73cbSSangeeta Misra  * Given a hc object name, return its information.  Used by libilb to
357dbed73cbSSangeeta Misra  * get hc info.
358dbed73cbSSangeeta Misra  */
359dbed73cbSSangeeta Misra ilb_status_t
ilbd_get_hc_info(const char * hc_name,uint32_t * rbuf,size_t * rbufsz)360dbed73cbSSangeeta Misra ilbd_get_hc_info(const char *hc_name, uint32_t *rbuf, size_t *rbufsz)
361dbed73cbSSangeeta Misra {
362dbed73cbSSangeeta Misra 	ilbd_hc_t	*hc;
363dbed73cbSSangeeta Misra 	ilb_hc_info_t	*hc_info;
364dbed73cbSSangeeta Misra 	ilb_comm_t	*ic = (ilb_comm_t *)rbuf;
365dbed73cbSSangeeta Misra 
366dbed73cbSSangeeta Misra 	hc = ilbd_get_hc(hc_name);
367dbed73cbSSangeeta Misra 	if (hc == NULL) {
368dbed73cbSSangeeta Misra 		logdebug("%s: healthcheck %s does not exist", __func__,
369dbed73cbSSangeeta Misra 		    hc_name);
370dbed73cbSSangeeta Misra 		return (ILB_STATUS_ENOENT);
371dbed73cbSSangeeta Misra 	}
372dbed73cbSSangeeta Misra 	ilbd_reply_ok(rbuf, rbufsz);
373dbed73cbSSangeeta Misra 	hc_info = (ilb_hc_info_t *)&ic->ic_data;
374dbed73cbSSangeeta Misra 
375dbed73cbSSangeeta Misra 	(void) strlcpy(hc_info->hci_name, hc->ihc_name, sizeof (hc->ihc_name));
376dbed73cbSSangeeta Misra 	(void) strlcpy(hc_info->hci_test, hc->ihc_test, sizeof (hc->ihc_test));
377dbed73cbSSangeeta Misra 	hc_info->hci_timeout = hc->ihc_timeout;
378dbed73cbSSangeeta Misra 	hc_info->hci_count = hc->ihc_count;
379dbed73cbSSangeeta Misra 	hc_info->hci_interval = hc->ihc_interval;
380dbed73cbSSangeeta Misra 	hc_info->hci_def_ping = hc->ihc_def_ping;
381dbed73cbSSangeeta Misra 
382dbed73cbSSangeeta Misra 	*rbufsz += sizeof (ilb_hc_info_t);
383dbed73cbSSangeeta Misra 
384dbed73cbSSangeeta Misra 	return (ILB_STATUS_OK);
385dbed73cbSSangeeta Misra }
386dbed73cbSSangeeta Misra 
387dbed73cbSSangeeta Misra static void
ilbd_hc_copy_srvs(uint32_t * rbuf,size_t * rbufsz,ilbd_hc_rule_t * hc_rule,const char * rulename)388dbed73cbSSangeeta Misra ilbd_hc_copy_srvs(uint32_t *rbuf, size_t *rbufsz, ilbd_hc_rule_t *hc_rule,
389dbed73cbSSangeeta Misra     const char *rulename)
390dbed73cbSSangeeta Misra {
391dbed73cbSSangeeta Misra 	ilbd_hc_srv_t		*tmp_srv;
392dbed73cbSSangeeta Misra 	ilb_hc_srv_t		*dst_srv;
393dbed73cbSSangeeta Misra 	ilb_hc_rule_srv_t	*srvs;
394dbed73cbSSangeeta Misra 	size_t			tmp_rbufsz;
395dbed73cbSSangeeta Misra 	int			i;
396dbed73cbSSangeeta Misra 
397dbed73cbSSangeeta Misra 	tmp_rbufsz = *rbufsz;
398dbed73cbSSangeeta Misra 	/* Set up the reply buffer.  rbufsz will be set to the new size. */
399dbed73cbSSangeeta Misra 	ilbd_reply_ok(rbuf, rbufsz);
400dbed73cbSSangeeta Misra 
401dbed73cbSSangeeta Misra 	/* Calculate how much space is left for holding server info. */
402dbed73cbSSangeeta Misra 	*rbufsz += sizeof (ilb_hc_rule_srv_t);
403dbed73cbSSangeeta Misra 	tmp_rbufsz -= *rbufsz;
404dbed73cbSSangeeta Misra 
405dbed73cbSSangeeta Misra 	srvs = (ilb_hc_rule_srv_t *)&((ilb_comm_t *)rbuf)->ic_data;
406dbed73cbSSangeeta Misra 
407dbed73cbSSangeeta Misra 	tmp_srv = list_head(&hc_rule->hcr_servers);
408dbed73cbSSangeeta Misra 	for (i = 0; tmp_srv != NULL && tmp_rbufsz >= sizeof (*dst_srv); i++) {
409dbed73cbSSangeeta Misra 		dst_srv = &srvs->rs_srvs[i];
410dbed73cbSSangeeta Misra 
411dbed73cbSSangeeta Misra 		(void) strlcpy(dst_srv->hcs_rule_name, rulename, ILB_NAMESZ);
412dbed73cbSSangeeta Misra 		(void) strlcpy(dst_srv->hcs_ID, tmp_srv->shc_sg_srv->sgs_srvID,
413dbed73cbSSangeeta Misra 		    ILB_NAMESZ);
414dbed73cbSSangeeta Misra 		(void) strlcpy(dst_srv->hcs_hc_name,
415dbed73cbSSangeeta Misra 		    tmp_srv->shc_hc->ihc_name, ILB_NAMESZ);
416dbed73cbSSangeeta Misra 		dst_srv->hcs_IP = tmp_srv->shc_sg_srv->sgs_addr;
417dbed73cbSSangeeta Misra 		dst_srv->hcs_fail_cnt = tmp_srv->shc_fail_cnt;
418dbed73cbSSangeeta Misra 		dst_srv->hcs_status = tmp_srv->shc_status;
419dbed73cbSSangeeta Misra 		dst_srv->hcs_rtt = tmp_srv->shc_rtt;
420dbed73cbSSangeeta Misra 		dst_srv->hcs_lasttime = tmp_srv->shc_lasttime;
421dbed73cbSSangeeta Misra 		dst_srv->hcs_nexttime = tmp_srv->shc_nexttime;
422dbed73cbSSangeeta Misra 
423dbed73cbSSangeeta Misra 		tmp_srv = list_next(&hc_rule->hcr_servers, tmp_srv);
424dbed73cbSSangeeta Misra 		tmp_rbufsz -= sizeof (*dst_srv);
425dbed73cbSSangeeta Misra 	}
426dbed73cbSSangeeta Misra 	srvs->rs_num_srvs = i;
427dbed73cbSSangeeta Misra 	*rbufsz += i * sizeof (*dst_srv);
428dbed73cbSSangeeta Misra }
429dbed73cbSSangeeta Misra 
430dbed73cbSSangeeta Misra /*
431dbed73cbSSangeeta Misra  * Given a rule name, return the hc status of its servers.
432dbed73cbSSangeeta Misra  */
433dbed73cbSSangeeta Misra ilb_status_t
ilbd_get_hc_srvs(const char * rulename,uint32_t * rbuf,size_t * rbufsz)434dbed73cbSSangeeta Misra ilbd_get_hc_srvs(const char *rulename, uint32_t *rbuf, size_t *rbufsz)
435dbed73cbSSangeeta Misra {
436dbed73cbSSangeeta Misra 	ilbd_hc_t	*hc;
437dbed73cbSSangeeta Misra 	ilbd_hc_rule_t	*hc_rule;
438dbed73cbSSangeeta Misra 
439dbed73cbSSangeeta Misra 	for (hc = list_head(&ilbd_hc_list); hc != NULL;
440dbed73cbSSangeeta Misra 	    hc = list_next(&ilbd_hc_list, hc)) {
441dbed73cbSSangeeta Misra 		for (hc_rule = list_head(&hc->ihc_rules); hc_rule != NULL;
442dbed73cbSSangeeta Misra 		    hc_rule = list_next(&hc->ihc_rules, hc_rule)) {
443dbed73cbSSangeeta Misra 			if (strcasecmp(hc_rule->hcr_rule->irl_name,
444dbed73cbSSangeeta Misra 			    rulename) != 0) {
445dbed73cbSSangeeta Misra 				continue;
446dbed73cbSSangeeta Misra 			}
447dbed73cbSSangeeta Misra 			ilbd_hc_copy_srvs(rbuf, rbufsz, hc_rule, rulename);
448dbed73cbSSangeeta Misra 			return (ILB_STATUS_OK);
449dbed73cbSSangeeta Misra 		}
450dbed73cbSSangeeta Misra 	}
451dbed73cbSSangeeta Misra 	return (ILB_STATUS_RULE_NO_HC);
452dbed73cbSSangeeta Misra }
453dbed73cbSSangeeta Misra 
454dbed73cbSSangeeta Misra /*
455dbed73cbSSangeeta Misra  * Initialize the hc timer and associate the notification of timeout to
456dbed73cbSSangeeta Misra  * the given event port.
457dbed73cbSSangeeta Misra  */
458dbed73cbSSangeeta Misra void
ilbd_hc_timer_init(int ev_port,ilbd_timer_event_obj_t * ev_obj)459dbed73cbSSangeeta Misra ilbd_hc_timer_init(int ev_port, ilbd_timer_event_obj_t *ev_obj)
460dbed73cbSSangeeta Misra {
461dbed73cbSSangeeta Misra 	struct sigevent sigev;
462dbed73cbSSangeeta Misra 	port_notify_t notify;
463dbed73cbSSangeeta Misra 
464dbed73cbSSangeeta Misra 	if ((ilbd_hc_timer_q = iu_tq_create()) == NULL) {
465dbed73cbSSangeeta Misra 		logerr("%s: cannot create hc timer queue", __func__);
466dbed73cbSSangeeta Misra 		exit(EXIT_FAILURE);
467dbed73cbSSangeeta Misra 	}
468dbed73cbSSangeeta Misra 	hc_timer_restarted = B_FALSE;
469dbed73cbSSangeeta Misra 
470dbed73cbSSangeeta Misra 	ev_obj->ev = ILBD_EVENT_TIMER;
471dbed73cbSSangeeta Misra 	ev_obj->timerid = -1;
472dbed73cbSSangeeta Misra 
473dbed73cbSSangeeta Misra 	notify.portnfy_port = ev_port;
474dbed73cbSSangeeta Misra 	notify.portnfy_user = ev_obj;
475dbed73cbSSangeeta Misra 	sigev.sigev_notify = SIGEV_PORT;
476dbed73cbSSangeeta Misra 	sigev.sigev_value.sival_ptr = &notify;
477dbed73cbSSangeeta Misra 	if (timer_create(CLOCK_REALTIME, &sigev, &ev_obj->timerid) == -1) {
478dbed73cbSSangeeta Misra 		logerr("%s: cannot create timer", __func__);
479dbed73cbSSangeeta Misra 		exit(EXIT_FAILURE);
480dbed73cbSSangeeta Misra 	}
481dbed73cbSSangeeta Misra }
482dbed73cbSSangeeta Misra 
483dbed73cbSSangeeta Misra /*
484dbed73cbSSangeeta Misra  * HC timeout handler.
485dbed73cbSSangeeta Misra  */
486dbed73cbSSangeeta Misra void
ilbd_hc_timeout(void)487dbed73cbSSangeeta Misra ilbd_hc_timeout(void)
488dbed73cbSSangeeta Misra {
489dbed73cbSSangeeta Misra 	(void) iu_expire_timers(ilbd_hc_timer_q);
490dbed73cbSSangeeta Misra 	hc_timer_restarted = B_TRUE;
491dbed73cbSSangeeta Misra }
492dbed73cbSSangeeta Misra 
493dbed73cbSSangeeta Misra /*
494dbed73cbSSangeeta Misra  * Set up the timer to fire at the earliest timeout.
495dbed73cbSSangeeta Misra  */
496dbed73cbSSangeeta Misra void
ilbd_hc_timer_update(ilbd_timer_event_obj_t * ev_obj)497dbed73cbSSangeeta Misra ilbd_hc_timer_update(ilbd_timer_event_obj_t *ev_obj)
498dbed73cbSSangeeta Misra {
499dbed73cbSSangeeta Misra 	itimerspec_t itimeout;
500dbed73cbSSangeeta Misra 	int timeout;
501dbed73cbSSangeeta Misra 
502dbed73cbSSangeeta Misra 	/*
503dbed73cbSSangeeta Misra 	 * There is no change on the timer list, so no need to set up the
504dbed73cbSSangeeta Misra 	 * timer again.
505dbed73cbSSangeeta Misra 	 */
506dbed73cbSSangeeta Misra 	if (!hc_timer_restarted)
507dbed73cbSSangeeta Misra 		return;
508dbed73cbSSangeeta Misra 
509dbed73cbSSangeeta Misra restart:
510dbed73cbSSangeeta Misra 	if ((timeout = iu_earliest_timer(ilbd_hc_timer_q)) == INFTIM) {
511dbed73cbSSangeeta Misra 		hc_timer_restarted = B_FALSE;
512dbed73cbSSangeeta Misra 		return;
513dbed73cbSSangeeta Misra 	} else if (timeout == 0) {
514dbed73cbSSangeeta Misra 		/*
515dbed73cbSSangeeta Misra 		 * Handle the timeout immediately.  After that (clearing all
516dbed73cbSSangeeta Misra 		 * the expired timers), check to  see if there are still
517dbed73cbSSangeeta Misra 		 * timers running.  If yes, start them.
518dbed73cbSSangeeta Misra 		 */
519dbed73cbSSangeeta Misra 		(void) iu_expire_timers(ilbd_hc_timer_q);
520dbed73cbSSangeeta Misra 		goto restart;
521dbed73cbSSangeeta Misra 	}
522dbed73cbSSangeeta Misra 
523dbed73cbSSangeeta Misra 	itimeout.it_value.tv_sec = timeout / MILLISEC + 1;
524dbed73cbSSangeeta Misra 	itimeout.it_value.tv_nsec = 0;
525dbed73cbSSangeeta Misra 	itimeout.it_interval.tv_sec = 0;
526dbed73cbSSangeeta Misra 	itimeout.it_interval.tv_nsec = 0;
527dbed73cbSSangeeta Misra 
528dbed73cbSSangeeta Misra 	/*
529dbed73cbSSangeeta Misra 	 * Failure to set a timeout is "OK" since hopefully there will be
530dbed73cbSSangeeta Misra 	 * other events and timer_settime() will be called again.  So
531dbed73cbSSangeeta Misra 	 * we will only miss some timeouts.  But in the worst case, no event
532dbed73cbSSangeeta Misra 	 * will happen and ilbd will get stuck...
533dbed73cbSSangeeta Misra 	 */
534dbed73cbSSangeeta Misra 	if (timer_settime(ev_obj->timerid, 0, &itimeout, NULL) == -1)
535dbed73cbSSangeeta Misra 		logerr("%s: cannot set timer", __func__);
536dbed73cbSSangeeta Misra 	hc_timer_restarted = B_FALSE;
537dbed73cbSSangeeta Misra }
538dbed73cbSSangeeta Misra 
539dbed73cbSSangeeta Misra /*
540dbed73cbSSangeeta Misra  * Kill the probe process of a server.
541dbed73cbSSangeeta Misra  */
542dbed73cbSSangeeta Misra static void
ilbd_hc_kill_probe(ilbd_hc_srv_t * srv)543dbed73cbSSangeeta Misra ilbd_hc_kill_probe(ilbd_hc_srv_t *srv)
544dbed73cbSSangeeta Misra {
545dbed73cbSSangeeta Misra 	/*
546dbed73cbSSangeeta Misra 	 * First dissociate the fd from the event port.  It should not
547dbed73cbSSangeeta Misra 	 * fail.
548dbed73cbSSangeeta Misra 	 */
549dbed73cbSSangeeta Misra 	if (port_dissociate(srv->shc_ev_port, PORT_SOURCE_FD,
550dbed73cbSSangeeta Misra 	    srv->shc_child_fd) != 0) {
551dbed73cbSSangeeta Misra 		logdebug("%s: port_dissociate: %s", __func__, strerror(errno));
552dbed73cbSSangeeta Misra 	}
553dbed73cbSSangeeta Misra 	(void) close(srv->shc_child_fd);
554dbed73cbSSangeeta Misra 	free(srv->shc_ev);
555dbed73cbSSangeeta Misra 	srv->shc_ev = NULL;
556dbed73cbSSangeeta Misra 
557dbed73cbSSangeeta Misra 	/* Then kill the probe process. */
558dbed73cbSSangeeta Misra 	if (kill(srv->shc_child_pid, SIGKILL) != 0) {
559dbed73cbSSangeeta Misra 		logerr("%s: rule %s server %s: %s", __func__,
560dbed73cbSSangeeta Misra 		    srv->shc_hc_rule->hcr_rule->irl_name,
561dbed73cbSSangeeta Misra 		    srv->shc_sg_srv->sgs_srvID, strerror(errno));
562dbed73cbSSangeeta Misra 	}
563dbed73cbSSangeeta Misra 	/* Should not fail... */
564dbed73cbSSangeeta Misra 	if (waitpid(srv->shc_child_pid, NULL, 0) != srv->shc_child_pid) {
565dbed73cbSSangeeta Misra 		logdebug("%s: waitpid: rule %s server %s", __func__,
566dbed73cbSSangeeta Misra 		    srv->shc_hc_rule->hcr_rule->irl_name,
567dbed73cbSSangeeta Misra 		    srv->shc_sg_srv->sgs_srvID);
568dbed73cbSSangeeta Misra 	}
569dbed73cbSSangeeta Misra 	srv->shc_child_pid = 0;
570dbed73cbSSangeeta Misra }
571dbed73cbSSangeeta Misra 
572dbed73cbSSangeeta Misra /*
573dbed73cbSSangeeta Misra  * Disable the server, either because the server is dead or because a timer
574dbed73cbSSangeeta Misra  * cannot be started for this server.  Note that this only affects the
575dbed73cbSSangeeta Misra  * transient configuration, meaning only in memory.  The persistent
576dbed73cbSSangeeta Misra  * configuration is not affected.
577dbed73cbSSangeeta Misra  */
578dbed73cbSSangeeta Misra static void
ilbd_mark_server_disabled(ilbd_hc_srv_t * srv)579dbed73cbSSangeeta Misra ilbd_mark_server_disabled(ilbd_hc_srv_t *srv)
580dbed73cbSSangeeta Misra {
581dbed73cbSSangeeta Misra 	srv->shc_status = ILB_HCS_DISABLED;
582dbed73cbSSangeeta Misra 
583dbed73cbSSangeeta Misra 	/* Disable the server in kernel. */
584dbed73cbSSangeeta Misra 	if (ilbd_k_Xable_server(&srv->shc_sg_srv->sgs_addr,
585dbed73cbSSangeeta Misra 	    srv->shc_hc_rule->hcr_rule->irl_name,
586dbed73cbSSangeeta Misra 	    stat_declare_srv_dead) != ILB_STATUS_OK) {
587dbed73cbSSangeeta Misra 		logerr("%s: cannot disable server in kernel: rule %s "
588dbed73cbSSangeeta Misra 		    "server %s", __func__,
589dbed73cbSSangeeta Misra 		    srv->shc_hc_rule->hcr_rule->irl_name,
590dbed73cbSSangeeta Misra 		    srv->shc_sg_srv->sgs_srvID);
591dbed73cbSSangeeta Misra 	}
592dbed73cbSSangeeta Misra }
593dbed73cbSSangeeta Misra 
594dbed73cbSSangeeta Misra /*
595dbed73cbSSangeeta Misra  * A probe fails, set the state of the server.
596dbed73cbSSangeeta Misra  */
597dbed73cbSSangeeta Misra static void
ilbd_set_fail_state(ilbd_hc_srv_t * srv)598dbed73cbSSangeeta Misra ilbd_set_fail_state(ilbd_hc_srv_t *srv)
599dbed73cbSSangeeta Misra {
600dbed73cbSSangeeta Misra 	if (++srv->shc_fail_cnt < srv->shc_hc->ihc_count) {
601dbed73cbSSangeeta Misra 		/* Probe again */
602dbed73cbSSangeeta Misra 		ilbd_hc_probe_timer(ilbd_hc_timer_q, srv);
603dbed73cbSSangeeta Misra 		return;
604dbed73cbSSangeeta Misra 	}
605dbed73cbSSangeeta Misra 
606dbed73cbSSangeeta Misra 	logdebug("%s: rule %s server %s fails %u", __func__,
607dbed73cbSSangeeta Misra 	    srv->shc_hc_rule->hcr_rule->irl_name, srv->shc_sg_srv->sgs_srvID,
608dbed73cbSSangeeta Misra 	    srv->shc_fail_cnt);
609dbed73cbSSangeeta Misra 
610dbed73cbSSangeeta Misra 	/*
611dbed73cbSSangeeta Misra 	 * If this is a ping test, mark the server as
612dbed73cbSSangeeta Misra 	 * unreachable instead of dead.
613dbed73cbSSangeeta Misra 	 */
614dbed73cbSSangeeta Misra 	if (srv->shc_hc->ihc_test_type == ILBD_HC_PING ||
615dbed73cbSSangeeta Misra 	    srv->shc_state == ilbd_hc_def_pinging) {
616dbed73cbSSangeeta Misra 		srv->shc_status = ILB_HCS_UNREACH;
617dbed73cbSSangeeta Misra 	} else {
618dbed73cbSSangeeta Misra 		srv->shc_status = ILB_HCS_DEAD;
619dbed73cbSSangeeta Misra 	}
620dbed73cbSSangeeta Misra 
621dbed73cbSSangeeta Misra 	/* Disable the server in kernel. */
622dbed73cbSSangeeta Misra 	if (ilbd_k_Xable_server(&srv->shc_sg_srv->sgs_addr,
623dbed73cbSSangeeta Misra 	    srv->shc_hc_rule->hcr_rule->irl_name, stat_declare_srv_dead) !=
624dbed73cbSSangeeta Misra 	    ILB_STATUS_OK) {
625dbed73cbSSangeeta Misra 		logerr("%s: cannot disable server in kernel: rule %s "
626dbed73cbSSangeeta Misra 		    "server %s", __func__,
627dbed73cbSSangeeta Misra 		    srv->shc_hc_rule->hcr_rule->irl_name,
628dbed73cbSSangeeta Misra 		    srv->shc_sg_srv->sgs_srvID);
629dbed73cbSSangeeta Misra 	}
630dbed73cbSSangeeta Misra 
631dbed73cbSSangeeta Misra 	/* Still keep probing in case the server is alive again. */
632dbed73cbSSangeeta Misra 	if (ilbd_hc_restart_timer(srv->shc_hc, srv) != ILB_STATUS_OK) {
633dbed73cbSSangeeta Misra 		/* Only thing to do is to disable the server... */
634dbed73cbSSangeeta Misra 		logerr("%s: cannot restart timer: rule %s server %s", __func__,
635dbed73cbSSangeeta Misra 		    srv->shc_hc_rule->hcr_rule->irl_name,
636dbed73cbSSangeeta Misra 		    srv->shc_sg_srv->sgs_srvID);
637dbed73cbSSangeeta Misra 		srv->shc_status = ILB_HCS_DISABLED;
638dbed73cbSSangeeta Misra 	}
639dbed73cbSSangeeta Misra }
640dbed73cbSSangeeta Misra 
641dbed73cbSSangeeta Misra /*
642dbed73cbSSangeeta Misra  * A probe process has not returned for the ihc_timeout period, we should
643dbed73cbSSangeeta Misra  * kill it.  This function is the handler of this.
644dbed73cbSSangeeta Misra  */
645dbed73cbSSangeeta Misra /* ARGSUSED */
646dbed73cbSSangeeta Misra static void
ilbd_hc_kill_timer(iu_tq_t * tq,void * arg)647dbed73cbSSangeeta Misra ilbd_hc_kill_timer(iu_tq_t *tq, void *arg)
648dbed73cbSSangeeta Misra {
649dbed73cbSSangeeta Misra 	ilbd_hc_srv_t *srv = (ilbd_hc_srv_t *)arg;
650dbed73cbSSangeeta Misra 
651dbed73cbSSangeeta Misra 	ilbd_hc_kill_probe(srv);
652dbed73cbSSangeeta Misra 	ilbd_set_fail_state(srv);
653dbed73cbSSangeeta Misra }
654dbed73cbSSangeeta Misra 
655dbed73cbSSangeeta Misra /*
656dbed73cbSSangeeta Misra  * Probe timeout handler.  Send out the appropriate probe.
657dbed73cbSSangeeta Misra  */
658dbed73cbSSangeeta Misra /* ARGSUSED */
659dbed73cbSSangeeta Misra static void
ilbd_hc_probe_timer(iu_tq_t * tq,void * arg)660dbed73cbSSangeeta Misra ilbd_hc_probe_timer(iu_tq_t *tq, void *arg)
661dbed73cbSSangeeta Misra {
662dbed73cbSSangeeta Misra 	ilbd_hc_srv_t *srv = (ilbd_hc_srv_t *)arg;
663dbed73cbSSangeeta Misra 
664dbed73cbSSangeeta Misra 	/*
665dbed73cbSSangeeta Misra 	 * If starting the probe fails, just pretend that the timeout has
666dbed73cbSSangeeta Misra 	 * extended.
667dbed73cbSSangeeta Misra 	 */
668dbed73cbSSangeeta Misra 	if (!ilbd_run_probe(srv)) {
669dbed73cbSSangeeta Misra 		/*
670dbed73cbSSangeeta Misra 		 * If we cannot restart the timer, the only thing we can do
671dbed73cbSSangeeta Misra 		 * is to disable this server.  Hopefully the sys admin will
672dbed73cbSSangeeta Misra 		 * notice this and enable this server again later.
673dbed73cbSSangeeta Misra 		 */
674dbed73cbSSangeeta Misra 		if (ilbd_hc_restart_timer(srv->shc_hc, srv) != ILB_STATUS_OK) {
675dbed73cbSSangeeta Misra 			logerr("%s: cannot restart timer: rule %s server %s, "
676dbed73cbSSangeeta Misra 			    "disabling it", __func__,
677dbed73cbSSangeeta Misra 			    srv->shc_hc_rule->hcr_rule->irl_name,
678dbed73cbSSangeeta Misra 			    srv->shc_sg_srv->sgs_srvID);
679dbed73cbSSangeeta Misra 			ilbd_mark_server_disabled(srv);
680dbed73cbSSangeeta Misra 		}
681dbed73cbSSangeeta Misra 		return;
682dbed73cbSSangeeta Misra 	}
683dbed73cbSSangeeta Misra 
684dbed73cbSSangeeta Misra 	/*
685dbed73cbSSangeeta Misra 	 * Similar to above, if kill timer cannot be started, disable the
686dbed73cbSSangeeta Misra 	 * server.
687dbed73cbSSangeeta Misra 	 */
688dbed73cbSSangeeta Misra 	if ((srv->shc_tid = iu_schedule_timer(ilbd_hc_timer_q,
689dbed73cbSSangeeta Misra 	    srv->shc_hc->ihc_timeout, ilbd_hc_kill_timer, srv)) == -1) {
690dbed73cbSSangeeta Misra 		logerr("%s: cannot start kill timer: rule %s server %s, "
691dbed73cbSSangeeta Misra 		    "disabling it", __func__,
692dbed73cbSSangeeta Misra 		    srv->shc_hc_rule->hcr_rule->irl_name,
693dbed73cbSSangeeta Misra 		    srv->shc_sg_srv->sgs_srvID);
694dbed73cbSSangeeta Misra 		ilbd_mark_server_disabled(srv);
695dbed73cbSSangeeta Misra 	}
696dbed73cbSSangeeta Misra 	hc_timer_restarted = B_TRUE;
697dbed73cbSSangeeta Misra }
698dbed73cbSSangeeta Misra 
699dbed73cbSSangeeta Misra /* Restart the periodic timer for a given server. */
700dbed73cbSSangeeta Misra static ilb_status_t
ilbd_hc_restart_timer(ilbd_hc_t * hc,ilbd_hc_srv_t * srv)701dbed73cbSSangeeta Misra ilbd_hc_restart_timer(ilbd_hc_t *hc, ilbd_hc_srv_t *srv)
702dbed73cbSSangeeta Misra {
703dbed73cbSSangeeta Misra 	int timeout;
704dbed73cbSSangeeta Misra 
705dbed73cbSSangeeta Misra 	/* Don't allow the timeout interval to be less than 1s */
706dbed73cbSSangeeta Misra 	timeout = MAX((hc->ihc_interval >> 1) + (gethrtime() %
707dbed73cbSSangeeta Misra 	    (hc->ihc_interval + 1)), 1);
708dbed73cbSSangeeta Misra 
709dbed73cbSSangeeta Misra 	/*
710dbed73cbSSangeeta Misra 	 * If the probe is actually a ping probe, there is no need to
711dbed73cbSSangeeta Misra 	 * do default pinging.  Just skip the step.
712dbed73cbSSangeeta Misra 	 */
713dbed73cbSSangeeta Misra 	if (hc->ihc_def_ping && hc->ihc_test_type != ILBD_HC_PING)
714dbed73cbSSangeeta Misra 		srv->shc_state = ilbd_hc_def_pinging;
715dbed73cbSSangeeta Misra 	else
716dbed73cbSSangeeta Misra 		srv->shc_state = ilbd_hc_probing;
717dbed73cbSSangeeta Misra 	srv->shc_tid = iu_schedule_timer(ilbd_hc_timer_q, timeout,
718dbed73cbSSangeeta Misra 	    ilbd_hc_probe_timer, srv);
719dbed73cbSSangeeta Misra 
720dbed73cbSSangeeta Misra 	if (srv->shc_tid == -1)
721dbed73cbSSangeeta Misra 		return (ILB_STATUS_TIMER);
722dbed73cbSSangeeta Misra 	srv->shc_lasttime = time(NULL);
723dbed73cbSSangeeta Misra 	srv->shc_nexttime = time(NULL) + timeout;
724dbed73cbSSangeeta Misra 
725dbed73cbSSangeeta Misra 	hc_timer_restarted = B_TRUE;
726dbed73cbSSangeeta Misra 	return (ILB_STATUS_OK);
727dbed73cbSSangeeta Misra }
728dbed73cbSSangeeta Misra 
729dbed73cbSSangeeta Misra /* Helper routine to associate a server with its hc object. */
730dbed73cbSSangeeta Misra static ilb_status_t
ilbd_hc_srv_add(ilbd_hc_t * hc,ilbd_hc_rule_t * hc_rule,const ilb_sg_srv_t * srv,int ev_port)731dbed73cbSSangeeta Misra ilbd_hc_srv_add(ilbd_hc_t *hc, ilbd_hc_rule_t *hc_rule,
732dbed73cbSSangeeta Misra     const ilb_sg_srv_t *srv, int ev_port)
733dbed73cbSSangeeta Misra {
734dbed73cbSSangeeta Misra 	ilbd_hc_srv_t *new_srv;
735dbed73cbSSangeeta Misra 	ilb_status_t ret;
736dbed73cbSSangeeta Misra 
737dbed73cbSSangeeta Misra 	if ((new_srv = calloc(1, sizeof (ilbd_hc_srv_t))) == NULL)
738dbed73cbSSangeeta Misra 		return (ILB_STATUS_ENOMEM);
739dbed73cbSSangeeta Misra 	new_srv->shc_hc = hc;
740dbed73cbSSangeeta Misra 	new_srv->shc_hc_rule = hc_rule;
741dbed73cbSSangeeta Misra 	new_srv->shc_sg_srv = srv;
742dbed73cbSSangeeta Misra 	new_srv->shc_ev_port = ev_port;
743dbed73cbSSangeeta Misra 	new_srv->shc_tid = -1;
744dbed73cbSSangeeta Misra 	new_srv->shc_nexttime = time(NULL);
745dbed73cbSSangeeta Misra 	new_srv->shc_lasttime = new_srv->shc_nexttime;
746dbed73cbSSangeeta Misra 
747dbed73cbSSangeeta Misra 	if ((hc_rule->hcr_rule->irl_flags & ILB_FLAGS_RULE_ENABLED) &&
748dbed73cbSSangeeta Misra 	    ILB_IS_SRV_ENABLED(srv->sgs_flags)) {
749dbed73cbSSangeeta Misra 		new_srv->shc_status = ILB_HCS_UNINIT;
750dbed73cbSSangeeta Misra 		ret = ilbd_hc_restart_timer(hc, new_srv);
751dbed73cbSSangeeta Misra 		if (ret != ILB_STATUS_OK) {
752dbed73cbSSangeeta Misra 			free(new_srv);
753dbed73cbSSangeeta Misra 			return (ret);
754dbed73cbSSangeeta Misra 		}
755dbed73cbSSangeeta Misra 	} else {
756dbed73cbSSangeeta Misra 		new_srv->shc_status = ILB_HCS_DISABLED;
757dbed73cbSSangeeta Misra 	}
758dbed73cbSSangeeta Misra 
759dbed73cbSSangeeta Misra 	list_insert_tail(&hc_rule->hcr_servers, new_srv);
760dbed73cbSSangeeta Misra 	return (ILB_STATUS_OK);
761dbed73cbSSangeeta Misra }
762dbed73cbSSangeeta Misra 
763dbed73cbSSangeeta Misra /* Handy macro to cancel a server's timer. */
764dbed73cbSSangeeta Misra #define	HC_CANCEL_TIMER(srv)						\
765dbed73cbSSangeeta Misra {									\
766dbed73cbSSangeeta Misra 	void *arg;							\
767dbed73cbSSangeeta Misra 	int ret;							\
768dbed73cbSSangeeta Misra 	if ((srv)->shc_tid != -1) {					\
769dbed73cbSSangeeta Misra 		ret = iu_cancel_timer(ilbd_hc_timer_q, (srv)->shc_tid, &arg); \
770dbed73cbSSangeeta Misra 		(srv)->shc_tid = -1;					\
771dbed73cbSSangeeta Misra 		assert(ret == 1);					\
772dbed73cbSSangeeta Misra 		assert(arg == (srv));					\
773dbed73cbSSangeeta Misra 	}								\
774dbed73cbSSangeeta Misra 	hc_timer_restarted = B_TRUE;					\
775dbed73cbSSangeeta Misra }
776dbed73cbSSangeeta Misra 
777dbed73cbSSangeeta Misra /* Helper routine to dissociate a server from its hc object. */
778dbed73cbSSangeeta Misra static ilb_status_t
ilbd_hc_srv_rem(ilbd_hc_rule_t * hc_rule,const ilb_sg_srv_t * srv)779dbed73cbSSangeeta Misra ilbd_hc_srv_rem(ilbd_hc_rule_t *hc_rule, const ilb_sg_srv_t *srv)
780dbed73cbSSangeeta Misra {
781dbed73cbSSangeeta Misra 	ilbd_hc_srv_t *tmp_srv;
782dbed73cbSSangeeta Misra 
783dbed73cbSSangeeta Misra 	for (tmp_srv = list_head(&hc_rule->hcr_servers); tmp_srv != NULL;
784dbed73cbSSangeeta Misra 	    tmp_srv = list_next(&hc_rule->hcr_servers, tmp_srv)) {
785dbed73cbSSangeeta Misra 		if (tmp_srv->shc_sg_srv == srv) {
786dbed73cbSSangeeta Misra 			list_remove(&hc_rule->hcr_servers, tmp_srv);
787dbed73cbSSangeeta Misra 			HC_CANCEL_TIMER(tmp_srv);
788dbed73cbSSangeeta Misra 			if (tmp_srv->shc_child_pid != 0)
789dbed73cbSSangeeta Misra 				ilbd_hc_kill_probe(tmp_srv);
790dbed73cbSSangeeta Misra 			free(tmp_srv);
791dbed73cbSSangeeta Misra 			return (ILB_STATUS_OK);
792dbed73cbSSangeeta Misra 		}
793dbed73cbSSangeeta Misra 	}
794dbed73cbSSangeeta Misra 	return (ILB_STATUS_ENOENT);
795dbed73cbSSangeeta Misra }
796dbed73cbSSangeeta Misra 
797dbed73cbSSangeeta Misra /* Helper routine to dissociate all servers of a rule from its hc object. */
798dbed73cbSSangeeta Misra static void
ilbd_hc_srv_rem_all(ilbd_hc_rule_t * hc_rule)799dbed73cbSSangeeta Misra ilbd_hc_srv_rem_all(ilbd_hc_rule_t *hc_rule)
800dbed73cbSSangeeta Misra {
801dbed73cbSSangeeta Misra 	ilbd_hc_srv_t *srv;
802dbed73cbSSangeeta Misra 
803dbed73cbSSangeeta Misra 	while ((srv = list_remove_head(&hc_rule->hcr_servers)) != NULL) {
804dbed73cbSSangeeta Misra 		HC_CANCEL_TIMER(srv);
805dbed73cbSSangeeta Misra 		if (srv->shc_child_pid != 0)
806dbed73cbSSangeeta Misra 			ilbd_hc_kill_probe(srv);
807dbed73cbSSangeeta Misra 		free(srv);
808dbed73cbSSangeeta Misra 	}
809dbed73cbSSangeeta Misra }
810dbed73cbSSangeeta Misra 
811dbed73cbSSangeeta Misra /* Associate a rule with its hc object. */
812dbed73cbSSangeeta Misra ilb_status_t
ilbd_hc_associate_rule(const ilbd_rule_t * rule,int ev_port)813dbed73cbSSangeeta Misra ilbd_hc_associate_rule(const ilbd_rule_t *rule, int ev_port)
814dbed73cbSSangeeta Misra {
815dbed73cbSSangeeta Misra 	ilbd_hc_t	*hc;
816dbed73cbSSangeeta Misra 	ilbd_hc_rule_t	*hc_rule;
817dbed73cbSSangeeta Misra 	ilb_status_t	ret;
818dbed73cbSSangeeta Misra 	ilbd_sg_t	*sg;
819