xref: /illumos-gate/usr/src/cmd/svc/startd/method.c (revision 5f8fc372)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright 2018 Joyent, Inc.
25  */
26 
27 /*
28  * method.c - method execution functions
29  *
30  * This file contains the routines needed to run a method:  a fork(2)-exec(2)
31  * invocation monitored using either the contract filesystem or waitpid(2).
32  * (Plain fork1(2) support is provided in fork.c.)
33  *
34  * Contract Transfer
35  *   When we restart a service, we want to transfer any contracts that the old
36  *   service's contract inherited.  This means that (a) we must not abandon the
37  *   old contract when the service dies and (b) we must write the id of the old
38  *   contract into the terms of the new contract.  There should be limits to
39  *   (a), though, since we don't want to keep the contract around forever.  To
40  *   this end we'll say that services in the offline state may have a contract
41  *   to be transfered and services in the disabled or maintenance states cannot.
42  *   This means that when a service transitions from online (or degraded) to
43  *   offline, the contract should be preserved, and when the service transitions
44  *   from offline to online (i.e., the start method), we'll transfer inherited
45  *   contracts.
46  */
47 
48 #include <sys/contract/process.h>
49 #include <sys/ctfs.h>
50 #include <sys/stat.h>
51 #include <sys/time.h>
52 #include <sys/types.h>
53 #include <sys/uio.h>
54 #include <sys/wait.h>
55 #include <alloca.h>
56 #include <assert.h>
57 #include <errno.h>
58 #include <fcntl.h>
59 #include <libcontract.h>
60 #include <libcontract_priv.h>
61 #include <libgen.h>
62 #include <librestart.h>
63 #include <libscf.h>
64 #include <limits.h>
65 #include <port.h>
66 #include <sac.h>
67 #include <signal.h>
68 #include <stdlib.h>
69 #include <string.h>
70 #include <strings.h>
71 #include <unistd.h>
72 #include <atomic.h>
73 #include <poll.h>
74 #include <libscf_priv.h>
75 
76 #include "startd.h"
77 
78 #define	SBIN_SH		"/sbin/sh"
79 
80 /*
81  * Used to tell if contracts are in the process of being
82  * stored into the svc.startd internal hash table.
83  */
84 volatile uint16_t	storing_contract = 0;
85 
86 /*
87  * Mapping from restart_on method-type to contract events.  Must correspond to
88  * enum method_restart_t.
89  */
90 static uint_t method_events[] = {
91 	/* METHOD_RESTART_ALL */
92 	CT_PR_EV_HWERR | CT_PR_EV_SIGNAL | CT_PR_EV_CORE | CT_PR_EV_EMPTY,
93 	/* METHOD_RESTART_EXTERNAL_FAULT */
94 	CT_PR_EV_HWERR | CT_PR_EV_SIGNAL,
95 	/* METHOD_RESTART_ANY_FAULT */
96 	CT_PR_EV_HWERR | CT_PR_EV_SIGNAL | CT_PR_EV_CORE
97 };
98 
99 /*
100  * method_record_start(restarter_inst_t *)
101  *   Record a service start for rate limiting.  Place the current time
102  *   in the circular array of instance starts.
103  */
104 static void
method_record_start(restarter_inst_t * inst)105 method_record_start(restarter_inst_t *inst)
106 {
107 	int index = inst->ri_start_index++ % RINST_START_TIMES;
108 
109 	inst->ri_start_time[index] = gethrtime();
110 }
111 
112 /*
113  * method_rate_critical(restarter_inst_t *)
114  *    Return true if the average start interval is less than the permitted
115  *    interval.  The implicit interval defaults to RINST_FAILURE_RATE_NS and
116  *    RINST_START_TIMES but may be overridden with the svc properties
117  *    startd/critical_failure_count and startd/critical_failure_period
118  *    which represent the number of failures to consider and the amount of
119  *    time in seconds in which that number may occur, respectively. Note that
120  *    this time is measured as of the transition to 'enabled' rather than wall
121  *    clock time.
122  *    Implicit success if insufficient measurements for an average exist.
123  */
124 int
method_rate_critical(restarter_inst_t * inst)125 method_rate_critical(restarter_inst_t *inst)
126 {
127 	hrtime_t critical_failure_period;
128 	uint_t critical_failure_count = RINST_START_TIMES;
129 	uint_t n = inst->ri_start_index;
130 	hrtime_t avg_ns = 0;
131 	uint64_t scf_fr, scf_st;
132 	scf_propvec_t *prop = NULL;
133 	scf_propvec_t restart_critical[] = {
134 		{ "critical_failure_period", NULL, SCF_TYPE_INTEGER, NULL, 0 },
135 		{ "critical_failure_count", NULL, SCF_TYPE_INTEGER, NULL, 0 },
136 		{ NULL }
137 	};
138 
139 	if (instance_is_wait_style(inst))
140 		critical_failure_period = RINST_WT_SVC_FAILURE_RATE_NS;
141 	else
142 		critical_failure_period = RINST_FAILURE_RATE_NS;
143 
144 	restart_critical[0].pv_ptr = &scf_fr;
145 	restart_critical[1].pv_ptr = &scf_st;
146 
147 	if (scf_read_propvec(inst->ri_i.i_fmri, "startd",
148 	    B_TRUE, restart_critical, &prop) != SCF_FAILED) {
149 		/*
150 		 * critical_failure_period is expressed
151 		 * in seconds but tracked in ns
152 		 */
153 		critical_failure_period = (hrtime_t)scf_fr * NANOSEC;
154 		critical_failure_count = (uint_t)scf_st;
155 	}
156 	if (inst->ri_start_index < critical_failure_count)
157 		return (0);
158 
159 	avg_ns =
160 	    (inst->ri_start_time[(n - 1) % critical_failure_count] -
161 	    inst->ri_start_time[n % critical_failure_count]) /
162 	    (critical_failure_count - 1);
163 
164 	return (avg_ns < critical_failure_period);
165 }
166 
167 /*
168  * int method_is_transient()
169  *   Determine if the method for the given instance is transient,
170  *   from a contract perspective. Return 1 if it is, and 0 if it isn't.
171  */
172 static int
method_is_transient(restarter_inst_t * inst,int type)173 method_is_transient(restarter_inst_t *inst, int type)
174 {
175 	if (instance_is_transient_style(inst) || type != METHOD_START)
176 		return (1);
177 	else
178 		return (0);
179 }
180 
181 /*
182  * void method_store_contract()
183  *   Store the newly created contract id into local structures and
184  *   the repository.  If the repository connection is broken it is rebound.
185  */
186 static void
method_store_contract(restarter_inst_t * inst,int type,ctid_t * cid)187 method_store_contract(restarter_inst_t *inst, int type, ctid_t *cid)
188 {
189 	int r;
190 	boolean_t primary;
191 
192 	if (errno = contract_latest(cid))
193 		uu_die("%s: Couldn't get new contract's id", inst->ri_i.i_fmri);
194 
195 	primary = !method_is_transient(inst, type);
196 
197 	if (!primary) {
198 		if (inst->ri_i.i_transient_ctid != 0) {
199 			log_framework(LOG_INFO,
200 			    "%s: transient ctid expected to be 0 but "
201 			    "was set to %ld\n", inst->ri_i.i_fmri,
202 			    inst->ri_i.i_transient_ctid);
203 		}
204 
205 		inst->ri_i.i_transient_ctid = *cid;
206 	} else {
207 		if (inst->ri_i.i_primary_ctid != 0) {
208 			/*
209 			 * There was an old contract that we transferred.
210 			 * Remove it.
211 			 */
212 			method_remove_contract(inst, B_TRUE, B_FALSE);
213 		}
214 
215 		if (inst->ri_i.i_primary_ctid != 0) {
216 			log_framework(LOG_INFO,
217 			    "%s: primary ctid expected to be 0 but "
218 			    "was set to %ld\n", inst->ri_i.i_fmri,
219 			    inst->ri_i.i_primary_ctid);
220 		}
221 
222 		inst->ri_i.i_primary_ctid = *cid;
223 		inst->ri_i.i_primary_ctid_stopped = 0;
224 
225 		log_framework(LOG_DEBUG, "Storing primary contract %ld for "
226 		    "%s.\n", *cid, inst->ri_i.i_fmri);
227 
228 		contract_hash_store(*cid, inst->ri_id);
229 	}
230 
231 again:
232 	if (inst->ri_mi_deleted)
233 		return;
234 
235 	r = restarter_store_contract(inst->ri_m_inst, *cid, primary ?
236 	    RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
237 	switch (r) {
238 	case 0:
239 		break;
240 
241 	case ECANCELED:
242 		inst->ri_mi_deleted = B_TRUE;
243 		break;
244 
245 	case ECONNABORTED:
246 		libscf_handle_rebind(scf_instance_handle(inst->ri_m_inst));
247 		/* FALLTHROUGH */
248 
249 	case EBADF:
250 		libscf_reget_instance(inst);
251 		goto again;
252 
253 	case ENOMEM:
254 	case EPERM:
255 	case EACCES:
256 	case EROFS:
257 		uu_die("%s: Couldn't store contract id %ld",
258 		    inst->ri_i.i_fmri, *cid);
259 		/* NOTREACHED */
260 
261 	case EINVAL:
262 	default:
263 		bad_error("restarter_store_contract", r);
264 	}
265 }
266 
267 /*
268  * void method_remove_contract()
269  *   Remove any non-permanent contracts from internal structures and
270  *   the repository, then abandon them.
271  *   Returns
272  *     0 - success
273  *     ECANCELED - inst was deleted from the repository
274  *
275  *   If the repository connection was broken, it is rebound.
276  */
277 void
method_remove_contract(restarter_inst_t * inst,boolean_t primary,boolean_t abandon)278 method_remove_contract(restarter_inst_t *inst, boolean_t primary,
279     boolean_t abandon)
280 {
281 	ctid_t * const ctidp = primary ? &inst->ri_i.i_primary_ctid :
282 	    &inst->ri_i.i_transient_ctid;
283 
284 	int r;
285 
286 	assert(*ctidp != 0);
287 
288 	log_framework(LOG_DEBUG, "Removing %s contract %lu for %s.\n",
289 	    primary ? "primary" : "transient", *ctidp, inst->ri_i.i_fmri);
290 
291 	if (abandon)
292 		contract_abandon(*ctidp);
293 
294 again:
295 	if (inst->ri_mi_deleted) {
296 		r = ECANCELED;
297 		goto out;
298 	}
299 
300 	r = restarter_remove_contract(inst->ri_m_inst, *ctidp, primary ?
301 	    RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
302 	switch (r) {
303 	case 0:
304 		break;
305 
306 	case ECANCELED:
307 		inst->ri_mi_deleted = B_TRUE;
308 		break;
309 
310 	case ECONNABORTED:
311 		libscf_handle_rebind(scf_instance_handle(inst->ri_m_inst));
312 		/* FALLTHROUGH */
313 
314 	case EBADF:
315 		libscf_reget_instance(inst);
316 		goto again;
317 
318 	case ENOMEM:
319 	case EPERM:
320 	case EACCES:
321 	case EROFS:
322 		log_error(LOG_INFO, "%s: Couldn't remove contract id %ld: "
323 		    "%s.\n", inst->ri_i.i_fmri, *ctidp, strerror(r));
324 		break;
325 
326 	case EINVAL:
327 	default:
328 		bad_error("restarter_remove_contract", r);
329 	}
330 
331 out:
332 	if (primary)
333 		contract_hash_remove(*ctidp);
334 
335 	*ctidp = 0;
336 }
337 
338 static const char *method_names[] = { "start", "stop", "refresh" };
339 
340 /*
341  * int method_ready_contract(restarter_inst_t *, int, method_restart_t, int)
342  *
343  *   Activate a contract template for the type method of inst.  type,
344  *   restart_on, and cte_mask dictate the critical events term of the contract.
345  *   Returns
346  *     0 - success
347  *     ECANCELED - inst has been deleted from the repository
348  */
349 static int
method_ready_contract(restarter_inst_t * inst,int type,method_restart_t restart_on,uint_t cte_mask)350 method_ready_contract(restarter_inst_t *inst, int type,
351     method_restart_t restart_on, uint_t cte_mask)
352 {
353 	int tmpl, err, istrans, iswait, ret;
354 	uint_t cevents, fevents;
355 
356 	/*
357 	 * Correctly supporting wait-style services is tricky without
358 	 * rearchitecting startd to cope with multiple event sources
359 	 * simultaneously trying to stop an instance.  Until a better
360 	 * solution is implemented, we avoid this problem for
361 	 * wait-style services by making contract events fatal and
362 	 * letting the wait code alone handle stopping the service.
363 	 */
364 	iswait = instance_is_wait_style(inst);
365 	istrans = method_is_transient(inst, type);
366 
367 	tmpl = open64(CTFS_ROOT "/process/template", O_RDWR);
368 	if (tmpl == -1)
369 		uu_die("Could not create contract template");
370 
371 	/*
372 	 * We assume non-login processes are unlikely to create
373 	 * multiple process groups, and set CT_PR_PGRPONLY for all
374 	 * wait-style services' contracts.
375 	 */
376 	err = ct_pr_tmpl_set_param(tmpl, CT_PR_INHERIT | CT_PR_REGENT |
377 	    (iswait ? CT_PR_PGRPONLY : 0));
378 	assert(err == 0);
379 
380 	if (istrans) {
381 		cevents = 0;
382 		fevents = 0;
383 	} else {
384 		assert(restart_on >= 0);
385 		assert(restart_on <= METHOD_RESTART_ANY_FAULT);
386 		cevents = method_events[restart_on] & ~cte_mask;
387 		fevents = iswait ?
388 		    (method_events[restart_on] & ~cte_mask & CT_PR_ALLFATAL) :
389 		    0;
390 	}
391 
392 	err = ct_tmpl_set_critical(tmpl, cevents);
393 	assert(err == 0);
394 
395 	err = ct_tmpl_set_informative(tmpl, 0);
396 	assert(err == 0);
397 	err = ct_pr_tmpl_set_fatal(tmpl, fevents);
398 	assert(err == 0);
399 
400 	err = ct_tmpl_set_cookie(tmpl, istrans ?  METHOD_OTHER_COOKIE :
401 	    METHOD_START_COOKIE);
402 	assert(err == 0);
403 
404 	if (type == METHOD_START && inst->ri_i.i_primary_ctid != 0) {
405 		ret = ct_pr_tmpl_set_transfer(tmpl, inst->ri_i.i_primary_ctid);
406 		switch (ret) {
407 		case 0:
408 			break;
409 
410 		case ENOTEMPTY:
411 			/* No contracts for you! */
412 			method_remove_contract(inst, B_TRUE, B_TRUE);
413 			if (inst->ri_mi_deleted) {
414 				ret = ECANCELED;
415 				goto out;
416 			}
417 			break;
418 
419 		case EINVAL:
420 		case ESRCH:
421 		case EACCES:
422 		default:
423 			bad_error("ct_pr_tmpl_set_transfer", ret);
424 		}
425 	}
426 
427 	err = ct_pr_tmpl_set_svc_fmri(tmpl, inst->ri_i.i_fmri);
428 	assert(err == 0);
429 	err = ct_pr_tmpl_set_svc_aux(tmpl, method_names[type]);
430 	assert(err == 0);
431 
432 	err = ct_tmpl_activate(tmpl);
433 	assert(err == 0);
434 
435 	ret = 0;
436 
437 out:
438 	err = close(tmpl);
439 	assert(err == 0);
440 
441 	return (ret);
442 }
443 
444 static void
exec_method(const restarter_inst_t * inst,int type,const char * method,struct method_context * mcp,uint8_t need_session)445 exec_method(const restarter_inst_t *inst, int type, const char *method,
446     struct method_context *mcp, uint8_t need_session)
447 {
448 	char *cmd;
449 	const char *errf;
450 	char **nenv;
451 	int rsmc_errno = 0;
452 
453 	cmd = uu_msprintf("exec %s", method);
454 
455 	if (inst->ri_utmpx_prefix[0] != '\0' && inst->ri_utmpx_prefix != NULL)
456 		(void) utmpx_mark_init(getpid(), inst->ri_utmpx_prefix);
457 
458 	setlog(inst->ri_logstem);
459 	log_instance(inst, B_FALSE, "Executing %s method (\"%s\").",
460 	    method_names[type], method);
461 
462 	if (need_session)
463 		(void) setpgrp();
464 
465 	/* Set credentials. */
466 	rsmc_errno = restarter_set_method_context(mcp, &errf);
467 	if (rsmc_errno != 0) {
468 		log_instance(inst, B_FALSE,
469 		    "svc.startd could not set context for method: ");
470 
471 		if (rsmc_errno == -1) {
472 			if (strcmp(errf, "core_set_process_path") == 0) {
473 				log_instance(inst, B_FALSE,
474 				    "Could not set corefile path.");
475 			} else if (strcmp(errf, "setproject") == 0) {
476 				log_instance(inst, B_FALSE, "%s: a resource "
477 				    "control assignment failed", errf);
478 			} else if (strcmp(errf, "pool_set_binding") == 0) {
479 				log_instance(inst, B_FALSE, "%s: a system "
480 				    "error occurred", errf);
481 			} else {
482 #ifndef NDEBUG
483 				uu_warn("%s:%d: Bad function name \"%s\" for "
484 				    "error %d from "
485 				    "restarter_set_method_context().\n",
486 				    __FILE__, __LINE__, errf, rsmc_errno);
487 #endif
488 				abort();
489 			}
490 
491 			exit(1);
492 		}
493 
494 		if (errf != NULL && strcmp(errf, "pool_set_binding") == 0) {
495 			switch (rsmc_errno) {
496 			case ENOENT:
497 				log_instance(inst, B_FALSE, "%s: the pool "
498 				    "could not be found", errf);
499 				break;
500 
501 			case EBADF:
502 				log_instance(inst, B_FALSE, "%s: the "
503 				    "configuration is invalid", errf);
504 				break;
505 
506 			case EINVAL:
507 				log_instance(inst, B_FALSE, "%s: pool name "
508 				    "\"%s\" is invalid", errf,
509 				    mcp->resource_pool);
510 				break;
511 
512 			default:
513 #ifndef NDEBUG
514 				uu_warn("%s:%d: Bad error %d for function %s "
515 				    "in restarter_set_method_context().\n",
516 				    __FILE__, __LINE__, rsmc_errno, errf);
517 #endif
518 				abort();
519 			}
520 
521 			exit(SMF_EXIT_ERR_CONFIG);
522 		}
523 
524 		if (errf != NULL && strcmp(errf, "chdir") == 0) {
525 			switch (rsmc_errno) {
526 			case EACCES:
527 			case EFAULT:
528 			case EIO:
529 			case ELOOP:
530 			case ENAMETOOLONG:
531 			case ENOENT:
532 			case ENOLINK:
533 			case ENOTDIR:
534 				log_instance(inst, B_FALSE, "%s: %s (\"%s\")",
535 				    errf,
536 				    strerror(rsmc_errno), mcp->working_dir);
537 				break;
538 
539 			default:
540 #ifndef NDEBUG
541 				uu_warn("%s:%d: Bad error %d for function %s "
542 				    "in restarter_set_method_context().\n",
543 				    __FILE__, __LINE__, rsmc_errno, errf);
544 #endif
545 				abort();
546 			}
547 
548 			exit(SMF_EXIT_ERR_CONFIG);
549 		}
550 
551 		if (errf != NULL) {
552 			errno = rsmc_errno;
553 			perror(errf);
554 
555 			switch (rsmc_errno) {
556 			case EINVAL:
557 			case EPERM:
558 			case ENOENT:
559 			case ENAMETOOLONG:
560 			case ERANGE:
561 			case ESRCH:
562 				exit(SMF_EXIT_ERR_CONFIG);
563 				/* NOTREACHED */
564 
565 			default:
566 				exit(1);
567 			}
568 		}
569 
570 		switch (rsmc_errno) {
571 		case ENOMEM:
572 			log_instance(inst, B_FALSE, "Out of memory.");
573 			exit(1);
574 			/* NOTREACHED */
575 
576 		case ENOENT:
577 			log_instance(inst, B_FALSE, "Missing passwd entry for "
578 			    "user.");
579 			exit(SMF_EXIT_ERR_CONFIG);
580 			/* NOTREACHED */
581 
582 		default:
583 #ifndef NDEBUG
584 			uu_warn("%s:%d: Bad miscellaneous error %d from "
585 			    "restarter_set_method_context().\n", __FILE__,
586 			    __LINE__, rsmc_errno);
587 #endif
588 			abort();
589 		}
590 	}
591 
592 	nenv = set_smf_env(mcp->env, mcp->env_sz, NULL, inst,
593 	    method_names[type]);
594 
595 	log_preexec();
596 
597 	(void) execle(SBIN_SH, SBIN_SH, "-c", cmd, NULL, nenv);
598 
599 	exit(10);
600 }
601 
602 static void
write_status(restarter_inst_t * inst,const char * mname,int stat)603 write_status(restarter_inst_t *inst, const char *mname, int stat)
604 {
605 	int r;
606 
607 again:
608 	if (inst->ri_mi_deleted)
609 		return;
610 
611 	r = libscf_write_method_status(inst->ri_m_inst, mname, stat);
612 	switch (r) {
613 	case 0:
614 		break;
615 
616 	case ECONNABORTED:
617 		libscf_reget_instance(inst);
618 		goto again;
619 
620 	case ECANCELED:
621 		inst->ri_mi_deleted = 1;
622 		break;
623 
624 	case EPERM:
625 	case EACCES:
626 	case EROFS:
627 		log_framework(LOG_INFO, "Could not write exit status "
628 		    "for %s method of %s: %s.\n", mname,
629 		    inst->ri_i.i_fmri, strerror(r));
630 		break;
631 
632 	case ENAMETOOLONG:
633 	default:
634 		bad_error("libscf_write_method_status", r);
635 	}
636 }
637 
638 /*
639  * int method_run()
640  *   Execute the type method of instp.  If it requires a fork(), wait for it
641  *   to return and return its exit code in *exit_code.  Otherwise set
642  *   *exit_code to 0 if the method succeeds & -1 if it fails.  If the
643  *   repository connection is broken, it is rebound, but inst may not be
644  *   reset.
645  *   Returns
646  *     0 - success
647  *     EINVAL - A correct method or method context couldn't be retrieved.
648  *     EIO - Contract kill failed.
649  *     EFAULT - Method couldn't be executed successfully.
650  *     ELOOP - Retry threshold exceeded.
651  *     ECANCELED - inst was deleted from the repository before method was run
652  *     ERANGE - Timeout retry threshold exceeded.
653  *     EAGAIN - Failed due to external cause, retry.
654  */
655 int
method_run(restarter_inst_t ** instp,int type,int * exit_code)656 method_run(restarter_inst_t **instp, int type, int *exit_code)
657 {
658 	char *method;
659 	int ret_status;
660 	pid_t pid;
661 	method_restart_t restart_on;
662 	uint_t cte_mask;
663 	uint8_t need_session;
664 	scf_handle_t *h;
665 	scf_snapshot_t *snap;
666 	const char *mname;
667 	mc_error_t *m_error;
668 	struct method_context *mcp;
669 	int result = 0, timeout_fired = 0;
670 	int sig, r;
671 	boolean_t transient;
672 	uint64_t timeout;
673 	uint8_t timeout_retry;
674 	ctid_t ctid;
675 	int ctfd = -1;
676 	restarter_inst_t *inst = *instp;
677 	int id = inst->ri_id;
678 	int forkerr;
679 
680 	assert(MUTEX_HELD(&inst->ri_lock));
681 	assert(instance_in_transition(inst));
682 
683 	if (inst->ri_mi_deleted)
684 		return (ECANCELED);
685 
686 	*exit_code = SMF_EXIT_OK;
687 
688 	assert(0 <= type && type <= 2);
689 	mname = method_names[type];
690 
691 	if (type == METHOD_START)
692 		inst->ri_pre_online_hook();
693 
694 	h = scf_instance_handle(inst->ri_m_inst);
695 
696 	snap = scf_snapshot_create(h);
697 	if (snap == NULL ||
698 	    scf_instance_get_snapshot(inst->ri_m_inst, "running", snap) != 0) {
699 		log_framework(LOG_DEBUG,
700 		    "Could not get running snapshot for %s.  "
701 		    "Using editing version to run method %s.\n",
702 		    inst->ri_i.i_fmri, mname);
703 		scf_snapshot_destroy(snap);
704 		snap = NULL;
705 	}
706 
707 	/*
708 	 * After this point, we may be logging to the instance log.
709 	 * Make sure we've noted where that log is as a property of
710 	 * the instance.
711 	 */
712 	r = libscf_note_method_log(inst->ri_m_inst, st->st_log_prefix,
713 	    inst->ri_logstem);
714 	if (r != 0) {
715 		log_framework(LOG_WARNING,
716 		    "%s: couldn't note log location: %s\n",
717 		    inst->ri_i.i_fmri, strerror(r));
718 	}
719 
720 	if ((method = libscf_get_method(h, type, inst, snap, &restart_on,
721 	    &cte_mask, &need_session, &timeout, &timeout_retry)) == NULL) {
722 		if (errno == LIBSCF_PGROUP_ABSENT)  {
723 			log_framework(LOG_DEBUG,
724 			    "%s: instance has no method property group '%s'.\n",
725 			    inst->ri_i.i_fmri, mname);
726 			if (type == METHOD_REFRESH)
727 				log_instance(inst, B_TRUE, "No '%s' method "
728 				    "defined.  Treating as :true.", mname);
729 			else
730 				log_instance(inst, B_TRUE, "Method property "
731 				    "group '%s' is not present.", mname);
732 			scf_snapshot_destroy(snap);
733 			return (0);
734 		} else if (errno == LIBSCF_PROPERTY_ABSENT)  {
735 			log_framework(LOG_DEBUG,
736 			    "%s: instance has no '%s/exec' method property.\n",
737 			    inst->ri_i.i_fmri, mname);
738 			log_instance(inst, B_TRUE, "Method property '%s/exec "
739 			    "is not present.", mname);
740 			scf_snapshot_destroy(snap);
741 			return (0);
742 		} else {
743 			log_error(LOG_WARNING,
744 			    "%s: instance libscf_get_method failed\n",
745 			    inst->ri_i.i_fmri);
746 			scf_snapshot_destroy(snap);
747 			return (EINVAL);
748 		}
749 	}
750 
751 	/* open service contract if stopping a non-transient service */
752 	if (type == METHOD_STOP && (!instance_is_transient_style(inst))) {
753 		if (inst->ri_i.i_primary_ctid == 0) {
754 			/* service is not running, nothing to stop */
755 			log_framework(LOG_DEBUG, "%s: instance has no primary "
756 			    "contract, no service to stop.\n",
757 			    inst->ri_i.i_fmri);
758 			scf_snapshot_destroy(snap);
759 			return (0);
760 		}
761 		if ((ctfd = contract_open(inst->ri_i.i_primary_ctid, "process",
762 		    "events", O_RDONLY)) < 0) {
763 			result = EFAULT;
764 			log_instance(inst, B_TRUE, "Could not open service "
765 			    "contract %ld.  Stop method not run.",
766 			    inst->ri_i.i_primary_ctid);
767 			goto out;
768 		}
769 	}
770 
771 	if (restarter_is_null_method(method)) {
772 		log_framework(LOG_DEBUG, "%s: null method succeeds\n",
773 		    inst->ri_i.i_fmri);
774 
775 		log_instance(inst, B_TRUE, "Executing %s method (null).",
776 		    mname);
777 
778 		if (type == METHOD_START)
779 			write_status(inst, mname, 0);
780 		goto out;
781 	}
782 
783 	sig = restarter_is_kill_method(method);
784 	if (sig >= 0) {
785 
786 		if (inst->ri_i.i_primary_ctid == 0) {
787 			log_error(LOG_ERR, "%s: :kill with no contract\n",
788 			    inst->ri_i.i_fmri);
789 			log_instance(inst, B_TRUE, "Invalid use of \":kill\" "
790 			    "as stop method for transient service.");
791 			result = EINVAL;
792 			goto out;
793 		}
794 
795 		log_framework(LOG_DEBUG,
796 		    "%s: :killing contract with signal %d\n",
797 		    inst->ri_i.i_fmri, sig);
798 
799 		log_instance(inst, B_TRUE, "Executing %s method (:kill).",
800 		    mname);
801 
802 		if (contract_kill(inst->ri_i.i_primary_ctid, sig,
803 		    inst->ri_i.i_fmri) != 0) {
804 			result = EIO;
805 			goto out;
806 		} else
807 			goto assured_kill;
808 	}
809 
810 	log_framework(LOG_DEBUG, "%s: forking to run method %s\n",
811 	    inst->ri_i.i_fmri, method);
812 
813 	m_error = restarter_get_method_context(RESTARTER_METHOD_CONTEXT_VERSION,
814 	    inst->ri_m_inst, snap, mname, method, &mcp);
815 
816 	if (m_error != NULL) {
817 		log_instance(inst, B_TRUE, "%s", m_error->msg);
818 		restarter_mc_error_destroy(m_error);
819 		result = EINVAL;
820 		goto out;
821 	}
822 
823 	r = method_ready_contract(inst, type, restart_on, cte_mask);
824 	if (r != 0) {
825 		assert(r == ECANCELED);
826 		assert(inst->ri_mi_deleted);
827 		restarter_free_method_context(mcp);
828 		result = ECANCELED;
829 		goto out;
830 	}
831 
832 	/*
833 	 * Validate safety of method contexts, to save children work.
834 	 */
835 	if (!restarter_rm_libs_loadable())
836 		log_framework(LOG_DEBUG, "%s: method contexts limited "
837 		    "to root-accessible libraries\n", inst->ri_i.i_fmri);
838 
839 	/*
840 	 * For wait-style svc, sanity check that method exists to prevent an
841 	 * infinite loop.
842 	 */
843 	if (instance_is_wait_style(inst) && type == METHOD_START) {
844 		char *pend;
845 		struct stat64 sbuf;
846 
847 		/*
848 		 * We need to handle start method strings that have arguments,
849 		 * such as '/lib/svc/method/console-login %i'.
850 		 */
851 		if ((pend = strchr(method, ' ')) != NULL)
852 			*pend = '\0';
853 
854 		if (*method == '/' && stat64(method, &sbuf) == -1 &&
855 		    errno == ENOENT) {
856 			log_instance(inst, B_TRUE, "Missing start method (%s), "
857 			    "changing state to maintenance.", method);
858 			restarter_free_method_context(mcp);
859 			result = ENOENT;
860 			goto out;
861 		}
862 		if (pend != NULL)
863 			*pend = ' ';
864 	}
865 
866 	/*
867 	 * If the service is restarting too quickly, send it to
868 	 * maintenance.
869 	 */
870 	if (type == METHOD_START) {
871 		method_record_start(inst);
872 		if (method_rate_critical(inst) &&
873 		    !instance_is_wait_style(inst)) {
874 			log_instance(inst, B_TRUE, "Restarting too quickly, "
875 			    "changing state to maintenance.");
876 			result = ELOOP;
877 			restarter_free_method_context(mcp);
878 			goto out;
879 		}
880 	}
881 
882 	atomic_add_16(&storing_contract, 1);
883 	pid = startd_fork1(&forkerr);
884 	if (pid == 0)
885 		exec_method(inst, type, method, mcp, need_session);
886 
887 	if (pid == -1) {
888 		atomic_add_16(&storing_contract, -1);
889 		if (forkerr == EAGAIN)
890 			result = EAGAIN;
891 		else
892 			result = EFAULT;
893 
894 		log_error(LOG_WARNING,
895 		    "%s: Couldn't fork to execute method %s: %s\n",
896 		    inst->ri_i.i_fmri, method, strerror(forkerr));
897 
898 		restarter_free_method_context(mcp);
899 		goto out;
900 	}
901 
902 
903 	/*
904 	 * Get the contract id, decide whether it is primary or transient, and
905 	 * stash it in inst & the repository.
906 	 */
907 	method_store_contract(inst, type, &ctid);
908 	atomic_add_16(&storing_contract, -1);
909 
910 	restarter_free_method_context(mcp);
911 
912 	/*
913 	 * Similarly for the start method PID.
914 	 */
915 	if (type == METHOD_START && !inst->ri_mi_deleted)
916 		(void) libscf_write_start_pid(inst->ri_m_inst, pid);
917 
918 	if (instance_is_wait_style(inst) && type == METHOD_START) {
919 		/* Wait style instances don't get timeouts on start methods. */
920 		if (wait_register(pid, inst->ri_i.i_fmri, 1, 0)) {
921 			log_error(LOG_WARNING,
922 			    "%s: couldn't register %ld for wait\n",
923 			    inst->ri_i.i_fmri, pid);
924 			result = EFAULT;
925 			goto contract_out;
926 		}
927 		write_status(inst, mname, 0);
928 
929 	} else {
930 		int r, err;
931 		time_t start_time;
932 		time_t end_time;
933 
934 		/*
935 		 * Because on upgrade/live-upgrade we may have no chance
936 		 * to override faulty timeout values on the way to
937 		 * manifest import, all services on the path to manifest
938 		 * import are treated the same as INFINITE timeout services.
939 		 */
940 
941 		start_time = time(NULL);
942 		if (timeout != METHOD_TIMEOUT_INFINITE && !is_timeout_ovr(inst))
943 			timeout_insert(inst, ctid, timeout);
944 		else
945 			timeout = METHOD_TIMEOUT_INFINITE;
946 
947 		/* Unlock the instance while waiting for the method. */
948 		MUTEX_UNLOCK(&inst->ri_lock);
949 
950 		do {
951 			r = waitpid(pid, &ret_status, 0);
952 		} while (r == -1 && errno == EINTR);
953 		if (r == -1)
954 			err = errno;
955 
956 		/* Re-grab the lock. */
957 		inst = inst_lookup_by_id(id);
958 
959 		/*
960 		 * inst can't be removed, as the removal thread waits
961 		 * for completion of this one.
962 		 */
963 		assert(inst != NULL);
964 		*instp = inst;
965 
966 		if (inst->ri_timeout != NULL && inst->ri_timeout->te_fired)
967 			timeout_fired = 1;
968 
969 		timeout_remove(inst, ctid);
970 
971 		log_framework(LOG_DEBUG,
972 		    "%s method for %s exited with status %d.\n", mname,
973 		    inst->ri_i.i_fmri, WEXITSTATUS(ret_status));
974 
975 		if (r == -1) {
976 			log_error(LOG_WARNING,
977 			    "Couldn't waitpid() for %s method of %s (%s).\n",
978 			    mname, inst->ri_i.i_fmri, strerror(err));
979 			result = EFAULT;
980 			goto contract_out;
981 		}
982 
983 		if (type == METHOD_START)
984 			write_status(inst, mname, ret_status);
985 
986 		/* return ERANGE if this service doesn't retry on timeout */
987 		if (timeout_fired == 1 && timeout_retry == 0) {
988 			result = ERANGE;
989 			goto contract_out;
990 		}
991 
992 		if (!WIFEXITED(ret_status) &&
993 		    WEXITSTATUS(ret_status) != SMF_EXIT_NODAEMON) {
994 			/*
995 			 * If method didn't exit itself (it was killed by an
996 			 * external entity, etc.), consider the entire
997 			 * method_run as failed.
998 			 */
999 			if (WIFSIGNALED(ret_status)) {
1000 				char buf[SIG2STR_MAX];
1001 				(void) sig2str(WTERMSIG(ret_status), buf);
1002 
1003 				log_error(LOG_WARNING, "%s: Method \"%s\" "
1004 				    "failed due to signal %s.\n",
1005 				    inst->ri_i.i_fmri, method, buf);
1006 				log_instance(inst, B_TRUE, "Method \"%s\" "
1007 				    "failed due to signal %s.", mname, buf);
1008 			} else {
1009 				log_error(LOG_WARNING, "%s: Method \"%s\" "
1010 				    "failed with exit status %d.\n",
1011 				    inst->ri_i.i_fmri, method,
1012 				    WEXITSTATUS(ret_status));
1013 				log_instance(inst, B_TRUE, "Method \"%s\" "
1014 				    "failed with exit status %d.", mname,
1015 				    WEXITSTATUS(ret_status));
1016 			}
1017 			result = EAGAIN;
1018 			goto contract_out;
1019 		}
1020 
1021 		*exit_code = WEXITSTATUS(ret_status);
1022 		if (*exit_code != SMF_EXIT_OK &&
1023 		    *exit_code != SMF_EXIT_NODAEMON) {
1024 			log_error(LOG_WARNING,
1025 			    "%s: Method \"%s\" failed with exit status %d.\n",
1026 			    inst->ri_i.i_fmri, method, WEXITSTATUS(ret_status));
1027 		}
1028 
1029 		log_instance(inst, B_TRUE, "Method \"%s\" exited with status "
1030 		    "%d.", mname, *exit_code);
1031 
1032 		/* Note: we will take this path for SMF_EXIT_NODAEMON */
1033 		if (*exit_code != SMF_EXIT_OK)
1034 			goto contract_out;
1035 
1036 		end_time = time(NULL);
1037 
1038 		/* Give service contract remaining seconds to empty */
1039 		if (timeout != METHOD_TIMEOUT_INFINITE)
1040 			timeout -= (end_time - start_time);
1041 	}
1042 
1043 assured_kill:
1044 	/*
1045 	 * For stop methods, assure that the service contract has emptied
1046 	 * before returning.
1047 	 */
1048 	if (type == METHOD_STOP && (!instance_is_transient_style(inst)) &&
1049 	    !(contract_is_empty(inst->ri_i.i_primary_ctid))) {
1050 		int times = 0;
1051 
1052 		if (timeout != METHOD_TIMEOUT_INFINITE)
1053 			timeout_insert(inst, inst->ri_i.i_primary_ctid,
1054 			    timeout);
1055 
1056 		for (;;) {
1057 			/*
1058 			 * Check frequently at first, then back off.  This
1059 			 * keeps startd from idling while shutting down.
1060 			 */
1061 			if (times < 20) {
1062 				(void) poll(NULL, 0, 5);
1063 				times++;
1064 			} else {
1065 				(void) poll(NULL, 0, 100);
1066 			}
1067 			if (contract_is_empty(inst->ri_i.i_primary_ctid))
1068 				break;
1069 		}
1070 
1071 		if (timeout != METHOD_TIMEOUT_INFINITE)
1072 			if (inst->ri_timeout->te_fired)
1073 				result = EFAULT;
1074 
1075 		timeout_remove(inst, inst->ri_i.i_primary_ctid);
1076 	}
1077 
1078 contract_out:
1079 	/*
1080 	 * Abandon contracts for transient methods, methods that exit with
1081 	 * SMF_EXIT_NODAEMON & methods that fail.
1082 	 */
1083 	transient = method_is_transient(inst, type);
1084 	if ((transient || *exit_code != SMF_EXIT_OK || result != 0) &&
1085 	    (restarter_is_kill_method(method) < 0))
1086 		method_remove_contract(inst, !transient, B_TRUE);
1087 
1088 out:
1089 	if (ctfd >= 0)
1090 		(void) close(ctfd);
1091 	scf_snapshot_destroy(snap);
1092 	free(method);
1093 	return (result);
1094 }
1095 
1096 /*
1097  * The method thread executes a service method to effect a state transition.
1098  * The next_state of info->sf_id should be non-_NONE on entrance, and it will
1099  * be _NONE on exit (state will either be what next_state was (on success), or
1100  * it will be _MAINT (on error)).
1101  *
1102  * There are six classes of methods to consider: start & other (stop, refresh)
1103  * for each of "normal" services, wait services, and transient services.  For
1104  * each, the method must be fetched from the repository & executed.  fork()ed
1105  * methods must be waited on, except for the start method of wait services
1106  * (which must be registered with the wait subsystem via wait_register()).  If
1107  * the method succeeded (returned 0), then for start methods its contract
1108  * should be recorded as the primary contract for the service.  For other
1109  * methods, it should be abandoned.  If the method fails, then depending on
1110  * the failure, either the method should be reexecuted or the service should
1111  * be put into maintenance.  Either way the contract should be abandoned.
1112  */
1113 void *
method_thread(void * arg)1114 method_thread(void *arg)
1115 {
1116 	fork_info_t *info = arg;
1117 	restarter_inst_t *inst;
1118 	scf_handle_t	*local_handle;
1119 	scf_instance_t	*s_inst = NULL;
1120 	int r, exit_code;
1121 	boolean_t retryable;
1122 	restarter_str_t reason;
1123 
1124 	(void) pthread_setname_np(pthread_self(), "method");
1125 
1126 	assert(0 <= info->sf_method_type && info->sf_method_type <= 2);
1127 
1128 	/* Get (and lock) the restarter_inst_t. */
1129 	inst = inst_lookup_by_id(info->sf_id);
1130 
1131 	assert(inst->ri_method_thread != 0);
1132 	assert(instance_in_transition(inst) == 1);
1133 
1134 	/*
1135 	 * We cannot leave this function with inst in transition, because
1136 	 * protocol.c withholds messages for inst otherwise.
1137 	 */
1138 
1139 	log_framework(LOG_DEBUG, "method_thread() running %s method for %s.\n",
1140 	    method_names[info->sf_method_type], inst->ri_i.i_fmri);
1141 
1142 	local_handle = libscf_handle_create_bound_loop();
1143 
1144 rebind_retry:
1145 	/* get scf_instance_t */
1146 	switch (r = libscf_fmri_get_instance(local_handle, inst->ri_i.i_fmri,
1147 	    &s_inst)) {
1148 	case 0:
1149 		break;
1150 
1151 	case ECONNABORTED:
1152 		libscf_handle_rebind(local_handle);
1153 		goto rebind_retry;
1154 
1155 	case ENOENT:
1156 		/*
1157 		 * It's not there, but we need to call this so protocol.c
1158 		 * doesn't think it's in transition anymore.
1159 		 */
1160 		(void) restarter_instance_update_states(local_handle, inst,
1161 		    inst->ri_i.i_state, RESTARTER_STATE_NONE, RERR_NONE,
1162 		    restarter_str_none);
1163 		goto out;
1164 
1165 	case EINVAL:
1166 	case ENOTSUP:
1167 	default:
1168 		bad_error("libscf_fmri_get_instance", r);
1169 	}
1170 
1171 	inst->ri_m_inst = s_inst;
1172 	inst->ri_mi_deleted = B_FALSE;
1173 
1174 retry:
1175 	if (info->sf_method_type == METHOD_START)
1176 		log_transition(inst, START_REQUESTED);
1177 
1178 	r = method_run(&inst, info->sf_method_type, &exit_code);
1179 
1180 	if (r == 0 &&
1181 	    (exit_code == SMF_EXIT_OK || exit_code == SMF_EXIT_NODAEMON)) {
1182 		/* Success! */
1183 		assert(inst->ri_i.i_next_state != RESTARTER_STATE_NONE);
1184 
1185 		/*
1186 		 * When a stop method succeeds, remove the primary contract of
1187 		 * the service, unless we're going to offline, in which case
1188 		 * retain the contract so we can transfer inherited contracts to
1189 		 * the replacement service.
1190 		 */
1191 
1192 		if (info->sf_method_type == METHOD_STOP &&
1193 		    inst->ri_i.i_primary_ctid != 0) {
1194 			if (inst->ri_i.i_next_state == RESTARTER_STATE_OFFLINE)
1195 				inst->ri_i.i_primary_ctid_stopped = 1;
1196 			else
1197 				method_remove_contract(inst, B_TRUE, B_TRUE);
1198 		}
1199 
1200 		/*
1201 		 * For methods that exit with SMF_EXIT_NODAEMON, we already
1202 		 * called method_remove_contract in method_run.
1203 		 */
1204 
1205 		/*
1206 		 * We don't care whether the handle was rebound because this is
1207 		 * the last thing we do with it.
1208 		 */
1209 		(void) restarter_instance_update_states(local_handle, inst,
1210 		    inst->ri_i.i_next_state, RESTARTER_STATE_NONE,
1211 		    info->sf_event_type, info->sf_reason);
1212 
1213 		(void) update_fault_count(inst, FAULT_COUNT_RESET);
1214 
1215 		goto out;
1216 	}
1217 
1218 	/* Failure.  Retry or go to maintenance. */
1219 
1220 	if (r != 0 && r != EAGAIN) {
1221 		retryable = B_FALSE;
1222 	} else {
1223 		switch (exit_code) {
1224 		case SMF_EXIT_ERR_CONFIG:
1225 		case SMF_EXIT_ERR_NOSMF:
1226 		case SMF_EXIT_ERR_PERM:
1227 		case SMF_EXIT_ERR_FATAL:
1228 			retryable = B_FALSE;
1229 			break;
1230 
1231 		default:
1232 			retryable = B_TRUE;
1233 		}
1234 	}
1235 
1236 	if (retryable && update_fault_count(inst, FAULT_COUNT_INCR) != 1)
1237 		goto retry;
1238 
1239 	/* maintenance */
1240 	if (r == ELOOP)
1241 		log_transition(inst, START_FAILED_REPEATEDLY);
1242 	else if (r == ERANGE)
1243 		log_transition(inst, START_FAILED_TIMEOUT_FATAL);
1244 	else if (exit_code == SMF_EXIT_ERR_CONFIG)
1245 		log_transition(inst, START_FAILED_CONFIGURATION);
1246 	else if (exit_code == SMF_EXIT_ERR_FATAL)
1247 		log_transition(inst, START_FAILED_FATAL);
1248 	else
1249 		log_transition(inst, START_FAILED_OTHER);
1250 
1251 	if (r == ELOOP) {
1252 		reason = restarter_str_restarting_too_quickly;
1253 	} else if (retryable) {
1254 		reason = restarter_str_fault_threshold_reached;
1255 	} else {
1256 		reason = restarter_str_method_failed;
1257 	}
1258 
1259 	(void) restarter_instance_update_states(local_handle, inst,
1260 	    RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE, RERR_FAULT,
1261 	    reason);
1262 
1263 	if (!method_is_transient(inst, info->sf_method_type) &&
1264 	    inst->ri_i.i_primary_ctid != 0)
1265 		method_remove_contract(inst, B_TRUE, B_TRUE);
1266 
1267 out:
1268 	inst->ri_method_thread = 0;
1269 
1270 	/*
1271 	 * Unlock the mutex after broadcasting to avoid a race condition
1272 	 * with restarter_delete_inst() when the 'inst' structure is freed.
1273 	 */
1274 	(void) pthread_cond_broadcast(&inst->ri_method_cv);
1275 	MUTEX_UNLOCK(&inst->ri_lock);
1276 
1277 	scf_instance_destroy(s_inst);
1278 	scf_handle_destroy(local_handle);
1279 	startd_free(info, sizeof (fork_info_t));
1280 	return (NULL);
1281 }
1282