1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2018 Joyent, Inc.
25 */
26
27/*
28 * method.c - method execution functions
29 *
30 * This file contains the routines needed to run a method:  a fork(2)-exec(2)
31 * invocation monitored using either the contract filesystem or waitpid(2).
32 * (Plain fork1(2) support is provided in fork.c.)
33 *
34 * Contract Transfer
35 *   When we restart a service, we want to transfer any contracts that the old
36 *   service's contract inherited.  This means that (a) we must not abandon the
37 *   old contract when the service dies and (b) we must write the id of the old
38 *   contract into the terms of the new contract.  There should be limits to
39 *   (a), though, since we don't want to keep the contract around forever.  To
40 *   this end we'll say that services in the offline state may have a contract
41 *   to be transfered and services in the disabled or maintenance states cannot.
42 *   This means that when a service transitions from online (or degraded) to
43 *   offline, the contract should be preserved, and when the service transitions
44 *   from offline to online (i.e., the start method), we'll transfer inherited
45 *   contracts.
46 */
47
48#include <sys/contract/process.h>
49#include <sys/ctfs.h>
50#include <sys/stat.h>
51#include <sys/time.h>
52#include <sys/types.h>
53#include <sys/uio.h>
54#include <sys/wait.h>
55#include <alloca.h>
56#include <assert.h>
57#include <errno.h>
58#include <fcntl.h>
59#include <libcontract.h>
60#include <libcontract_priv.h>
61#include <libgen.h>
62#include <librestart.h>
63#include <libscf.h>
64#include <limits.h>
65#include <port.h>
66#include <sac.h>
67#include <signal.h>
68#include <stdlib.h>
69#include <string.h>
70#include <strings.h>
71#include <unistd.h>
72#include <atomic.h>
73#include <poll.h>
74#include <libscf_priv.h>
75
76#include "startd.h"
77
78#define	SBIN_SH		"/sbin/sh"
79
80/*
81 * Used to tell if contracts are in the process of being
82 * stored into the svc.startd internal hash table.
83 */
84volatile uint16_t	storing_contract = 0;
85
86/*
87 * Mapping from restart_on method-type to contract events.  Must correspond to
88 * enum method_restart_t.
89 */
90static uint_t method_events[] = {
91	/* METHOD_RESTART_ALL */
92	CT_PR_EV_HWERR | CT_PR_EV_SIGNAL | CT_PR_EV_CORE | CT_PR_EV_EMPTY,
93	/* METHOD_RESTART_EXTERNAL_FAULT */
94	CT_PR_EV_HWERR | CT_PR_EV_SIGNAL,
95	/* METHOD_RESTART_ANY_FAULT */
96	CT_PR_EV_HWERR | CT_PR_EV_SIGNAL | CT_PR_EV_CORE
97};
98
99/*
100 * method_record_start(restarter_inst_t *)
101 *   Record a service start for rate limiting.  Place the current time
102 *   in the circular array of instance starts.
103 */
104static void
105method_record_start(restarter_inst_t *inst)
106{
107	int index = inst->ri_start_index++ % RINST_START_TIMES;
108
109	inst->ri_start_time[index] = gethrtime();
110}
111
112/*
113 * method_rate_critical(restarter_inst_t *)
114 *    Return true if the average start interval is less than the permitted
115 *    interval.  The implicit interval defaults to RINST_FAILURE_RATE_NS and
116 *    RINST_START_TIMES but may be overridden with the svc properties
117 *    startd/critical_failure_count and startd/critical_failure_period
118 *    which represent the number of failures to consider and the amount of
119 *    time in seconds in which that number may occur, respectively. Note that
120 *    this time is measured as of the transition to 'enabled' rather than wall
121 *    clock time.
122 *    Implicit success if insufficient measurements for an average exist.
123 */
124int
125method_rate_critical(restarter_inst_t *inst)
126{
127	hrtime_t critical_failure_period;
128	uint_t critical_failure_count = RINST_START_TIMES;
129	uint_t n = inst->ri_start_index;
130	hrtime_t avg_ns = 0;
131	uint64_t scf_fr, scf_st;
132	scf_propvec_t *prop = NULL;
133	scf_propvec_t restart_critical[] = {
134		{ "critical_failure_period", NULL, SCF_TYPE_INTEGER, NULL, 0 },
135		{ "critical_failure_count", NULL, SCF_TYPE_INTEGER, NULL, 0 },
136		{ NULL }
137	};
138
139	if (instance_is_wait_style(inst))
140		critical_failure_period = RINST_WT_SVC_FAILURE_RATE_NS;
141	else
142		critical_failure_period = RINST_FAILURE_RATE_NS;
143
144	restart_critical[0].pv_ptr = &scf_fr;
145	restart_critical[1].pv_ptr = &scf_st;
146
147	if (scf_read_propvec(inst->ri_i.i_fmri, "startd",
148	    B_TRUE, restart_critical, &prop) != SCF_FAILED) {
149		/*
150		 * critical_failure_period is expressed
151		 * in seconds but tracked in ns
152		 */
153		critical_failure_period = (hrtime_t)scf_fr * NANOSEC;
154		critical_failure_count = (uint_t)scf_st;
155	}
156	if (inst->ri_start_index < critical_failure_count)
157		return (0);
158
159	avg_ns =
160	    (inst->ri_start_time[(n - 1) % critical_failure_count] -
161	    inst->ri_start_time[n % critical_failure_count]) /
162	    (critical_failure_count - 1);
163
164	return (avg_ns < critical_failure_period);
165}
166
167/*
168 * int method_is_transient()
169 *   Determine if the method for the given instance is transient,
170 *   from a contract perspective. Return 1 if it is, and 0 if it isn't.
171 */
172static int
173method_is_transient(restarter_inst_t *inst, int type)
174{
175	if (instance_is_transient_style(inst) || type != METHOD_START)
176		return (1);
177	else
178		return (0);
179}
180
181/*
182 * void method_store_contract()
183 *   Store the newly created contract id into local structures and
184 *   the repository.  If the repository connection is broken it is rebound.
185 */
186static void
187method_store_contract(restarter_inst_t *inst, int type, ctid_t *cid)
188{
189	int r;
190	boolean_t primary;
191
192	if (errno = contract_latest(cid))
193		uu_die("%s: Couldn't get new contract's id", inst->ri_i.i_fmri);
194
195	primary = !method_is_transient(inst, type);
196
197	if (!primary) {
198		if (inst->ri_i.i_transient_ctid != 0) {
199			log_framework(LOG_INFO,
200			    "%s: transient ctid expected to be 0 but "
201			    "was set to %ld\n", inst->ri_i.i_fmri,
202			    inst->ri_i.i_transient_ctid);
203		}
204
205		inst->ri_i.i_transient_ctid = *cid;
206	} else {
207		if (inst->ri_i.i_primary_ctid != 0) {
208			/*
209			 * There was an old contract that we transferred.
210			 * Remove it.
211			 */
212			method_remove_contract(inst, B_TRUE, B_FALSE);
213		}
214
215		if (inst->ri_i.i_primary_ctid != 0) {
216			log_framework(LOG_INFO,
217			    "%s: primary ctid expected to be 0 but "
218			    "was set to %ld\n", inst->ri_i.i_fmri,
219			    inst->ri_i.i_primary_ctid);
220		}
221
222		inst->ri_i.i_primary_ctid = *cid;
223		inst->ri_i.i_primary_ctid_stopped = 0;
224
225		log_framework(LOG_DEBUG, "Storing primary contract %ld for "
226		    "%s.\n", *cid, inst->ri_i.i_fmri);
227
228		contract_hash_store(*cid, inst->ri_id);
229	}
230
231again:
232	if (inst->ri_mi_deleted)
233		return;
234
235	r = restarter_store_contract(inst->ri_m_inst, *cid, primary ?
236	    RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
237	switch (r) {
238	case 0:
239		break;
240
241	case ECANCELED:
242		inst->ri_mi_deleted = B_TRUE;
243		break;
244
245	case ECONNABORTED:
246		libscf_handle_rebind(scf_instance_handle(inst->ri_m_inst));
247		/* FALLTHROUGH */
248
249	case EBADF:
250		libscf_reget_instance(inst);
251		goto again;
252
253	case ENOMEM:
254	case EPERM:
255	case EACCES:
256	case EROFS:
257		uu_die("%s: Couldn't store contract id %ld",
258		    inst->ri_i.i_fmri, *cid);
259		/* NOTREACHED */
260
261	case EINVAL:
262	default:
263		bad_error("restarter_store_contract", r);
264	}
265}
266
267/*
268 * void method_remove_contract()
269 *   Remove any non-permanent contracts from internal structures and
270 *   the repository, then abandon them.
271 *   Returns
272 *     0 - success
273 *     ECANCELED - inst was deleted from the repository
274 *
275 *   If the repository connection was broken, it is rebound.
276 */
277void
278method_remove_contract(restarter_inst_t *inst, boolean_t primary,
279    boolean_t abandon)
280{
281	ctid_t * const ctidp = primary ? &inst->ri_i.i_primary_ctid :
282	    &inst->ri_i.i_transient_ctid;
283
284	int r;
285
286	assert(*ctidp != 0);
287
288	log_framework(LOG_DEBUG, "Removing %s contract %lu for %s.\n",
289	    primary ? "primary" : "transient", *ctidp, inst->ri_i.i_fmri);
290
291	if (abandon)
292		contract_abandon(*ctidp);
293
294again:
295	if (inst->ri_mi_deleted) {
296		r = ECANCELED;
297		goto out;
298	}
299
300	r = restarter_remove_contract(inst->ri_m_inst, *ctidp, primary ?
301	    RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
302	switch (r) {
303	case 0:
304		break;
305
306	case ECANCELED:
307		inst->ri_mi_deleted = B_TRUE;
308		break;
309
310	case ECONNABORTED:
311		libscf_handle_rebind(scf_instance_handle(inst->ri_m_inst));
312		/* FALLTHROUGH */
313
314	case EBADF:
315		libscf_reget_instance(inst);
316		goto again;
317
318	case ENOMEM:
319	case EPERM:
320	case EACCES:
321	case EROFS:
322		log_error(LOG_INFO, "%s: Couldn't remove contract id %ld: "
323		    "%s.\n", inst->ri_i.i_fmri, *ctidp, strerror(r));
324		break;
325
326	case EINVAL:
327	default:
328		bad_error("restarter_remove_contract", r);
329	}
330
331out:
332	if (primary)
333		contract_hash_remove(*ctidp);
334
335	*ctidp = 0;
336}
337
338static const char *method_names[] = { "start", "stop", "refresh" };
339
340/*
341 * int method_ready_contract(restarter_inst_t *, int, method_restart_t, int)
342 *
343 *   Activate a contract template for the type method of inst.  type,
344 *   restart_on, and cte_mask dictate the critical events term of the contract.
345 *   Returns
346 *     0 - success
347 *     ECANCELED - inst has been deleted from the repository
348 */
349static int
350method_ready_contract(restarter_inst_t *inst, int type,
351    method_restart_t restart_on, uint_t cte_mask)
352{
353	int tmpl, err, istrans, iswait, ret;
354	uint_t cevents, fevents;
355
356	/*
357	 * Correctly supporting wait-style services is tricky without
358	 * rearchitecting startd to cope with multiple event sources
359	 * simultaneously trying to stop an instance.  Until a better
360	 * solution is implemented, we avoid this problem for
361	 * wait-style services by making contract events fatal and
362	 * letting the wait code alone handle stopping the service.
363	 */
364	iswait = instance_is_wait_style(inst);
365	istrans = method_is_transient(inst, type);
366
367	tmpl = open64(CTFS_ROOT "/process/template", O_RDWR);
368	if (tmpl == -1)
369		uu_die("Could not create contract template");
370
371	/*
372	 * We assume non-login processes are unlikely to create
373	 * multiple process groups, and set CT_PR_PGRPONLY for all
374	 * wait-style services' contracts.
375	 */
376	err = ct_pr_tmpl_set_param(tmpl, CT_PR_INHERIT | CT_PR_REGENT |
377	    (iswait ? CT_PR_PGRPONLY : 0));
378	assert(err == 0);
379
380	if (istrans) {
381		cevents = 0;
382		fevents = 0;
383	} else {
384		assert(restart_on >= 0);
385		assert(restart_on <= METHOD_RESTART_ANY_FAULT);
386		cevents = method_events[restart_on] & ~cte_mask;
387		fevents = iswait ?
388		    (method_events[restart_on] & ~cte_mask & CT_PR_ALLFATAL) :
389		    0;
390	}
391
392	err = ct_tmpl_set_critical(tmpl, cevents);
393	assert(err == 0);
394
395	err = ct_tmpl_set_informative(tmpl, 0);
396	assert(err == 0);
397	err = ct_pr_tmpl_set_fatal(tmpl, fevents);
398	assert(err == 0);
399
400	err = ct_tmpl_set_cookie(tmpl, istrans ?  METHOD_OTHER_COOKIE :
401	    METHOD_START_COOKIE);
402	assert(err == 0);
403
404	if (type == METHOD_START && inst->ri_i.i_primary_ctid != 0) {
405		ret = ct_pr_tmpl_set_transfer(tmpl, inst->ri_i.i_primary_ctid);
406		switch (ret) {
407		case 0:
408			break;
409
410		case ENOTEMPTY:
411			/* No contracts for you! */
412			method_remove_contract(inst, B_TRUE, B_TRUE);
413			if (inst->ri_mi_deleted) {
414				ret = ECANCELED;
415				goto out;
416			}
417			break;
418
419		case EINVAL:
420		case ESRCH:
421		case EACCES:
422		default:
423			bad_error("ct_pr_tmpl_set_transfer", ret);
424		}
425	}
426
427	err = ct_pr_tmpl_set_svc_fmri(tmpl, inst->ri_i.i_fmri);
428	assert(err == 0);
429	err = ct_pr_tmpl_set_svc_aux(tmpl, method_names[type]);
430	assert(err == 0);
431
432	err = ct_tmpl_activate(tmpl);
433	assert(err == 0);
434
435	ret = 0;
436
437out:
438	err = close(tmpl);
439	assert(err == 0);
440
441	return (ret);
442}
443
444static void
445exec_method(const restarter_inst_t *inst, int type, const char *method,
446    struct method_context *mcp, uint8_t need_session)
447{
448	char *cmd;
449	const char *errf;
450	char **nenv;
451	int rsmc_errno = 0;
452
453	cmd = uu_msprintf("exec %s", method);
454
455	if (inst->ri_utmpx_prefix[0] != '\0' && inst->ri_utmpx_prefix != NULL)
456		(void) utmpx_mark_init(getpid(), inst->ri_utmpx_prefix);
457
458	setlog(inst->ri_logstem);
459	log_instance(inst, B_FALSE, "Executing %s method (\"%s\").",
460	    method_names[type], method);
461
462	if (need_session)
463		(void) setpgrp();
464
465	/* Set credentials. */
466	rsmc_errno = restarter_set_method_context(mcp, &errf);
467	if (rsmc_errno != 0) {
468		log_instance(inst, B_FALSE,
469		    "svc.startd could not set context for method: ");
470
471		if (rsmc_errno == -1) {
472			if (strcmp(errf, "core_set_process_path") == 0) {
473				log_instance(inst, B_FALSE,
474				    "Could not set corefile path.");
475			} else if (strcmp(errf, "setproject") == 0) {
476				log_instance(inst, B_FALSE, "%s: a resource "
477				    "control assignment failed", errf);
478			} else if (strcmp(errf, "pool_set_binding") == 0) {
479				log_instance(inst, B_FALSE, "%s: a system "
480				    "error occurred", errf);
481			} else {
482#ifndef NDEBUG
483				uu_warn("%s:%d: Bad function name \"%s\" for "
484				    "error %d from "
485				    "restarter_set_method_context().\n",
486				    __FILE__, __LINE__, errf, rsmc_errno);
487#endif
488				abort();
489			}
490
491			exit(1);
492		}
493
494		if (errf != NULL && strcmp(errf, "pool_set_binding") == 0) {
495			switch (rsmc_errno) {
496			case ENOENT:
497				log_instance(inst, B_FALSE, "%s: the pool "
498				    "could not be found", errf);
499				break;
500
501			case EBADF:
502				log_instance(inst, B_FALSE, "%s: the "
503				    "configuration is invalid", errf);
504				break;
505
506			case EINVAL:
507				log_instance(inst, B_FALSE, "%s: pool name "
508				    "\"%s\" is invalid", errf,
509				    mcp->resource_pool);
510				break;
511
512			default:
513#ifndef NDEBUG
514				uu_warn("%s:%d: Bad error %d for function %s "
515				    "in restarter_set_method_context().\n",
516				    __FILE__, __LINE__, rsmc_errno, errf);
517#endif
518				abort();
519			}
520
521			exit(SMF_EXIT_ERR_CONFIG);
522		}
523
524		if (errf != NULL && strcmp(errf, "chdir") == 0) {
525			switch (rsmc_errno) {
526			case EACCES:
527			case EFAULT:
528			case EIO:
529			case ELOOP:
530			case ENAMETOOLONG:
531			case ENOENT:
532			case ENOLINK:
533			case ENOTDIR:
534				log_instance(inst, B_FALSE, "%s: %s (\"%s\")",
535				    errf,
536				    strerror(rsmc_errno), mcp->working_dir);
537				break;
538
539			default:
540#ifndef NDEBUG
541				uu_warn("%s:%d: Bad error %d for function %s "
542				    "in restarter_set_method_context().\n",
543				    __FILE__, __LINE__, rsmc_errno, errf);
544#endif
545				abort();
546			}
547
548			exit(SMF_EXIT_ERR_CONFIG);
549		}
550
551		if (errf != NULL) {
552			errno = rsmc_errno;
553			perror(errf);
554
555			switch (rsmc_errno) {
556			case EINVAL:
557			case EPERM:
558			case ENOENT:
559			case ENAMETOOLONG:
560			case ERANGE:
561			case ESRCH:
562				exit(SMF_EXIT_ERR_CONFIG);
563				/* NOTREACHED */
564
565			default:
566				exit(1);
567			}
568		}
569
570		switch (rsmc_errno) {
571		case ENOMEM:
572			log_instance(inst, B_FALSE, "Out of memory.");
573			exit(1);
574			/* NOTREACHED */
575
576		case ENOENT:
577			log_instance(inst, B_FALSE, "Missing passwd entry for "
578			    "user.");
579			exit(SMF_EXIT_ERR_CONFIG);
580			/* NOTREACHED */
581
582		default:
583#ifndef NDEBUG
584			uu_warn("%s:%d: Bad miscellaneous error %d from "
585			    "restarter_set_method_context().\n", __FILE__,
586			    __LINE__, rsmc_errno);
587#endif
588			abort();
589		}
590	}
591
592	nenv = set_smf_env(mcp->env, mcp->env_sz, NULL, inst,
593	    method_names[type]);
594
595	log_preexec();
596
597	(void) execle(SBIN_SH, SBIN_SH, "-c", cmd, NULL, nenv);
598
599	exit(10);
600}
601
602static void
603write_status(restarter_inst_t *inst, const char *mname, int stat)
604{
605	int r;
606
607again:
608	if (inst->ri_mi_deleted)
609		return;
610
611	r = libscf_write_method_status(inst->ri_m_inst, mname, stat);
612	switch (r) {
613	case 0:
614		break;
615
616	case ECONNABORTED:
617		libscf_reget_instance(inst);
618		goto again;
619
620	case ECANCELED:
621		inst->ri_mi_deleted = 1;
622		break;
623
624	case EPERM:
625	case EACCES:
626	case EROFS:
627		log_framework(LOG_INFO, "Could not write exit status "
628		    "for %s method of %s: %s.\n", mname,
629		    inst->ri_i.i_fmri, strerror(r));
630		break;
631
632	case ENAMETOOLONG:
633	default:
634		bad_error("libscf_write_method_status", r);
635	}
636}
637
638/*
639 * int method_run()
640 *   Execute the type method of instp.  If it requires a fork(), wait for it
641 *   to return and return its exit code in *exit_code.  Otherwise set
642 *   *exit_code to 0 if the method succeeds & -1 if it fails.  If the
643 *   repository connection is broken, it is rebound, but inst may not be
644 *   reset.
645 *   Returns
646 *     0 - success
647 *     EINVAL - A correct method or method context couldn't be retrieved.
648 *     EIO - Contract kill failed.
649 *     EFAULT - Method couldn't be executed successfully.
650 *     ELOOP - Retry threshold exceeded.
651 *     ECANCELED - inst was deleted from the repository before method was run
652 *     ERANGE - Timeout retry threshold exceeded.
653 *     EAGAIN - Failed due to external cause, retry.
654 */
655int
656method_run(restarter_inst_t **instp, int type, int *exit_code)
657{
658	char *method;
659	int ret_status;
660	pid_t pid;
661	method_restart_t restart_on;
662	uint_t cte_mask;
663	uint8_t need_session;
664	scf_handle_t *h;
665	scf_snapshot_t *snap;
666	const char *mname;
667	mc_error_t *m_error;
668	struct method_context *mcp;
669	int result = 0, timeout_fired = 0;
670	int sig, r;
671	boolean_t transient;
672	uint64_t timeout;
673	uint8_t timeout_retry;
674	ctid_t ctid;
675	int ctfd = -1;
676	restarter_inst_t *inst = *instp;
677	int id = inst->ri_id;
678	int forkerr;
679
680	assert(MUTEX_HELD(&inst->ri_lock));
681	assert(instance_in_transition(inst));
682
683	if (inst->ri_mi_deleted)
684		return (ECANCELED);
685
686	*exit_code = SMF_EXIT_OK;
687
688	assert(0 <= type && type <= 2);
689	mname = method_names[type];
690
691	if (type == METHOD_START)
692		inst->ri_pre_online_hook();
693
694	h = scf_instance_handle(inst->ri_m_inst);
695
696	snap = scf_snapshot_create(h);
697	if (snap == NULL ||
698	    scf_instance_get_snapshot(inst->ri_m_inst, "running", snap) != 0) {
699		log_framework(LOG_DEBUG,
700		    "Could not get running snapshot for %s.  "
701		    "Using editing version to run method %s.\n",
702		    inst->ri_i.i_fmri, mname);
703		scf_snapshot_destroy(snap);
704		snap = NULL;
705	}
706
707	/*
708	 * After this point, we may be logging to the instance log.
709	 * Make sure we've noted where that log is as a property of
710	 * the instance.
711	 */
712	r = libscf_note_method_log(inst->ri_m_inst, st->st_log_prefix,
713	    inst->ri_logstem);
714	if (r != 0) {
715		log_framework(LOG_WARNING,
716		    "%s: couldn't note log location: %s\n",
717		    inst->ri_i.i_fmri, strerror(r));
718	}
719
720	if ((method = libscf_get_method(h, type, inst, snap, &restart_on,
721	    &cte_mask, &need_session, &timeout, &timeout_retry)) == NULL) {
722		if (errno == LIBSCF_PGROUP_ABSENT)  {
723			log_framework(LOG_DEBUG,
724			    "%s: instance has no method property group '%s'.\n",
725			    inst->ri_i.i_fmri, mname);
726			if (type == METHOD_REFRESH)
727				log_instance(inst, B_TRUE, "No '%s' method "
728				    "defined.  Treating as :true.", mname);
729			else
730				log_instance(inst, B_TRUE, "Method property "
731				    "group '%s' is not present.", mname);
732			scf_snapshot_destroy(snap);
733			return (0);
734		} else if (errno == LIBSCF_PROPERTY_ABSENT)  {
735			log_framework(LOG_DEBUG,
736			    "%s: instance has no '%s/exec' method property.\n",
737			    inst->ri_i.i_fmri, mname);
738			log_instance(inst, B_TRUE, "Method property '%s/exec "
739			    "is not present.", mname);
740			scf_snapshot_destroy(snap);
741			return (0);
742		} else {
743			log_error(LOG_WARNING,
744			    "%s: instance libscf_get_method failed\n",
745			    inst->ri_i.i_fmri);
746			scf_snapshot_destroy(snap);
747			return (EINVAL);
748		}
749	}
750
751	/* open service contract if stopping a non-transient service */
752	if (type == METHOD_STOP && (!instance_is_transient_style(inst))) {
753		if (inst->ri_i.i_primary_ctid == 0) {
754			/* service is not running, nothing to stop */
755			log_framework(LOG_DEBUG, "%s: instance has no primary "
756			    "contract, no service to stop.\n",
757			    inst->ri_i.i_fmri);
758			scf_snapshot_destroy(snap);
759			return (0);
760		}
761		if ((ctfd = contract_open(inst->ri_i.i_primary_ctid, "process",
762		    "events", O_RDONLY)) < 0) {
763			result = EFAULT;
764			log_instance(inst, B_TRUE, "Could not open service "
765			    "contract %ld.  Stop method not run.",
766			    inst->ri_i.i_primary_ctid);
767			goto out;
768		}
769	}
770
771	if (restarter_is_null_method(method)) {
772		log_framework(LOG_DEBUG, "%s: null method succeeds\n",
773		    inst->ri_i.i_fmri);
774
775		log_instance(inst, B_TRUE, "Executing %s method (null).",
776		    mname);
777
778		if (type == METHOD_START)
779			write_status(inst, mname, 0);
780		goto out;
781	}
782
783	sig = restarter_is_kill_method(method);
784	if (sig >= 0) {
785
786		if (inst->ri_i.i_primary_ctid == 0) {
787			log_error(LOG_ERR, "%s: :kill with no contract\n",
788			    inst->ri_i.i_fmri);
789			log_instance(inst, B_TRUE, "Invalid use of \":kill\" "
790			    "as stop method for transient service.");
791			result = EINVAL;
792			goto out;
793		}
794
795		log_framework(LOG_DEBUG,
796		    "%s: :killing contract with signal %d\n",
797		    inst->ri_i.i_fmri, sig);
798
799		log_instance(inst, B_TRUE, "Executing %s method (:kill).",
800		    mname);
801
802		if (contract_kill(inst->ri_i.i_primary_ctid, sig,
803		    inst->ri_i.i_fmri) != 0) {
804			result = EIO;
805			goto out;
806		} else
807			goto assured_kill;
808	}
809
810	log_framework(LOG_DEBUG, "%s: forking to run method %s\n",
811	    inst->ri_i.i_fmri, method);
812
813	m_error = restarter_get_method_context(RESTARTER_METHOD_CONTEXT_VERSION,
814	    inst->ri_m_inst, snap, mname, method, &mcp);
815
816	if (m_error != NULL) {
817		log_instance(inst, B_TRUE, "%s", m_error->msg);
818		restarter_mc_error_destroy(m_error);
819		result = EINVAL;
820		goto out;
821	}
822
823	r = method_ready_contract(inst, type, restart_on, cte_mask);
824	if (r != 0) {
825		assert(r == ECANCELED);
826		assert(inst->ri_mi_deleted);
827		restarter_free_method_context(mcp);
828		result = ECANCELED;
829		goto out;
830	}
831
832	/*
833	 * Validate safety of method contexts, to save children work.
834	 */
835	if (!restarter_rm_libs_loadable())
836		log_framework(LOG_DEBUG, "%s: method contexts limited "
837		    "to root-accessible libraries\n", inst->ri_i.i_fmri);
838
839	/*
840	 * For wait-style svc, sanity check that method exists to prevent an
841	 * infinite loop.
842	 */
843	if (instance_is_wait_style(inst) && type == METHOD_START) {
844		char *pend;
845		struct stat64 sbuf;
846
847		/*
848		 * We need to handle start method strings that have arguments,
849		 * such as '/lib/svc/method/console-login %i'.
850		 */
851		if ((pend = strchr(method, ' ')) != NULL)
852			*pend = '\0';
853
854		if (*method == '/' && stat64(method, &sbuf) == -1 &&
855		    errno == ENOENT) {
856			log_instance(inst, B_TRUE, "Missing start method (%s), "
857			    "changing state to maintenance.", method);
858			restarter_free_method_context(mcp);
859			result = ENOENT;
860			goto out;
861		}
862		if (pend != NULL)
863			*pend = ' ';
864	}
865
866	/*
867	 * If the service is restarting too quickly, send it to
868	 * maintenance.
869	 */
870	if (type == METHOD_START) {
871		method_record_start(inst);
872		if (method_rate_critical(inst) &&
873		    !instance_is_wait_style(inst)) {
874			log_instance(inst, B_TRUE, "Restarting too quickly, "
875			    "changing state to maintenance.");
876			result = ELOOP;
877			restarter_free_method_context(mcp);
878			goto out;
879		}
880	}
881
882	atomic_add_16(&storing_contract, 1);
883	pid = startd_fork1(&forkerr);
884	if (pid == 0)
885		exec_method(inst, type, method, mcp, need_session);
886
887	if (pid == -1) {
888		atomic_add_16(&storing_contract, -1);
889		if (forkerr == EAGAIN)
890			result = EAGAIN;
891		else
892			result = EFAULT;
893
894		log_error(LOG_WARNING,
895		    "%s: Couldn't fork to execute method %s: %s\n",
896		    inst->ri_i.i_fmri, method, strerror(forkerr));
897
898		restarter_free_method_context(mcp);
899		goto out;
900	}
901
902
903	/*
904	 * Get the contract id, decide whether it is primary or transient, and
905	 * stash it in inst & the repository.
906	 */
907	method_store_contract(inst, type, &ctid);
908	atomic_add_16(&storing_contract, -1);
909
910	restarter_free_method_context(mcp);
911
912	/*
913	 * Similarly for the start method PID.
914	 */
915	if (type == METHOD_START && !inst->ri_mi_deleted)
916		(void) libscf_write_start_pid(inst->ri_m_inst, pid);
917
918	if (instance_is_wait_style(inst) && type == METHOD_START) {
919		/* Wait style instances don't get timeouts on start methods. */
920		if (wait_register(pid, inst->ri_i.i_fmri, 1, 0)) {
921			log_error(LOG_WARNING,
922			    "%s: couldn't register %ld for wait\n",
923			    inst->ri_i.i_fmri, pid);
924			result = EFAULT;
925			goto contract_out;
926		}
927		write_status(inst, mname, 0);
928
929	} else {
930		int r, err;
931		time_t start_time;
932		time_t end_time;
933
934		/*
935		 * Because on upgrade/live-upgrade we may have no chance
936		 * to override faulty timeout values on the way to
937		 * manifest import, all services on the path to manifest
938		 * import are treated the same as INFINITE timeout services.
939		 */
940
941		start_time = time(NULL);
942		if (timeout != METHOD_TIMEOUT_INFINITE && !is_timeout_ovr(inst))
943			timeout_insert(inst, ctid, timeout);
944		else
945			timeout = METHOD_TIMEOUT_INFINITE;
946
947		/* Unlock the instance while waiting for the method. */
948		MUTEX_UNLOCK(&inst->ri_lock);
949
950		do {
951			r = waitpid(pid, &ret_status, 0);
952		} while (r == -1 && errno == EINTR);
953		if (r == -1)
954			err = errno;
955
956		/* Re-grab the lock. */
957		inst = inst_lookup_by_id(id);
958
959		/*
960		 * inst can't be removed, as the removal thread waits
961		 * for completion of this one.
962		 */
963		assert(inst != NULL);
964		*instp = inst;
965
966		if (inst->ri_timeout != NULL && inst->ri_timeout->te_fired)
967			timeout_fired = 1;
968
969		timeout_remove(inst, ctid);
970
971		log_framework(LOG_DEBUG,
972		    "%s method for %s exited with status %d.\n", mname,
973		    inst->ri_i.i_fmri, WEXITSTATUS(ret_status));
974
975		if (r == -1) {
976			log_error(LOG_WARNING,
977			    "Couldn't waitpid() for %s method of %s (%s).\n",
978			    mname, inst->ri_i.i_fmri, strerror(err));
979			result = EFAULT;
980			goto contract_out;
981		}
982
983		if (type == METHOD_START)
984			write_status(inst, mname, ret_status);
985
986		/* return ERANGE if this service doesn't retry on timeout */
987		if (timeout_fired == 1 && timeout_retry == 0) {
988			result = ERANGE;
989			goto contract_out;
990		}
991
992		if (!WIFEXITED(ret_status) &&
993		    WEXITSTATUS(ret_status) != SMF_EXIT_NODAEMON) {
994			/*
995			 * If method didn't exit itself (it was killed by an
996			 * external entity, etc.), consider the entire
997			 * method_run as failed.
998			 */
999			if (WIFSIGNALED(ret_status)) {
1000				char buf[SIG2STR_MAX];
1001				(void) sig2str(WTERMSIG(ret_status), buf);
1002
1003				log_error(LOG_WARNING, "%s: Method \"%s\" "
1004				    "failed due to signal %s.\n",
1005				    inst->ri_i.i_fmri, method, buf);
1006				log_instance(inst, B_TRUE, "Method \"%s\" "
1007				    "failed due to signal %s.", mname, buf);
1008			} else {
1009				log_error(LOG_WARNING, "%s: Method \"%s\" "
1010				    "failed with exit status %d.\n",
1011				    inst->ri_i.i_fmri, method,
1012				    WEXITSTATUS(ret_status));
1013				log_instance(inst, B_TRUE, "Method \"%s\" "
1014				    "failed with exit status %d.", mname,
1015				    WEXITSTATUS(ret_status));
1016			}
1017			result = EAGAIN;
1018			goto contract_out;
1019		}
1020
1021		*exit_code = WEXITSTATUS(ret_status);
1022		if (*exit_code != SMF_EXIT_OK &&
1023		    *exit_code != SMF_EXIT_NODAEMON) {
1024			log_error(LOG_WARNING,
1025			    "%s: Method \"%s\" failed with exit status %d.\n",
1026			    inst->ri_i.i_fmri, method, WEXITSTATUS(ret_status));
1027		}
1028
1029		log_instance(inst, B_TRUE, "Method \"%s\" exited with status "
1030		    "%d.", mname, *exit_code);
1031
1032		/* Note: we will take this path for SMF_EXIT_NODAEMON */
1033		if (*exit_code != SMF_EXIT_OK)
1034			goto contract_out;
1035
1036		end_time = time(NULL);
1037
1038		/* Give service contract remaining seconds to empty */
1039		if (timeout != METHOD_TIMEOUT_INFINITE)
1040			timeout -= (end_time - start_time);
1041	}
1042
1043assured_kill:
1044	/*
1045	 * For stop methods, assure that the service contract has emptied
1046	 * before returning.
1047	 */
1048	if (type == METHOD_STOP && (!instance_is_transient_style(inst)) &&
1049	    !(contract_is_empty(inst->ri_i.i_primary_ctid))) {
1050		int times = 0;
1051
1052		if (timeout != METHOD_TIMEOUT_INFINITE)
1053			timeout_insert(inst, inst->ri_i.i_primary_ctid,
1054			    timeout);
1055
1056		for (;;) {
1057			/*
1058			 * Check frequently at first, then back off.  This
1059			 * keeps startd from idling while shutting down.
1060			 */
1061			if (times < 20) {
1062				(void) poll(NULL, 0, 5);
1063				times++;
1064			} else {
1065				(void) poll(NULL, 0, 100);
1066			}
1067			if (contract_is_empty(inst->ri_i.i_primary_ctid))
1068				break;
1069		}
1070
1071		if (timeout != METHOD_TIMEOUT_INFINITE)
1072			if (inst->ri_timeout->te_fired)
1073				result = EFAULT;
1074
1075		timeout_remove(inst, inst->ri_i.i_primary_ctid);
1076	}
1077
1078contract_out:
1079	/*
1080	 * Abandon contracts for transient methods, methods that exit with
1081	 * SMF_EXIT_NODAEMON & methods that fail.
1082	 */
1083	transient = method_is_transient(inst, type);
1084	if ((transient || *exit_code != SMF_EXIT_OK || result != 0) &&
1085	    (restarter_is_kill_method(method) < 0))
1086		method_remove_contract(inst, !transient, B_TRUE);
1087
1088out:
1089	if (ctfd >= 0)
1090		(void) close(ctfd);
1091	scf_snapshot_destroy(snap);
1092	free(method);
1093	return (result);
1094}
1095
1096/*
1097 * The method thread executes a service method to effect a state transition.
1098 * The next_state of info->sf_id should be non-_NONE on entrance, and it will
1099 * be _NONE on exit (state will either be what next_state was (on success), or
1100 * it will be _MAINT (on error)).
1101 *
1102 * There are six classes of methods to consider: start & other (stop, refresh)
1103 * for each of "normal" services, wait services, and transient services.  For
1104 * each, the method must be fetched from the repository & executed.  fork()ed
1105 * methods must be waited on, except for the start method of wait services
1106 * (which must be registered with the wait subsystem via wait_register()).  If
1107 * the method succeeded (returned 0), then for start methods its contract
1108 * should be recorded as the primary contract for the service.  For other
1109 * methods, it should be abandoned.  If the method fails, then depending on
1110 * the failure, either the method should be reexecuted or the service should
1111 * be put into maintenance.  Either way the contract should be abandoned.
1112 */
1113void *
1114method_thread(void *arg)
1115{
1116	fork_info_t *info = arg;
1117	restarter_inst_t *inst;
1118	scf_handle_t	*local_handle;
1119	scf_instance_t	*s_inst = NULL;
1120	int r, exit_code;
1121	boolean_t retryable;
1122	restarter_str_t reason;
1123
1124	(void) pthread_setname_np(pthread_self(), "method");
1125
1126	assert(0 <= info->sf_method_type && info->sf_method_type <= 2);
1127
1128	/* Get (and lock) the restarter_inst_t. */
1129	inst = inst_lookup_by_id(info->sf_id);
1130
1131	assert(inst->ri_method_thread != 0);
1132	assert(instance_in_transition(inst) == 1);
1133
1134	/*
1135	 * We cannot leave this function with inst in transition, because
1136	 * protocol.c withholds messages for inst otherwise.
1137	 */
1138
1139	log_framework(LOG_DEBUG, "method_thread() running %s method for %s.\n",
1140	    method_names[info->sf_method_type], inst->ri_i.i_fmri);
1141
1142	local_handle = libscf_handle_create_bound_loop();
1143
1144rebind_retry:
1145	/* get scf_instance_t */
1146	switch (r = libscf_fmri_get_instance(local_handle, inst->ri_i.i_fmri,
1147	    &s_inst)) {
1148	case 0:
1149		break;
1150
1151	case ECONNABORTED:
1152		libscf_handle_rebind(local_handle);
1153		goto rebind_retry;
1154
1155	case ENOENT:
1156		/*
1157		 * It's not there, but we need to call this so protocol.c
1158		 * doesn't think it's in transition anymore.
1159		 */
1160		(void) restarter_instance_update_states(local_handle, inst,
1161		    inst->ri_i.i_state, RESTARTER_STATE_NONE, RERR_NONE,
1162		    restarter_str_none);
1163		goto out;
1164
1165	case EINVAL:
1166	case ENOTSUP:
1167	default:
1168		bad_error("libscf_fmri_get_instance", r);
1169	}
1170
1171	inst->ri_m_inst = s_inst;
1172	inst->ri_mi_deleted = B_FALSE;
1173
1174retry:
1175	if (info->sf_method_type == METHOD_START)
1176		log_transition(inst, START_REQUESTED);
1177
1178	r = method_run(&inst, info->sf_method_type, &exit_code);
1179
1180	if (r == 0 &&
1181	    (exit_code == SMF_EXIT_OK || exit_code == SMF_EXIT_NODAEMON)) {
1182		/* Success! */
1183		assert(inst->ri_i.i_next_state != RESTARTER_STATE_NONE);
1184
1185		/*
1186		 * When a stop method succeeds, remove the primary contract of
1187		 * the service, unless we're going to offline, in which case
1188		 * retain the contract so we can transfer inherited contracts to
1189		 * the replacement service.
1190		 */
1191
1192		if (info->sf_method_type == METHOD_STOP &&
1193		    inst->ri_i.i_primary_ctid != 0) {
1194			if (inst->ri_i.i_next_state == RESTARTER_STATE_OFFLINE)
1195				inst->ri_i.i_primary_ctid_stopped = 1;
1196			else
1197				method_remove_contract(inst, B_TRUE, B_TRUE);
1198		}
1199
1200		/*
1201		 * For methods that exit with SMF_EXIT_NODAEMON, we already
1202		 * called method_remove_contract in method_run.
1203		 */
1204
1205		/*
1206		 * We don't care whether the handle was rebound because this is
1207		 * the last thing we do with it.
1208		 */
1209		(void) restarter_instance_update_states(local_handle, inst,
1210		    inst->ri_i.i_next_state, RESTARTER_STATE_NONE,
1211		    info->sf_event_type, info->sf_reason);
1212
1213		(void) update_fault_count(inst, FAULT_COUNT_RESET);
1214
1215		goto out;
1216	}
1217
1218	/* Failure.  Retry or go to maintenance. */
1219
1220	if (r != 0 && r != EAGAIN) {
1221		retryable = B_FALSE;
1222	} else {
1223		switch (exit_code) {
1224		case SMF_EXIT_ERR_CONFIG:
1225		case SMF_EXIT_ERR_NOSMF:
1226		case SMF_EXIT_ERR_PERM:
1227		case SMF_EXIT_ERR_FATAL:
1228			retryable = B_FALSE;
1229			break;
1230
1231		default:
1232			retryable = B_TRUE;
1233		}
1234	}
1235
1236	if (retryable && update_fault_count(inst, FAULT_COUNT_INCR) != 1)
1237		goto retry;
1238
1239	/* maintenance */
1240	if (r == ELOOP)
1241		log_transition(inst, START_FAILED_REPEATEDLY);
1242	else if (r == ERANGE)
1243		log_transition(inst, START_FAILED_TIMEOUT_FATAL);
1244	else if (exit_code == SMF_EXIT_ERR_CONFIG)
1245		log_transition(inst, START_FAILED_CONFIGURATION);
1246	else if (exit_code == SMF_EXIT_ERR_FATAL)
1247		log_transition(inst, START_FAILED_FATAL);
1248	else
1249		log_transition(inst, START_FAILED_OTHER);
1250
1251	if (r == ELOOP) {
1252		reason = restarter_str_restarting_too_quickly;
1253	} else if (retryable) {
1254		reason = restarter_str_fault_threshold_reached;
1255	} else {
1256		reason = restarter_str_method_failed;
1257	}
1258
1259	(void) restarter_instance_update_states(local_handle, inst,
1260	    RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE, RERR_FAULT,
1261	    reason);
1262
1263	if (!method_is_transient(inst, info->sf_method_type) &&
1264	    inst->ri_i.i_primary_ctid != 0)
1265		method_remove_contract(inst, B_TRUE, B_TRUE);
1266
1267out:
1268	inst->ri_method_thread = 0;
1269
1270	/*
1271	 * Unlock the mutex after broadcasting to avoid a race condition
1272	 * with restarter_delete_inst() when the 'inst' structure is freed.
1273	 */
1274	(void) pthread_cond_broadcast(&inst->ri_method_cv);
1275	MUTEX_UNLOCK(&inst->ri_lock);
1276
1277	scf_instance_destroy(s_inst);
1278	scf_handle_destroy(local_handle);
1279	startd_free(info, sizeof (fork_info_t));
1280	return (NULL);
1281}
1282