1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2019 Joyent, Inc.
25 * Copyright (c) 2015, Syneto S.R.L. All rights reserved.
26 * Copyright 2016 Toomas Soome <tsoome@me.com>
27 * Copyright 2016 RackTop Systems.
28 */
29
30/*
31 * graph.c - master restarter graph engine
32 *
33 *   The graph engine keeps a dependency graph of all service instances on the
34 *   system, as recorded in the repository.  It decides when services should
35 *   be brought up or down based on service states and dependencies and sends
36 *   commands to restarters to effect any changes.  It also executes
37 *   administrator commands sent by svcadm via the repository.
38 *
39 *   The graph is stored in uu_list_t *dgraph and its vertices are
40 *   graph_vertex_t's, each of which has a name and an integer id unique to
41 *   its name (see dict.c).  A vertex's type attribute designates the type
42 *   of object it represents: GVT_INST for service instances, GVT_SVC for
43 *   service objects (since service instances may depend on another service,
44 *   rather than service instance), GVT_FILE for files (which services may
45 *   depend on), and GVT_GROUP for dependencies on multiple objects.  GVT_GROUP
46 *   vertices are necessary because dependency lists may have particular
47 *   grouping types (require any, require all, optional, or exclude) and
48 *   event-propagation characteristics.
49 *
50 *   The initial graph is built by libscf_populate_graph() invoking
51 *   dgraph_add_instance() for each instance in the repository.  The function
52 *   adds a GVT_SVC vertex for the service if one does not already exist, adds
53 *   a GVT_INST vertex named by the FMRI of the instance, and sets up the edges.
54 *   The resulting web of vertices & edges associated with an instance's vertex
55 *   includes
56 *
57 *     - an edge from the GVT_SVC vertex for the instance's service
58 *
59 *     - an edge to the GVT_INST vertex of the instance's resarter, if its
60 *       restarter is not svc.startd
61 *
62 *     - edges from other GVT_INST vertices if the instance is a restarter
63 *
64 *     - for each dependency property group in the instance's "running"
65 *       snapshot, an edge to a GVT_GROUP vertex named by the FMRI of the
66 *       instance and the name of the property group
67 *
68 *     - for each value of the "entities" property in each dependency property
69 *       group, an edge from the corresponding GVT_GROUP vertex to a
70 *       GVT_INST, GVT_SVC, or GVT_FILE vertex
71 *
72 *     - edges from GVT_GROUP vertices for each dependent instance
73 *
74 *   After the edges are set up the vertex's GV_CONFIGURED flag is set.  If
75 *   there are problems, or if a service is mentioned in a dependency but does
76 *   not exist in the repository, the GV_CONFIGURED flag will be clear.
77 *
78 *   The graph and all of its vertices are protected by the dgraph_lock mutex.
79 *   See restarter.c for more information.
80 *
81 *   The properties of an instance fall into two classes: immediate and
82 *   snapshotted.  Immediate properties should have an immediate effect when
83 *   changed.  Snapshotted properties should be read from a snapshot, so they
84 *   only change when the snapshot changes.  The immediate properties used by
85 *   the graph engine are general/enabled, general/restarter, and the properties
86 *   in the restarter_actions property group.  Since they are immediate, they
87 *   are not read out of a snapshot.  The snapshotted properties used by the
88 *   graph engine are those in the property groups with type "dependency" and
89 *   are read out of the "running" snapshot.  The "running" snapshot is created
90 *   by the the graph engine as soon as possible, and it is updated, along with
91 *   in-core copies of the data (dependency information for the graph engine) on
92 *   receipt of the refresh command from svcadm.  In addition, the graph engine
93 *   updates the "start" snapshot from the "running" snapshot whenever a service
94 *   comes online.
95 *
96 *   When a DISABLE event is requested by the administrator, svc.startd shutdown
97 *   the dependents first before shutting down the requested service.
98 *   In graph_enable_by_vertex, we create a subtree that contains the dependent
99 *   vertices by marking those vertices with the GV_TOOFFLINE flag. And we mark
100 *   the vertex to disable with the GV_TODISABLE flag. Once the tree is created,
101 *   we send the _ADMIN_DISABLE event to the leaves. The leaves will then
102 *   transition from STATE_ONLINE/STATE_DEGRADED to STATE_OFFLINE/STATE_MAINT.
103 *   In gt_enter_offline and gt_enter_maint if the vertex was in a subtree then
104 *   we clear the GV_TOOFFLINE flag and walk the dependencies to offline the new
105 *   exposed leaves. We do the same until we reach the last leaf (the one with
106 *   the GV_TODISABLE flag). If the vertex to disable is also part of a larger
107 *   subtree (eg. multiple DISABLE events on vertices in the same subtree) then
108 *   once the first vertex is disabled (GV_TODISABLE flag is removed), we
109 *   continue to propagate the offline event to the vertex's dependencies.
110 *
111 *
112 * SMF state transition notifications
113 *
114 *   When an instance of a service managed by SMF changes state, svc.startd may
115 *   publish a GPEC sysevent. All transitions to or from maintenance, a
116 *   transition cause by a hardware error will generate an event.
117 *   Other transitions will generate an event if there exist notification
118 *   parameter for that transition. Notification parameters are stored in the
119 *   SMF repository for the service/instance they refer to. System-wide
120 *   notification parameters are stored in the global instance.
121 *   svc.startd can be told to send events for all SMF state transitions despite
122 *   of notification parameters by setting options/info_events_all to true in
123 *   restarter:default
124 *
125 *   The set of transitions that generate events is cached in the
126 *   dgraph_vertex_t gv_stn_tset for service/instance and in the global
127 *   stn_global for the system-wide set. They are re-read when instances are
128 *   refreshed.
129 *
130 *   The GPEC events published by svc.startd are consumed by fmd(1M). After
131 *   processing these events, fmd(1M) publishes the processed events to
132 *   notification agents. The notification agents read the notification
133 *   parameters from the SMF repository through libscf(3LIB) interfaces and send
134 *   the notification, or not, based on those parameters.
135 *
136 *   Subscription and publishing to the GPEC channels is done with the
137 *   libfmevent(3LIB) wrappers fmev_[r]publish_*() and
138 *   fmev_shdl_(un)subscribe().
139 *
140 */
141
142#include <sys/uadmin.h>
143#include <sys/wait.h>
144
145#include <assert.h>
146#include <errno.h>
147#include <fcntl.h>
148#include <fm/libfmevent.h>
149#include <libscf.h>
150#include <libscf_priv.h>
151#include <librestart.h>
152#include <libuutil.h>
153#include <locale.h>
154#include <poll.h>
155#include <pthread.h>
156#include <signal.h>
157#include <stddef.h>
158#include <stdio.h>
159#include <stdlib.h>
160#include <string.h>
161#include <strings.h>
162#include <sys/statvfs.h>
163#include <sys/uadmin.h>
164#include <zone.h>
165#if defined(__x86)
166#include <libbe.h>
167#endif	/* __x86 */
168
169#include "startd.h"
170#include "protocol.h"
171
172
173#define	MILESTONE_NONE	((graph_vertex_t *)1)
174
175#define	CONSOLE_LOGIN_FMRI	"svc:/system/console-login:default"
176#define	FS_MINIMAL_FMRI		"svc:/system/filesystem/minimal:default"
177
178#define	VERTEX_REMOVED	0	/* vertex has been freed  */
179#define	VERTEX_INUSE	1	/* vertex is still in use */
180
181#define	IS_ENABLED(v) ((v)->gv_flags & (GV_ENABLED | GV_ENBLD_NOOVR))
182
183/*
184 * stn_global holds the tset for the system wide notification parameters.
185 * It is updated on refresh of svc:/system/svc/global:default
186 *
187 * There are two assumptions that relax the need for a mutex:
188 *     1. 32-bit value assignments are atomic
189 *     2. Its value is consumed only in one point at
190 *     dgraph_state_transition_notify(). There are no test and set races.
191 *
192 *     If either assumption is broken, we'll need a mutex to synchronize
193 *     access to stn_global
194 */
195int32_t stn_global;
196/*
197 * info_events_all holds a flag to override notification parameters and send
198 * Information events for all state transitions.
199 * same about the need of a mutex here.
200 */
201int info_events_all;
202
203/*
204 * Services in these states are not considered 'down' by the
205 * milestone/shutdown code.
206 */
207#define	up_state(state)	((state) == RESTARTER_STATE_ONLINE || \
208	(state) == RESTARTER_STATE_DEGRADED || \
209	(state) == RESTARTER_STATE_OFFLINE)
210
211#define	is_depgrp_bypassed(v) ((v->gv_type == GVT_GROUP) && \
212	((v->gv_depgroup == DEPGRP_EXCLUDE_ALL) || \
213	(v->gv_restart < RERR_RESTART)))
214
215#define	is_inst_bypassed(v) ((v->gv_type == GVT_INST) && \
216	((v->gv_flags & GV_TODISABLE) || \
217	(v->gv_flags & GV_TOOFFLINE)))
218
219static uu_list_pool_t *graph_edge_pool, *graph_vertex_pool;
220static uu_list_t *dgraph;
221static pthread_mutex_t dgraph_lock;
222
223/*
224 * milestone indicates the current subgraph.  When NULL, it is the entire
225 * graph.  When MILESTONE_NONE, it is the empty graph.  Otherwise, it is all
226 * services on which the target vertex depends.
227 */
228static graph_vertex_t *milestone = NULL;
229static boolean_t initial_milestone_set = B_FALSE;
230static pthread_cond_t initial_milestone_cv = PTHREAD_COND_INITIALIZER;
231
232/* protected by dgraph_lock */
233static boolean_t sulogin_thread_running = B_FALSE;
234static boolean_t sulogin_running = B_FALSE;
235static boolean_t console_login_ready = B_FALSE;
236
237/* Number of services to come down to complete milestone transition. */
238static uint_t non_subgraph_svcs;
239
240/*
241 * These variables indicate what should be done when we reach the milestone
242 * target milestone, i.e., when non_subgraph_svcs == 0.  They are acted upon in
243 * dgraph_set_instance_state().
244 */
245static int halting = -1;
246static boolean_t go_single_user_mode = B_FALSE;
247static boolean_t go_to_level1 = B_FALSE;
248
249/*
250 * Tracks when we started halting.
251 */
252static time_t halting_time = 0;
253
254/*
255 * This tracks the legacy runlevel to ensure we signal init and manage
256 * utmpx entries correctly.
257 */
258static char current_runlevel = '\0';
259
260/* Number of single user threads currently running */
261static pthread_mutex_t single_user_thread_lock;
262static int single_user_thread_count = 0;
263
264/* Statistics for dependency cycle-checking */
265static u_longlong_t dep_inserts = 0;
266static u_longlong_t dep_cycle_ns = 0;
267static u_longlong_t dep_insert_ns = 0;
268
269
270static const char * const emsg_invalid_restarter =
271	"Transitioning %s to maintenance, restarter FMRI %s is invalid "
272	"(see 'svcs -xv' for details).\n";
273static const char * const console_login_fmri = CONSOLE_LOGIN_FMRI;
274static const char * const single_user_fmri = SCF_MILESTONE_SINGLE_USER;
275static const char * const multi_user_fmri = SCF_MILESTONE_MULTI_USER;
276static const char * const multi_user_svr_fmri = SCF_MILESTONE_MULTI_USER_SERVER;
277
278
279/*
280 * These services define the system being "up".  If none of them can come
281 * online, then we will run sulogin on the console.  Note that the install ones
282 * are for the miniroot and when installing CDs after the first.  can_come_up()
283 * does the decision making, and an sulogin_thread() runs sulogin, which can be
284 * started by dgraph_set_instance_state() or single_user_thread().
285 *
286 * NOTE: can_come_up() relies on SCF_MILESTONE_SINGLE_USER being the first
287 * entry, which is only used when booting_to_single_user (boot -s) is set.
288 * This is because when doing a "boot -s", sulogin is started from specials.c
289 * after milestone/single-user comes online, for backwards compatibility.
290 * In this case, SCF_MILESTONE_SINGLE_USER needs to be part of up_svcs
291 * to ensure sulogin will be spawned if milestone/single-user cannot be reached.
292 */
293static const char * const up_svcs[] = {
294	SCF_MILESTONE_SINGLE_USER,
295	CONSOLE_LOGIN_FMRI,
296	"svc:/system/install-setup:default",
297	"svc:/system/install:default",
298	NULL
299};
300
301/* This array must have an element for each non-NULL element of up_svcs[]. */
302static graph_vertex_t *up_svcs_p[] = { NULL, NULL, NULL, NULL };
303
304/* These are for seed repository magic.  See can_come_up(). */
305static const char * const manifest_import = SCF_INSTANCE_MI;
306static graph_vertex_t *manifest_import_p = NULL;
307
308
309static char target_milestone_as_runlevel(void);
310static void graph_runlevel_changed(char rl, int online);
311static int dgraph_set_milestone(const char *, scf_handle_t *, boolean_t);
312static boolean_t should_be_in_subgraph(graph_vertex_t *v);
313static int mark_subtree(graph_edge_t *, void *);
314static boolean_t insubtree_dependents_down(graph_vertex_t *);
315
316/*
317 * graph_vertex_compare()
318 *	This function can compare either int *id or * graph_vertex_t *gv
319 *	values, as the vertex id is always the first element of a
320 *	graph_vertex structure.
321 */
322/* ARGSUSED */
323static int
324graph_vertex_compare(const void *lc_arg, const void *rc_arg, void *private)
325{
326	int lc_id = ((const graph_vertex_t *)lc_arg)->gv_id;
327	int rc_id = *(int *)rc_arg;
328
329	if (lc_id > rc_id)
330		return (1);
331	if (lc_id < rc_id)
332		return (-1);
333	return (0);
334}
335
336void
337graph_init()
338{
339	graph_edge_pool = startd_list_pool_create("graph_edges",
340	    sizeof (graph_edge_t), offsetof(graph_edge_t, ge_link), NULL,
341	    UU_LIST_POOL_DEBUG);
342	assert(graph_edge_pool != NULL);
343
344	graph_vertex_pool = startd_list_pool_create("graph_vertices",
345	    sizeof (graph_vertex_t), offsetof(graph_vertex_t, gv_link),
346	    graph_vertex_compare, UU_LIST_POOL_DEBUG);
347	assert(graph_vertex_pool != NULL);
348
349	(void) pthread_mutex_init(&dgraph_lock, &mutex_attrs);
350	(void) pthread_mutex_init(&single_user_thread_lock, &mutex_attrs);
351	dgraph = startd_list_create(graph_vertex_pool, NULL, UU_LIST_SORTED);
352	assert(dgraph != NULL);
353
354	if (!st->st_initial)
355		current_runlevel = utmpx_get_runlevel();
356
357	log_framework(LOG_DEBUG, "Initialized graph\n");
358}
359
360static graph_vertex_t *
361vertex_get_by_name(const char *name)
362{
363	int id;
364
365	assert(MUTEX_HELD(&dgraph_lock));
366
367	id = dict_lookup_byname(name);
368	if (id == -1)
369		return (NULL);
370
371	return (uu_list_find(dgraph, &id, NULL, NULL));
372}
373
374static graph_vertex_t *
375vertex_get_by_id(int id)
376{
377	assert(MUTEX_HELD(&dgraph_lock));
378
379	if (id == -1)
380		return (NULL);
381
382	return (uu_list_find(dgraph, &id, NULL, NULL));
383}
384
385/*
386 * Creates a new vertex with the given name, adds it to the graph, and returns
387 * a pointer to it.  The graph lock must be held by this thread on entry.
388 */
389static graph_vertex_t *
390graph_add_vertex(const char *name)
391{
392	int id;
393	graph_vertex_t *v;
394	void *p;
395	uu_list_index_t idx;
396
397	assert(MUTEX_HELD(&dgraph_lock));
398
399	id = dict_insert(name);
400
401	v = startd_zalloc(sizeof (*v));
402
403	v->gv_id = id;
404
405	v->gv_name = startd_alloc(strlen(name) + 1);
406	(void) strcpy(v->gv_name, name);
407
408	v->gv_dependencies = startd_list_create(graph_edge_pool, v, 0);
409	v->gv_dependents = startd_list_create(graph_edge_pool, v, 0);
410
411	p = uu_list_find(dgraph, &id, NULL, &idx);
412	assert(p == NULL);
413
414	uu_list_node_init(v, &v->gv_link, graph_vertex_pool);
415	uu_list_insert(dgraph, v, idx);
416
417	return (v);
418}
419
420/*
421 * Removes v from the graph and frees it.  The graph should be locked by this
422 * thread, and v should have no edges associated with it.
423 */
424static void
425graph_remove_vertex(graph_vertex_t *v)
426{
427	assert(MUTEX_HELD(&dgraph_lock));
428
429	assert(uu_list_numnodes(v->gv_dependencies) == 0);
430	assert(uu_list_numnodes(v->gv_dependents) == 0);
431	assert(v->gv_refs == 0);
432
433	startd_free(v->gv_name, strlen(v->gv_name) + 1);
434	uu_list_destroy(v->gv_dependencies);
435	uu_list_destroy(v->gv_dependents);
436	uu_list_remove(dgraph, v);
437
438	startd_free(v, sizeof (graph_vertex_t));
439}
440
441static void
442graph_add_edge(graph_vertex_t *fv, graph_vertex_t *tv)
443{
444	graph_edge_t *e, *re;
445	int r;
446
447	assert(MUTEX_HELD(&dgraph_lock));
448
449	e = startd_alloc(sizeof (graph_edge_t));
450	re = startd_alloc(sizeof (graph_edge_t));
451
452	e->ge_parent = fv;
453	e->ge_vertex = tv;
454
455	re->ge_parent = tv;
456	re->ge_vertex = fv;
457
458	uu_list_node_init(e, &e->ge_link, graph_edge_pool);
459	r = uu_list_insert_before(fv->gv_dependencies, NULL, e);
460	assert(r == 0);
461
462	uu_list_node_init(re, &re->ge_link, graph_edge_pool);
463	r = uu_list_insert_before(tv->gv_dependents, NULL, re);
464	assert(r == 0);
465}
466
467static void
468graph_remove_edge(graph_vertex_t *v, graph_vertex_t *dv)
469{
470	graph_edge_t *e;
471
472	for (e = uu_list_first(v->gv_dependencies);
473	    e != NULL;
474	    e = uu_list_next(v->gv_dependencies, e)) {
475		if (e->ge_vertex == dv) {
476			uu_list_remove(v->gv_dependencies, e);
477			startd_free(e, sizeof (graph_edge_t));
478			break;
479		}
480	}
481
482	for (e = uu_list_first(dv->gv_dependents);
483	    e != NULL;
484	    e = uu_list_next(dv->gv_dependents, e)) {
485		if (e->ge_vertex == v) {
486			uu_list_remove(dv->gv_dependents, e);
487			startd_free(e, sizeof (graph_edge_t));
488			break;
489		}
490	}
491}
492
493static void
494remove_inst_vertex(graph_vertex_t *v)
495{
496	graph_edge_t *e;
497	graph_vertex_t *sv;
498	int i;
499
500	assert(MUTEX_HELD(&dgraph_lock));
501	assert(uu_list_numnodes(v->gv_dependents) == 1);
502	assert(uu_list_numnodes(v->gv_dependencies) == 0);
503	assert(v->gv_refs == 0);
504	assert((v->gv_flags & GV_CONFIGURED) == 0);
505
506	e = uu_list_first(v->gv_dependents);
507	sv = e->ge_vertex;
508	graph_remove_edge(sv, v);
509
510	for (i = 0; up_svcs[i] != NULL; ++i) {
511		if (up_svcs_p[i] == v)
512			up_svcs_p[i] = NULL;
513	}
514
515	if (manifest_import_p == v)
516		manifest_import_p = NULL;
517
518	graph_remove_vertex(v);
519
520	if (uu_list_numnodes(sv->gv_dependencies) == 0 &&
521	    uu_list_numnodes(sv->gv_dependents) == 0 &&
522	    sv->gv_refs == 0)
523		graph_remove_vertex(sv);
524}
525
526static void
527graph_walk_dependents(graph_vertex_t *v, void (*func)(graph_vertex_t *, void *),
528    void *arg)
529{
530	graph_edge_t *e;
531
532	for (e = uu_list_first(v->gv_dependents);
533	    e != NULL;
534	    e = uu_list_next(v->gv_dependents, e))
535		func(e->ge_vertex, arg);
536}
537
538static void
539graph_walk_dependencies(graph_vertex_t *v,
540    void (*func)(graph_vertex_t *, void *), void *arg)
541{
542	graph_edge_t *e;
543
544	assert(MUTEX_HELD(&dgraph_lock));
545
546	for (e = uu_list_first(v->gv_dependencies);
547	    e != NULL;
548	    e = uu_list_next(v->gv_dependencies, e)) {
549
550		func(e->ge_vertex, arg);
551	}
552}
553
554/*
555 * Generic graph walking function.
556 *
557 * Given a vertex, this function will walk either dependencies
558 * (WALK_DEPENDENCIES) or dependents (WALK_DEPENDENTS) of a vertex recursively
559 * for the entire graph.  It will avoid cycles and never visit the same vertex
560 * twice.
561 *
562 * We avoid traversing exclusion dependencies, because they are allowed to
563 * create cycles in the graph.  When propagating satisfiability, there is no
564 * need to walk exclusion dependencies because exclude_all_satisfied() doesn't
565 * test for satisfiability.
566 *
567 * The walker takes two callbacks.  The first is called before examining the
568 * dependents of each vertex.  The second is called on each vertex after
569 * examining its dependents.  This allows is_path_to() to construct a path only
570 * after the target vertex has been found.
571 */
572typedef enum {
573	WALK_DEPENDENTS,
574	WALK_DEPENDENCIES
575} graph_walk_dir_t;
576
577typedef int (*graph_walk_cb_t)(graph_vertex_t *, void *);
578
579typedef struct graph_walk_info {
580	graph_walk_dir_t	gi_dir;
581	uchar_t			*gi_visited;	/* vertex bitmap */
582	int			(*gi_pre)(graph_vertex_t *, void *);
583	void			(*gi_post)(graph_vertex_t *, void *);
584	void			*gi_arg;	/* callback arg */
585	int			gi_ret;		/* return value */
586} graph_walk_info_t;
587
588static int
589graph_walk_recurse(graph_edge_t *e, graph_walk_info_t *gip)
590{
591	uu_list_t *list;
592	int r;
593	graph_vertex_t *v = e->ge_vertex;
594	int i;
595	uint_t b;
596
597	i = v->gv_id / 8;
598	b = 1 << (v->gv_id % 8);
599
600	/*
601	 * Check to see if we've visited this vertex already.
602	 */
603	if (gip->gi_visited[i] & b)
604		return (UU_WALK_NEXT);
605
606	gip->gi_visited[i] |= b;
607
608	/*
609	 * Don't follow exclusions.
610	 */
611	if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
612		return (UU_WALK_NEXT);
613
614	/*
615	 * Call pre-visit callback.  If this doesn't terminate the walk,
616	 * continue search.
617	 */
618	if ((gip->gi_ret = gip->gi_pre(v, gip->gi_arg)) == UU_WALK_NEXT) {
619		/*
620		 * Recurse using appropriate list.
621		 */
622		if (gip->gi_dir == WALK_DEPENDENTS)
623			list = v->gv_dependents;
624		else
625			list = v->gv_dependencies;
626
627		r = uu_list_walk(list, (uu_walk_fn_t *)graph_walk_recurse,
628		    gip, 0);
629		assert(r == 0);
630	}
631
632	/*
633	 * Callbacks must return either UU_WALK_NEXT or UU_WALK_DONE.
634	 */
635	assert(gip->gi_ret == UU_WALK_NEXT || gip->gi_ret == UU_WALK_DONE);
636
637	/*
638	 * If given a post-callback, call the function for every vertex.
639	 */
640	if (gip->gi_post != NULL)
641		(void) gip->gi_post(v, gip->gi_arg);
642
643	/*
644	 * Preserve the callback's return value.  If the callback returns
645	 * UU_WALK_DONE, then we propagate that to the caller in order to
646	 * terminate the walk.
647	 */
648	return (gip->gi_ret);
649}
650
651static void
652graph_walk(graph_vertex_t *v, graph_walk_dir_t dir,
653    int (*pre)(graph_vertex_t *, void *),
654    void (*post)(graph_vertex_t *, void *), void *arg)
655{
656	graph_walk_info_t gi;
657	graph_edge_t fake;
658	size_t sz = dictionary->dict_new_id / 8 + 1;
659
660	gi.gi_visited = startd_zalloc(sz);
661	gi.gi_pre = pre;
662	gi.gi_post = post;
663	gi.gi_arg = arg;
664	gi.gi_dir = dir;
665	gi.gi_ret = 0;
666
667	/*
668	 * Fake up an edge for the first iteration
669	 */
670	fake.ge_vertex = v;
671	(void) graph_walk_recurse(&fake, &gi);
672
673	startd_free(gi.gi_visited, sz);
674}
675
676typedef struct child_search {
677	int	id;		/* id of vertex to look for */
678	uint_t	depth;		/* recursion depth */
679	/*
680	 * While the vertex is not found, path is NULL.  After the search, if
681	 * the vertex was found then path should point to a -1-terminated
682	 * array of vertex id's which constitute the path to the vertex.
683	 */
684	int	*path;
685} child_search_t;
686
687static int
688child_pre(graph_vertex_t *v, void *arg)
689{
690	child_search_t *cs = arg;
691
692	cs->depth++;
693
694	if (v->gv_id == cs->id) {
695		cs->path = startd_alloc((cs->depth + 1) * sizeof (int));
696		cs->path[cs->depth] = -1;
697		return (UU_WALK_DONE);
698	}
699
700	return (UU_WALK_NEXT);
701}
702
703static void
704child_post(graph_vertex_t *v, void *arg)
705{
706	child_search_t *cs = arg;
707
708	cs->depth--;
709
710	if (cs->path != NULL)
711		cs->path[cs->depth] = v->gv_id;
712}
713
714/*
715 * Look for a path from from to to.  If one exists, returns a pointer to
716 * a NULL-terminated array of pointers to the vertices along the path.  If
717 * there is no path, returns NULL.
718 */
719static int *
720is_path_to(graph_vertex_t *from, graph_vertex_t *to)
721{
722	child_search_t cs;
723
724	cs.id = to->gv_id;
725	cs.depth = 0;
726	cs.path = NULL;
727
728	graph_walk(from, WALK_DEPENDENCIES, child_pre, child_post, &cs);
729
730	return (cs.path);
731}
732
733/*
734 * Given an array of int's as returned by is_path_to, allocates a string of
735 * their names joined by newlines.  Returns the size of the allocated buffer
736 * in *sz and frees path.
737 */
738static void
739path_to_str(int *path, char **cpp, size_t *sz)
740{
741	int i;
742	graph_vertex_t *v;
743	size_t allocd, new_allocd;
744	char *new, *name;
745
746	assert(MUTEX_HELD(&dgraph_lock));
747	assert(path[0] != -1);
748
749	allocd = 1;
750	*cpp = startd_alloc(1);
751	(*cpp)[0] = '\0';
752
753	for (i = 0; path[i] != -1; ++i) {
754		name = NULL;
755
756		v = vertex_get_by_id(path[i]);
757
758		if (v == NULL)
759			name = "<deleted>";
760		else if (v->gv_type == GVT_INST || v->gv_type == GVT_SVC)
761			name = v->gv_name;
762
763		if (name != NULL) {
764			new_allocd = allocd + strlen(name) + 1;
765			new = startd_alloc(new_allocd);
766			(void) strcpy(new, *cpp);
767			(void) strcat(new, name);
768			(void) strcat(new, "\n");
769
770			startd_free(*cpp, allocd);
771
772			*cpp = new;
773			allocd = new_allocd;
774		}
775	}
776
777	startd_free(path, sizeof (int) * (i + 1));
778
779	*sz = allocd;
780}
781
782
783/*
784 * This function along with run_sulogin() implements an exclusion relationship
785 * between system/console-login and sulogin.  run_sulogin() will fail if
786 * system/console-login is online, and the graph engine should call
787 * graph_clogin_start() to bring system/console-login online, which defers the
788 * start if sulogin is running.
789 */
790static void
791graph_clogin_start(graph_vertex_t *v)
792{
793	assert(MUTEX_HELD(&dgraph_lock));
794
795	if (sulogin_running)
796		console_login_ready = B_TRUE;
797	else
798		vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
799}
800
801static void
802graph_su_start(graph_vertex_t *v)
803{
804	/*
805	 * /etc/inittab used to have the initial /sbin/rcS as a 'sysinit'
806	 * entry with a runlevel of 'S', before jumping to the final
807	 * target runlevel (as set in initdefault).  We mimic that legacy
808	 * behavior here.
809	 */
810	utmpx_set_runlevel('S', '0', B_FALSE);
811	vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
812}
813
814static void
815graph_post_su_online(void)
816{
817	graph_runlevel_changed('S', 1);
818}
819
820static void
821graph_post_su_disable(void)
822{
823	graph_runlevel_changed('S', 0);
824}
825
826static void
827graph_post_mu_online(void)
828{
829	graph_runlevel_changed('2', 1);
830}
831
832static void
833graph_post_mu_disable(void)
834{
835	graph_runlevel_changed('2', 0);
836}
837
838static void
839graph_post_mus_online(void)
840{
841	graph_runlevel_changed('3', 1);
842}
843
844static void
845graph_post_mus_disable(void)
846{
847	graph_runlevel_changed('3', 0);
848}
849
850static struct special_vertex_info {
851	const char	*name;
852	void		(*start_f)(graph_vertex_t *);
853	void		(*post_online_f)(void);
854	void		(*post_disable_f)(void);
855} special_vertices[] = {
856	{ CONSOLE_LOGIN_FMRI, graph_clogin_start, NULL, NULL },
857	{ SCF_MILESTONE_SINGLE_USER, graph_su_start,
858	    graph_post_su_online, graph_post_su_disable },
859	{ SCF_MILESTONE_MULTI_USER, NULL,
860	    graph_post_mu_online, graph_post_mu_disable },
861	{ SCF_MILESTONE_MULTI_USER_SERVER, NULL,
862	    graph_post_mus_online, graph_post_mus_disable },
863	{ NULL },
864};
865
866
867void
868vertex_send_event(graph_vertex_t *v, restarter_event_type_t e)
869{
870	switch (e) {
871	case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
872		assert(v->gv_state == RESTARTER_STATE_UNINIT);
873
874		MUTEX_LOCK(&st->st_load_lock);
875		st->st_load_instances++;
876		MUTEX_UNLOCK(&st->st_load_lock);
877		break;
878
879	case RESTARTER_EVENT_TYPE_ENABLE:
880		log_framework(LOG_DEBUG, "Enabling %s.\n", v->gv_name);
881		assert(v->gv_state == RESTARTER_STATE_UNINIT ||
882		    v->gv_state == RESTARTER_STATE_DISABLED ||
883		    v->gv_state == RESTARTER_STATE_MAINT);
884		break;
885
886	case RESTARTER_EVENT_TYPE_DISABLE:
887	case RESTARTER_EVENT_TYPE_ADMIN_DISABLE:
888		log_framework(LOG_DEBUG, "Disabling %s.\n", v->gv_name);
889		assert(v->gv_state != RESTARTER_STATE_DISABLED);
890		break;
891
892	case RESTARTER_EVENT_TYPE_STOP_RESET:
893	case RESTARTER_EVENT_TYPE_STOP:
894		log_framework(LOG_DEBUG, "Stopping %s.\n", v->gv_name);
895		assert(v->gv_state == RESTARTER_STATE_DEGRADED ||
896		    v->gv_state == RESTARTER_STATE_ONLINE);
897		break;
898
899	case RESTARTER_EVENT_TYPE_START:
900		log_framework(LOG_DEBUG, "Starting %s.\n", v->gv_name);
901		assert(v->gv_state == RESTARTER_STATE_OFFLINE);
902		break;
903
904	case RESTARTER_EVENT_TYPE_REMOVE_INSTANCE:
905	case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
906	case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
907	case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
908	case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
909	case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
910	case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
911	case RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE:
912	case RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY:
913		break;
914
915	default:
916#ifndef NDEBUG
917		uu_warn("%s:%d: Bad event %d.\n", __FILE__, __LINE__, e);
918#endif
919		abort();
920	}
921
922	restarter_protocol_send_event(v->gv_name, v->gv_restarter_channel, e,
923	    v->gv_reason);
924}
925
926static void
927graph_unset_restarter(graph_vertex_t *v)
928{
929	assert(MUTEX_HELD(&dgraph_lock));
930	assert(v->gv_flags & GV_CONFIGURED);
931
932	vertex_send_event(v, RESTARTER_EVENT_TYPE_REMOVE_INSTANCE);
933
934	if (v->gv_restarter_id != -1) {
935		graph_vertex_t *rv;
936
937		rv = vertex_get_by_id(v->gv_restarter_id);
938		graph_remove_edge(v, rv);
939	}
940
941	v->gv_restarter_id = -1;
942	v->gv_restarter_channel = NULL;
943}
944
945/*
946 * Return VERTEX_REMOVED when the vertex passed in argument is deleted from the
947 * dgraph otherwise return VERTEX_INUSE.
948 */
949static int
950free_if_unrefed(graph_vertex_t *v)
951{
952	assert(MUTEX_HELD(&dgraph_lock));
953
954	if (v->gv_refs > 0)
955		return (VERTEX_INUSE);
956
957	if (v->gv_type == GVT_SVC &&
958	    uu_list_numnodes(v->gv_dependents) == 0 &&
959	    uu_list_numnodes(v->gv_dependencies) == 0) {
960		graph_remove_vertex(v);
961		return (VERTEX_REMOVED);
962	} else if (v->gv_type == GVT_INST &&
963	    (v->gv_flags & GV_CONFIGURED) == 0 &&
964	    uu_list_numnodes(v->gv_dependents) == 1 &&
965	    uu_list_numnodes(v->gv_dependencies) == 0) {
966		remove_inst_vertex(v);
967		return (VERTEX_REMOVED);
968	}
969
970	return (VERTEX_INUSE);
971}
972
973static void
974delete_depgroup(graph_vertex_t *v)
975{
976	graph_edge_t *e;
977	graph_vertex_t *dv;
978
979	assert(MUTEX_HELD(&dgraph_lock));
980	assert(v->gv_type == GVT_GROUP);
981	assert(uu_list_numnodes(v->gv_dependents) == 0);
982
983	while ((e = uu_list_first(v->gv_dependencies)) != NULL) {
984		dv = e->ge_vertex;
985
986		graph_remove_edge(v, dv);
987
988		switch (dv->gv_type) {
989		case GVT_INST:		/* instance dependency */
990		case GVT_SVC:		/* service dependency */
991			(void) free_if_unrefed(dv);
992			break;
993
994		case GVT_FILE:		/* file dependency */
995			assert(uu_list_numnodes(dv->gv_dependencies) == 0);
996			if (uu_list_numnodes(dv->gv_dependents) == 0)
997				graph_remove_vertex(dv);
998			break;
999
1000		default:
1001#ifndef NDEBUG
1002			uu_warn("%s:%d: Unexpected node type %d", __FILE__,
1003			    __LINE__, dv->gv_type);
1004#endif
1005			abort();
1006		}
1007	}
1008
1009	graph_remove_vertex(v);
1010}
1011
1012static int
1013delete_instance_deps_cb(graph_edge_t *e, void **ptrs)
1014{
1015	graph_vertex_t *v = ptrs[0];
1016	boolean_t delete_restarter_dep = (boolean_t)ptrs[1];
1017	graph_vertex_t *dv;
1018
1019	dv = e->ge_vertex;
1020
1021	/*
1022	 * We have four possibilities here:
1023	 *   - GVT_INST: restarter
1024	 *   - GVT_GROUP - GVT_INST: instance dependency
1025	 *   - GVT_GROUP - GVT_SVC - GV_INST: service dependency
1026	 *   - GVT_GROUP - GVT_FILE: file dependency
1027	 */
1028	switch (dv->gv_type) {
1029	case GVT_INST:	/* restarter */
1030		assert(dv->gv_id == v->gv_restarter_id);
1031		if (delete_restarter_dep)
1032			graph_remove_edge(v, dv);
1033		break;
1034
1035	case GVT_GROUP:	/* pg dependency */
1036		graph_remove_edge(v, dv);
1037		delete_depgroup(dv);
1038		break;
1039
1040	case GVT_FILE:
1041		/* These are currently not direct dependencies */
1042
1043	default:
1044#ifndef NDEBUG
1045		uu_warn("%s:%d: Bad vertex type %d.\n", __FILE__, __LINE__,
1046		    dv->gv_type);
1047#endif
1048		abort();
1049	}
1050
1051	return (UU_WALK_NEXT);
1052}
1053
1054static void
1055delete_instance_dependencies(graph_vertex_t *v, boolean_t delete_restarter_dep)
1056{
1057	void *ptrs[2];
1058	int r;
1059
1060	assert(MUTEX_HELD(&dgraph_lock));
1061	assert(v->gv_type == GVT_INST);
1062
1063	ptrs[0] = v;
1064	ptrs[1] = (void *)delete_restarter_dep;
1065
1066	r = uu_list_walk(v->gv_dependencies,
1067	    (uu_walk_fn_t *)delete_instance_deps_cb, &ptrs, UU_WALK_ROBUST);
1068	assert(r == 0);
1069}
1070
1071/*
1072 * int graph_insert_vertex_unconfigured()
1073 *   Insert a vertex without sending any restarter events. If the vertex
1074 *   already exists or creation is successful, return a pointer to it in *vp.
1075 *
1076 *   If type is not GVT_GROUP, dt can remain unset.
1077 *
1078 *   Returns 0, EEXIST, or EINVAL if the arguments are invalid (i.e., fmri
1079 *   doesn't agree with type, or type doesn't agree with dt).
1080 */
1081static int
1082graph_insert_vertex_unconfigured(const char *fmri, gv_type_t type,
1083    depgroup_type_t dt, restarter_error_t rt, graph_vertex_t **vp)
1084{
1085	int r;
1086	int i;
1087
1088	assert(MUTEX_HELD(&dgraph_lock));
1089
1090	switch (type) {
1091	case GVT_SVC:
1092	case GVT_INST:
1093		if (strncmp(fmri, "svc:", sizeof ("svc:") - 1) != 0)
1094			return (EINVAL);
1095		break;
1096
1097	case GVT_FILE:
1098		if (strncmp(fmri, "file:", sizeof ("file:") - 1) != 0)
1099			return (EINVAL);
1100		break;
1101
1102	case GVT_GROUP:
1103		if (dt <= 0 || rt < 0)
1104			return (EINVAL);
1105		break;
1106
1107	default:
1108#ifndef NDEBUG
1109		uu_warn("%s:%d: Unknown type %d.\n", __FILE__, __LINE__, type);
1110#endif
1111		abort();
1112	}
1113
1114	*vp = vertex_get_by_name(fmri);
1115	if (*vp != NULL)
1116		return (EEXIST);
1117
1118	*vp = graph_add_vertex(fmri);
1119
1120	(*vp)->gv_type = type;
1121	(*vp)->gv_depgroup = dt;
1122	(*vp)->gv_restart = rt;
1123
1124	(*vp)->gv_flags = 0;
1125	(*vp)->gv_state = RESTARTER_STATE_NONE;
1126
1127	for (i = 0; special_vertices[i].name != NULL; ++i) {
1128		if (strcmp(fmri, special_vertices[i].name) == 0) {
1129			(*vp)->gv_start_f = special_vertices[i].start_f;
1130			(*vp)->gv_post_online_f =
1131			    special_vertices[i].post_online_f;
1132			(*vp)->gv_post_disable_f =
1133			    special_vertices[i].post_disable_f;
1134			break;
1135		}
1136	}
1137
1138	(*vp)->gv_restarter_id = -1;
1139	(*vp)->gv_restarter_channel = 0;
1140
1141	if (type == GVT_INST) {
1142		char *sfmri;
1143		graph_vertex_t *sv;
1144
1145		sfmri = inst_fmri_to_svc_fmri(fmri);
1146		sv = vertex_get_by_name(sfmri);
1147		if (sv == NULL) {
1148			r = graph_insert_vertex_unconfigured(sfmri, GVT_SVC, 0,
1149			    0, &sv);
1150			assert(r == 0);
1151		}
1152		startd_free(sfmri, max_scf_fmri_size);
1153
1154		graph_add_edge(sv, *vp);
1155	}
1156
1157	/*
1158	 * If this vertex is in the subgraph, mark it as so, for both
1159	 * GVT_INST and GVT_SERVICE verteces.
1160	 * A GVT_SERVICE vertex can only be in the subgraph if another instance
1161	 * depends on it, in which case it's already been added to the graph
1162	 * and marked as in the subgraph (by refresh_vertex()).  If a
1163	 * GVT_SERVICE vertex was freshly added (by the code above), it means
1164	 * that it has no dependents, and cannot be in the subgraph.
1165	 * Regardless of this, we still check that gv_flags includes
1166	 * GV_INSUBGRAPH in the event that future behavior causes the above
1167	 * code to add a GVT_SERVICE vertex which should be in the subgraph.
1168	 */
1169
1170	(*vp)->gv_flags |= (should_be_in_subgraph(*vp)? GV_INSUBGRAPH : 0);
1171
1172	return (0);
1173}
1174
1175/*
1176 * Returns 0 on success or ELOOP if the dependency would create a cycle.
1177 */
1178static int
1179graph_insert_dependency(graph_vertex_t *fv, graph_vertex_t *tv, int **pathp)
1180{
1181	hrtime_t now;
1182
1183	assert(MUTEX_HELD(&dgraph_lock));
1184
1185	/* cycle detection */
1186	now = gethrtime();
1187
1188	/* Don't follow exclusions. */
1189	if (!(fv->gv_type == GVT_GROUP &&
1190	    fv->gv_depgroup == DEPGRP_EXCLUDE_ALL)) {
1191		*pathp = is_path_to(tv, fv);
1192		if (*pathp)
1193			return (ELOOP);
1194	}
1195
1196	dep_cycle_ns += gethrtime() - now;
1197	++dep_inserts;
1198	now = gethrtime();
1199
1200	graph_add_edge(fv, tv);
1201
1202	dep_insert_ns += gethrtime() - now;
1203
1204	/* Check if the dependency adds the "to" vertex to the subgraph */
1205	tv->gv_flags |= (should_be_in_subgraph(tv) ? GV_INSUBGRAPH : 0);
1206
1207	return (0);
1208}
1209
1210static int
1211inst_running(graph_vertex_t *v)
1212{
1213	assert(v->gv_type == GVT_INST);
1214
1215	if (v->gv_state == RESTARTER_STATE_ONLINE ||
1216	    v->gv_state == RESTARTER_STATE_DEGRADED)
1217		return (1);
1218
1219	return (0);
1220}
1221
1222/*
1223 * The dependency evaluation functions return
1224 *   1 - dependency satisfied
1225 *   0 - dependency unsatisfied
1226 *   -1 - dependency unsatisfiable (without administrator intervention)
1227 *
1228 * The functions also take a boolean satbility argument.  When true, the
1229 * functions may recurse in order to determine satisfiability.
1230 */
1231static int require_any_satisfied(graph_vertex_t *, boolean_t);
1232static int dependency_satisfied(graph_vertex_t *, boolean_t);
1233
1234/*
1235 * A require_all dependency is unsatisfied if any elements are unsatisfied.  It
1236 * is unsatisfiable if any elements are unsatisfiable.
1237 */
1238static int
1239require_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1240{
1241	graph_edge_t *edge;
1242	int i;
1243	boolean_t any_unsatisfied;
1244
1245	if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1246		return (1);
1247
1248	any_unsatisfied = B_FALSE;
1249
1250	for (edge = uu_list_first(groupv->gv_dependencies);
1251	    edge != NULL;
1252	    edge = uu_list_next(groupv->gv_dependencies, edge)) {
1253		i = dependency_satisfied(edge->ge_vertex, satbility);
1254		if (i == 1)
1255			continue;
1256
1257		log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1258		    "require_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1259		    edge->ge_vertex->gv_name, i == 0 ? "ed" : "able");
1260
1261		if (!satbility)
1262			return (0);
1263
1264		if (i == -1)
1265			return (-1);
1266
1267		any_unsatisfied = B_TRUE;
1268	}
1269
1270	return (any_unsatisfied ? 0 : 1);
1271}
1272
1273/*
1274 * A require_any dependency is satisfied if any element is satisfied.  It is
1275 * satisfiable if any element is satisfiable.
1276 */
1277static int
1278require_any_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1279{
1280	graph_edge_t *edge;
1281	int s;
1282	boolean_t satisfiable;
1283
1284	if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1285		return (1);
1286
1287	satisfiable = B_FALSE;
1288
1289	for (edge = uu_list_first(groupv->gv_dependencies);
1290	    edge != NULL;
1291	    edge = uu_list_next(groupv->gv_dependencies, edge)) {
1292		s = dependency_satisfied(edge->ge_vertex, satbility);
1293
1294		if (s == 1)
1295			return (1);
1296
1297		log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1298		    "require_any(%s): %s is unsatisfi%s.\n",
1299		    groupv->gv_name, edge->ge_vertex->gv_name,
1300		    s == 0 ? "ed" : "able");
1301
1302		if (satbility && s == 0)
1303			satisfiable = B_TRUE;
1304	}
1305
1306	return ((!satbility || satisfiable) ? 0 : -1);
1307}
1308
1309/*
1310 * An optional_all dependency only considers elements which are configured,
1311 * enabled, and not in maintenance.  If any are unsatisfied, then the dependency
1312 * is unsatisfied.
1313 *
1314 * Offline dependencies which are waiting for a dependency to come online are
1315 * unsatisfied.  Offline dependences which cannot possibly come online
1316 * (unsatisfiable) are always considered satisfied.
1317 */
1318static int
1319optional_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1320{
1321	graph_edge_t *edge;
1322	graph_vertex_t *v;
1323	boolean_t any_qualified;
1324	boolean_t any_unsatisfied;
1325	int i;
1326
1327	any_qualified = B_FALSE;
1328	any_unsatisfied = B_FALSE;
1329
1330	for (edge = uu_list_first(groupv->gv_dependencies);
1331	    edge != NULL;
1332	    edge = uu_list_next(groupv->gv_dependencies, edge)) {
1333		v = edge->ge_vertex;
1334
1335		switch (v->gv_type) {
1336		case GVT_INST:
1337			/* Skip missing instances */
1338			if ((v->gv_flags & GV_CONFIGURED) == 0)
1339				continue;
1340
1341			if (v->gv_state == RESTARTER_STATE_MAINT)
1342				continue;
1343
1344			any_qualified = B_TRUE;
1345			if (v->gv_state == RESTARTER_STATE_OFFLINE ||
1346			    v->gv_state == RESTARTER_STATE_DISABLED) {
1347				/*
1348				 * For offline/disabled dependencies,
1349				 * treat unsatisfiable as satisfied.
1350				 */
1351				i = dependency_satisfied(v, B_TRUE);
1352				if (i == -1)
1353					i = 1;
1354			} else {
1355				i = dependency_satisfied(v, satbility);
1356			}
1357			break;
1358
1359		case GVT_FILE:
1360			any_qualified = B_TRUE;
1361			i = dependency_satisfied(v, satbility);
1362
1363			break;
1364
1365		case GVT_SVC: {
1366			any_qualified = B_TRUE;
1367			i = optional_all_satisfied(v, satbility);
1368
1369			break;
1370		}
1371
1372		case GVT_GROUP:
1373		default:
1374#ifndef NDEBUG
1375			uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1376			    __LINE__, v->gv_type);
1377#endif
1378			abort();
1379		}
1380
1381		if (i == 1)
1382			continue;
1383
1384		log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1385		    "optional_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1386		    v->gv_name, i == 0 ? "ed" : "able");
1387
1388		if (!satbility)
1389			return (0);
1390		if (i == -1)
1391			return (-1);
1392		any_unsatisfied = B_TRUE;
1393	}
1394
1395	if (!any_qualified)
1396		return (1);
1397
1398	return (any_unsatisfied ? 0 : 1);
1399}
1400
1401/*
1402 * An exclude_all dependency is unsatisfied if any non-service element is
1403 * satisfied or any service instance which is configured, enabled, and not in
1404 * maintenance is satisfied.  Usually when unsatisfied, it is also
1405 * unsatisfiable.
1406 */
1407#define	LOG_EXCLUDE(u, v)						\
1408	log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,			\
1409	    "exclude_all(%s): %s is satisfied.\n",			\
1410	    (u)->gv_name, (v)->gv_name)
1411
1412/* ARGSUSED */
1413static int
1414exclude_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1415{
1416	graph_edge_t *edge, *e2;
1417	graph_vertex_t *v, *v2;
1418
1419	for (edge = uu_list_first(groupv->gv_dependencies);
1420	    edge != NULL;
1421	    edge = uu_list_next(groupv->gv_dependencies, edge)) {
1422		v = edge->ge_vertex;
1423
1424		switch (v->gv_type) {
1425		case GVT_INST:
1426			if ((v->gv_flags & GV_CONFIGURED) == 0)
1427				continue;
1428
1429			switch (v->gv_state) {
1430			case RESTARTER_STATE_ONLINE:
1431			case RESTARTER_STATE_DEGRADED:
1432				LOG_EXCLUDE(groupv, v);
1433				return (v->gv_flags & GV_ENABLED ? -1 : 0);
1434
1435			case RESTARTER_STATE_OFFLINE:
1436			case RESTARTER_STATE_UNINIT:
1437				LOG_EXCLUDE(groupv, v);
1438				return (0);
1439
1440			case RESTARTER_STATE_DISABLED:
1441			case RESTARTER_STATE_MAINT:
1442				continue;
1443
1444			default:
1445#ifndef NDEBUG
1446				uu_warn("%s:%d: Unexpected vertex state %d.\n",
1447				    __FILE__, __LINE__, v->gv_state);
1448#endif
1449				abort();
1450			}
1451			/* NOTREACHED */
1452
1453		case GVT_SVC:
1454			break;
1455
1456		case GVT_FILE:
1457			if (!file_ready(v))
1458				continue;
1459			LOG_EXCLUDE(groupv, v);
1460			return (-1);
1461
1462		case GVT_GROUP:
1463		default:
1464#ifndef NDEBUG
1465			uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1466			    __LINE__, v->gv_type);
1467#endif
1468			abort();
1469		}
1470
1471		/* v represents a service */
1472		if (uu_list_numnodes(v->gv_dependencies) == 0)
1473			continue;
1474
1475		for (e2 = uu_list_first(v->gv_dependencies);
1476		    e2 != NULL;
1477		    e2 = uu_list_next(v->gv_dependencies, e2)) {
1478			v2 = e2->ge_vertex;
1479			assert(v2->gv_type == GVT_INST);
1480
1481			if ((v2->gv_flags & GV_CONFIGURED) == 0)
1482				continue;
1483
1484			switch (v2->gv_state) {
1485			case RESTARTER_STATE_ONLINE:
1486			case RESTARTER_STATE_DEGRADED:
1487				LOG_EXCLUDE(groupv, v2);
1488				return (v2->gv_flags & GV_ENABLED ? -1 : 0);
1489
1490			case RESTARTER_STATE_OFFLINE:
1491			case RESTARTER_STATE_UNINIT:
1492				LOG_EXCLUDE(groupv, v2);
1493				return (0);
1494
1495			case RESTARTER_STATE_DISABLED:
1496			case RESTARTER_STATE_MAINT:
1497				continue;
1498
1499			default:
1500#ifndef NDEBUG
1501				uu_warn("%s:%d: Unexpected vertex type %d.\n",
1502				    __FILE__, __LINE__, v2->gv_type);
1503#endif
1504				abort();
1505			}
1506		}
1507	}
1508
1509	return (1);
1510}
1511
1512/*
1513 * int instance_satisfied()
1514 *   Determine if all the dependencies are satisfied for the supplied instance
1515 *   vertex. Return 1 if they are, 0 if they aren't, and -1 if they won't be
1516 *   without administrator intervention.
1517 */
1518static int
1519instance_satisfied(graph_vertex_t *v, boolean_t satbility)
1520{
1521	assert(v->gv_type == GVT_INST);
1522	assert(!inst_running(v));
1523
1524	return (require_all_satisfied(v, satbility));
1525}
1526
1527/*
1528 * Decide whether v can satisfy a dependency.  v can either be a child of
1529 * a group vertex, or of an instance vertex.
1530 */
1531static int
1532dependency_satisfied(graph_vertex_t *v, boolean_t satbility)
1533{
1534	switch (v->gv_type) {
1535	case GVT_INST:
1536		if ((v->gv_flags & GV_CONFIGURED) == 0) {
1537			if (v->gv_flags & GV_DEATHROW) {
1538				/*
1539				 * A dependency on an instance with GV_DEATHROW
1540				 * flag is always considered as satisfied.
1541				 */
1542				return (1);
1543			}
1544			return (-1);
1545		}
1546
1547		/*
1548		 * Vertices may be transitioning so we try to figure out if
1549		 * the end state is likely to satisfy the dependency instead
1550		 * of assuming the dependency is unsatisfied/unsatisfiable.
1551		 *
1552		 * Support for optional_all dependencies depends on us getting
1553		 * this right because unsatisfiable dependencies are treated
1554		 * as being satisfied.
1555		 */
1556		switch (v->gv_state) {
1557		case RESTARTER_STATE_ONLINE:
1558		case RESTARTER_STATE_DEGRADED:
1559			if (v->gv_flags & GV_TODISABLE)
1560				return (-1);
1561			if (v->gv_flags & GV_TOOFFLINE)
1562				return (0);
1563			return (1);
1564
1565		case RESTARTER_STATE_OFFLINE:
1566			if (!satbility || v->gv_flags & GV_TODISABLE)
1567				return (satbility ? -1 : 0);
1568			return (instance_satisfied(v, satbility) != -1 ?
1569			    0 : -1);
1570
1571		case RESTARTER_STATE_DISABLED:
1572			if (!satbility || !(v->gv_flags & GV_ENABLED))
1573				return (satbility ? -1 : 0);
1574			return (instance_satisfied(v, satbility) != -1 ?
1575			    0 : -1);
1576
1577		case RESTARTER_STATE_MAINT:
1578			return (-1);
1579
1580		case RESTARTER_STATE_UNINIT:
1581			return (0);
1582
1583		default:
1584#ifndef NDEBUG
1585			uu_warn("%s:%d: Unexpected vertex state %d.\n",
1586			    __FILE__, __LINE__, v->gv_state);
1587#endif
1588			abort();
1589			/* NOTREACHED */
1590		}
1591
1592	case GVT_SVC:
1593		if (uu_list_numnodes(v->gv_dependencies) == 0)
1594			return (-1);
1595		return (require_any_satisfied(v, satbility));
1596
1597	case GVT_FILE:
1598		/* i.e., we assume files will not be automatically generated */
1599		return (file_ready(v) ? 1 : -1);
1600
1601	case GVT_GROUP:
1602		break;
1603
1604	default:
1605#ifndef NDEBUG
1606		uu_warn("%s:%d: Unexpected node type %d.\n", __FILE__, __LINE__,
1607		    v->gv_type);
1608#endif
1609		abort();
1610		/* NOTREACHED */
1611	}
1612
1613	switch (v->gv_depgroup) {
1614	case DEPGRP_REQUIRE_ANY:
1615		return (require_any_satisfied(v, satbility));
1616
1617	case DEPGRP_REQUIRE_ALL:
1618		return (require_all_satisfied(v, satbility));
1619
1620	case DEPGRP_OPTIONAL_ALL:
1621		return (optional_all_satisfied(v, satbility));
1622
1623	case DEPGRP_EXCLUDE_ALL:
1624		return (exclude_all_satisfied(v, satbility));
1625
1626	default:
1627#ifndef NDEBUG
1628		uu_warn("%s:%d: Unknown dependency grouping %d.\n", __FILE__,
1629		    __LINE__, v->gv_depgroup);
1630#endif
1631		abort();
1632	}
1633}
1634
1635void
1636graph_start_if_satisfied(graph_vertex_t *v)
1637{
1638	if (v->gv_state == RESTARTER_STATE_OFFLINE &&
1639	    instance_satisfied(v, B_FALSE) == 1) {
1640		if (v->gv_start_f == NULL)
1641			vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
1642		else
1643			v->gv_start_f(v);
1644	}
1645}
1646
1647/*
1648 * propagate_satbility()
1649 *
1650 * This function is used when the given vertex changes state in such a way that
1651 * one of its dependents may become unsatisfiable.  This happens when an
1652 * instance transitions between offline -> online, or from !running ->
1653 * maintenance, as well as when an instance is removed from the graph.
1654 *
1655 * We have to walk all the dependents, since optional_all dependencies several
1656 * levels up could become (un)satisfied, instead of unsatisfiable.  For example,
1657 *
1658 *	+-----+  optional_all  +-----+  require_all  +-----+
1659 *	|  A  |--------------->|  B  |-------------->|  C  |
1660 *	+-----+                +-----+               +-----+
1661 *
1662 *	                                        offline -> maintenance
1663 *
1664 * If C goes into maintenance, it's not enough simply to check B.  Because A has
1665 * an optional dependency, what was previously an unsatisfiable situation is now
1666 * satisfied (B will never come online, even though its state hasn't changed).
1667 *
1668 * Note that it's not necessary to continue examining dependents after reaching
1669 * an optional_all dependency.  It's not possible for an optional_all dependency
1670 * to change satisfiability without also coming online, in which case we get a
1671 * start event and propagation continues naturally.  However, it does no harm to
1672 * continue propagating satisfiability (as it is a relatively rare event), and
1673 * keeps the walker code simple and generic.
1674 */
1675/*ARGSUSED*/
1676static int
1677satbility_cb(graph_vertex_t *v, void *arg)
1678{
1679	if (is_inst_bypassed(v))
1680		return (UU_WALK_NEXT);
1681
1682	if (v->gv_type == GVT_INST)
1683		graph_start_if_satisfied(v);
1684
1685	return (UU_WALK_NEXT);
1686}
1687
1688static void
1689propagate_satbility(graph_vertex_t *v)
1690{
1691	graph_walk(v, WALK_DEPENDENTS, satbility_cb, NULL, NULL);
1692}
1693
1694static void propagate_stop(graph_vertex_t *, void *);
1695
1696/*
1697 * propagate_start()
1698 *
1699 * This function is used to propagate a start event to the dependents of the
1700 * given vertex.  Any dependents that are offline but have their dependencies
1701 * satisfied are started.  Any dependents that are online and have restart_on
1702 * set to "restart" or "refresh" are restarted because their dependencies have
1703 * just changed.  This only happens with optional_all dependencies.
1704 */
1705static void
1706propagate_start(graph_vertex_t *v, void *arg)
1707{
1708	restarter_error_t err = (restarter_error_t)arg;
1709
1710	if (is_inst_bypassed(v))
1711		return;
1712
1713	switch (v->gv_type) {
1714	case GVT_INST:
1715		/* Restarter */
1716		if (inst_running(v)) {
1717			if (err == RERR_RESTART || err == RERR_REFRESH) {
1718				vertex_send_event(v,
1719				    RESTARTER_EVENT_TYPE_STOP_RESET);
1720			}
1721		} else {
1722			graph_start_if_satisfied(v);
1723		}
1724		break;
1725
1726	case GVT_GROUP:
1727		if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1728			graph_walk_dependents(v, propagate_stop,
1729			    (void *)RERR_RESTART);
1730			break;
1731		}
1732		err = v->gv_restart;
1733		/* FALLTHROUGH */
1734
1735	case GVT_SVC:
1736		graph_walk_dependents(v, propagate_start, (void *)err);
1737		break;
1738
1739	case GVT_FILE:
1740#ifndef NDEBUG
1741		uu_warn("%s:%d: propagate_start() encountered GVT_FILE.\n",
1742		    __FILE__, __LINE__);
1743#endif
1744		abort();
1745		/* NOTREACHED */
1746
1747	default:
1748#ifndef NDEBUG
1749		uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1750		    v->gv_type);
1751#endif
1752		abort();
1753	}
1754}
1755
1756/*
1757 * propagate_stop()
1758 *
1759 * This function is used to propagate a stop event to the dependents of the
1760 * given vertex.  Any dependents that are online (or in degraded state) with
1761 * the restart_on property set to "restart" or "refresh" will be stopped as
1762 * their dependencies have just changed, propagate_start() will start them
1763 * again once their dependencies have been re-satisfied.
1764 */
1765static void
1766propagate_stop(graph_vertex_t *v, void *arg)
1767{
1768	restarter_error_t err = (restarter_error_t)arg;
1769
1770	if (is_inst_bypassed(v))
1771		return;
1772
1773	switch (v->gv_type) {
1774	case GVT_INST:
1775		/* Restarter */
1776		if (err > RERR_NONE && inst_running(v)) {
1777			if (err == RERR_RESTART || err == RERR_REFRESH) {
1778				vertex_send_event(v,
1779				    RESTARTER_EVENT_TYPE_STOP_RESET);
1780			} else {
1781				vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1782			}
1783		}
1784		break;
1785
1786	case GVT_SVC:
1787		graph_walk_dependents(v, propagate_stop, arg);
1788		break;
1789
1790	case GVT_FILE:
1791#ifndef NDEBUG
1792		uu_warn("%s:%d: propagate_stop() encountered GVT_FILE.\n",
1793		    __FILE__, __LINE__);
1794#endif
1795		abort();
1796		/* NOTREACHED */
1797
1798	case GVT_GROUP:
1799		if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1800			graph_walk_dependents(v, propagate_start,
1801			    (void *)RERR_NONE);
1802			break;
1803		}
1804
1805		if (err == RERR_NONE || err > v->gv_restart)
1806			break;
1807
1808		graph_walk_dependents(v, propagate_stop, arg);
1809		break;
1810
1811	default:
1812#ifndef NDEBUG
1813		uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1814		    v->gv_type);
1815#endif
1816		abort();
1817	}
1818}
1819
1820void
1821offline_vertex(graph_vertex_t *v)
1822{
1823	scf_handle_t *h = libscf_handle_create_bound_loop();
1824	scf_instance_t *scf_inst = safe_scf_instance_create(h);
1825	scf_propertygroup_t *pg = safe_scf_pg_create(h);
1826	restarter_instance_state_t state, next_state;
1827	int r;
1828
1829	assert(v->gv_type == GVT_INST);
1830
1831	if (scf_inst == NULL)
1832		bad_error("safe_scf_instance_create", scf_error());
1833	if (pg == NULL)
1834		bad_error("safe_scf_pg_create", scf_error());
1835
1836	/* if the vertex is already going offline, return */
1837rep_retry:
1838	if (scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, scf_inst, NULL,
1839	    NULL, SCF_DECODE_FMRI_EXACT) != 0) {
1840		switch (scf_error()) {
1841		case SCF_ERROR_CONNECTION_BROKEN:
1842			libscf_handle_rebind(h);
1843			goto rep_retry;
1844
1845		case SCF_ERROR_NOT_FOUND:
1846			scf_pg_destroy(pg);
1847			scf_instance_destroy(scf_inst);
1848			(void) scf_handle_unbind(h);
1849			scf_handle_destroy(h);
1850			return;
1851		}
1852		uu_die("Can't decode FMRI %s: %s\n", v->gv_name,
1853		    scf_strerror(scf_error()));
1854	}
1855
1856	r = scf_instance_get_pg(scf_inst, SCF_PG_RESTARTER, pg);
1857	if (r != 0) {
1858		switch (scf_error()) {
1859		case SCF_ERROR_CONNECTION_BROKEN:
1860			libscf_handle_rebind(h);
1861			goto rep_retry;
1862
1863		case SCF_ERROR_NOT_SET:
1864		case SCF_ERROR_NOT_FOUND:
1865			scf_pg_destroy(pg);
1866			scf_instance_destroy(scf_inst);
1867			(void) scf_handle_unbind(h);
1868			scf_handle_destroy(h);
1869			return;
1870
1871		default:
1872			bad_error("scf_instance_get_pg", scf_error());
1873		}
1874	} else {
1875		r = libscf_read_states(pg, &state, &next_state);
1876		if (r == 0 && (next_state == RESTARTER_STATE_OFFLINE ||
1877		    next_state == RESTARTER_STATE_DISABLED)) {
1878			log_framework(LOG_DEBUG,
1879			    "%s: instance is already going down.\n",
1880			    v->gv_name);
1881			scf_pg_destroy(pg);
1882			scf_instance_destroy(scf_inst);
1883			(void) scf_handle_unbind(h);
1884			scf_handle_destroy(h);
1885			return;
1886		}
1887	}
1888
1889	scf_pg_destroy(pg);
1890	scf_instance_destroy(scf_inst);
1891	(void) scf_handle_unbind(h);
1892	scf_handle_destroy(h);
1893
1894	vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP_RESET);
1895}
1896
1897/*
1898 * void graph_enable_by_vertex()
1899 *   If admin is non-zero, this is an administrative request for change
1900 *   of the enabled property.  Thus, send the ADMIN_DISABLE rather than
1901 *   a plain DISABLE restarter event.
1902 */
1903void
1904graph_enable_by_vertex(graph_vertex_t *vertex, int enable, int admin)
1905{
1906	graph_vertex_t *v;
1907	int r;
1908
1909	assert(MUTEX_HELD(&dgraph_lock));
1910	assert((vertex->gv_flags & GV_CONFIGURED));
1911
1912	vertex->gv_flags = (vertex->gv_flags & ~GV_ENABLED) |
1913	    (enable ? GV_ENABLED : 0);
1914
1915	if (enable) {
1916		if (vertex->gv_state != RESTARTER_STATE_OFFLINE &&
1917		    vertex->gv_state != RESTARTER_STATE_DEGRADED &&
1918		    vertex->gv_state != RESTARTER_STATE_ONLINE) {
1919			/*
1920			 * In case the vertex was notified to go down,
1921			 * but now can return online, clear the _TOOFFLINE
1922			 * and _TODISABLE flags.
1923			 */
1924			vertex->gv_flags &= ~GV_TOOFFLINE;
1925			vertex->gv_flags &= ~GV_TODISABLE;
1926
1927			vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ENABLE);
1928		}
1929
1930		/*
1931		 * Wait for state update from restarter before sending _START or
1932		 * _STOP.
1933		 */
1934
1935		return;
1936	}
1937
1938	if (vertex->gv_state == RESTARTER_STATE_DISABLED)
1939		return;
1940
1941	if (!admin) {
1942		vertex_send_event(vertex, RESTARTER_EVENT_TYPE_DISABLE);
1943
1944		/*
1945		 * Wait for state update from restarter before sending _START or
1946		 * _STOP.
1947		 */
1948
1949		return;
1950	}
1951
1952	/*
1953	 * If it is a DISABLE event requested by the administrator then we are
1954	 * offlining the dependents first.
1955	 */
1956
1957	/*
1958	 * Set GV_TOOFFLINE for the services we are offlining. We cannot
1959	 * clear the GV_TOOFFLINE bits from all the services because
1960	 * other DISABLE events might be handled at the same time.
1961	 */
1962	vertex->gv_flags |= GV_TOOFFLINE;
1963
1964	/* remember which vertex to disable... */
1965	vertex->gv_flags |= GV_TODISABLE;
1966
1967	log_framework(LOG_DEBUG, "Marking in-subtree vertices before "
1968	    "disabling %s.\n", vertex->gv_name);
1969
1970	/* set GV_TOOFFLINE for its dependents */
1971	r = uu_list_walk(vertex->gv_dependents, (uu_walk_fn_t *)mark_subtree,
1972	    NULL, 0);
1973	assert(r == 0);
1974
1975	/* disable the instance now if there is nothing else to offline */
1976	if (insubtree_dependents_down(vertex) == B_TRUE) {
1977		vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
1978		return;
1979	}
1980
1981	/*
1982	 * This loop is similar to the one used for the graph reversal shutdown
1983	 * and could be improved in term of performance for the subtree reversal
1984	 * disable case.
1985	 */
1986	for (v = uu_list_first(dgraph); v != NULL;
1987	    v = uu_list_next(dgraph, v)) {
1988		/* skip the vertex we are disabling for now */
1989		if (v == vertex)
1990			continue;
1991
1992		if (v->gv_type != GVT_INST ||
1993		    (v->gv_flags & GV_CONFIGURED) == 0 ||
1994		    (v->gv_flags & GV_ENABLED) == 0 ||
1995		    (v->gv_flags & GV_TOOFFLINE) == 0)
1996			continue;
1997
1998		if ((v->gv_state != RESTARTER_STATE_ONLINE) &&
1999		    (v->gv_state != RESTARTER_STATE_DEGRADED)) {
2000			/* continue if there is nothing to offline */
2001			continue;
2002		}
2003
2004		/*
2005		 * Instances which are up need to come down before we're
2006		 * done, but we can only offline the leaves here. An
2007		 * instance is a leaf when all its dependents are down.
2008		 */
2009		if (insubtree_dependents_down(v) == B_TRUE) {
2010			log_framework(LOG_DEBUG, "Offlining in-subtree "
2011			    "instance %s for %s.\n",
2012			    v->gv_name, vertex->gv_name);
2013			offline_vertex(v);
2014		}
2015	}
2016}
2017
2018static int configure_vertex(graph_vertex_t *, scf_instance_t *);
2019
2020/*
2021 * Set the restarter for v to fmri_arg.  That is, make sure a vertex for
2022 * fmri_arg exists, make v depend on it, and send _ADD_INSTANCE for v.  If
2023 * v is already configured and fmri_arg indicates the current restarter, do
2024 * nothing.  If v is configured and fmri_arg is a new restarter, delete v's
2025 * dependency on the restarter, send _REMOVE_INSTANCE for v, and set the new
2026 * restarter.  Returns 0 on success, EINVAL if the FMRI is invalid,
2027 * ECONNABORTED if the repository connection is broken, and ELOOP
2028 * if the dependency would create a cycle.  In the last case, *pathp will
2029 * point to a -1-terminated array of ids which compose the path from v to
2030 * restarter_fmri.
2031 */
2032int
2033graph_change_restarter(graph_vertex_t *v, const char *fmri_arg, scf_handle_t *h,
2034    int **pathp)
2035{
2036	char *restarter_fmri = NULL;
2037	graph_vertex_t *rv;
2038	int err;
2039	int id;
2040
2041	assert(MUTEX_HELD(&dgraph_lock));
2042
2043	if (fmri_arg[0] != '\0') {
2044		err = fmri_canonify(fmri_arg, &restarter_fmri, B_TRUE);
2045		if (err != 0) {
2046			assert(err == EINVAL);
2047			return (err);
2048		}
2049	}
2050
2051	if (restarter_fmri == NULL ||
2052	    strcmp(restarter_fmri, SCF_SERVICE_STARTD) == 0) {
2053		if (v->gv_flags & GV_CONFIGURED) {
2054			if (v->gv_restarter_id == -1) {
2055				if (restarter_fmri != NULL)
2056					startd_free(restarter_fmri,
2057					    max_scf_fmri_size);
2058				return (0);
2059			}
2060
2061			graph_unset_restarter(v);
2062		}
2063
2064		/* Master restarter, nothing to do. */
2065		v->gv_restarter_id = -1;
2066		v->gv_restarter_channel = NULL;
2067		vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2068		return (0);
2069	}
2070
2071	if (v->gv_flags & GV_CONFIGURED) {
2072		id = dict_lookup_byname(restarter_fmri);
2073		if (id != -1 && v->gv_restarter_id == id) {
2074			startd_free(restarter_fmri, max_scf_fmri_size);
2075			return (0);
2076		}
2077
2078		graph_unset_restarter(v);
2079	}
2080
2081	err = graph_insert_vertex_unconfigured(restarter_fmri, GVT_INST, 0,
2082	    RERR_NONE, &rv);
2083	startd_free(restarter_fmri, max_scf_fmri_size);
2084	assert(err == 0 || err == EEXIST);
2085
2086	if (rv->gv_delegate_initialized == 0) {
2087		if ((rv->gv_delegate_channel = restarter_protocol_init_delegate(
2088		    rv->gv_name)) == NULL)
2089			return (EINVAL);
2090		rv->gv_delegate_initialized = 1;
2091	}
2092	v->gv_restarter_id = rv->gv_id;
2093	v->gv_restarter_channel = rv->gv_delegate_channel;
2094
2095	err = graph_insert_dependency(v, rv, pathp);
2096	if (err != 0) {
2097		assert(err == ELOOP);
2098		return (ELOOP);
2099	}
2100
2101	vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2102
2103	if (!(rv->gv_flags & GV_CONFIGURED)) {
2104		scf_instance_t *inst;
2105
2106		err = libscf_fmri_get_instance(h, rv->gv_name, &inst);
2107		switch (err) {
2108		case 0:
2109			err = configure_vertex(rv, inst);
2110			scf_instance_destroy(inst);
2111			switch (err) {
2112			case 0:
2113			case ECANCELED:
2114				break;
2115
2116			case ECONNABORTED:
2117				return (ECONNABORTED);
2118
2119			default:
2120				bad_error("configure_vertex", err);
2121			}
2122			break;
2123
2124		case ECONNABORTED:
2125			return (ECONNABORTED);
2126
2127		case ENOENT:
2128			break;
2129
2130		case ENOTSUP:
2131			/*
2132			 * The fmri doesn't specify an instance - translate
2133			 * to EINVAL.
2134			 */
2135			return (EINVAL);
2136
2137		case EINVAL:
2138		default:
2139			bad_error("libscf_fmri_get_instance", err);
2140		}
2141	}
2142
2143	return (0);
2144}
2145
2146
2147/*
2148 * Add all of the instances of the service named by fmri to the graph.
2149 * Returns
2150 *   0 - success
2151 *   ENOENT - service indicated by fmri does not exist
2152 *
2153 * In both cases *reboundp will be B_TRUE if the handle was rebound, or B_FALSE
2154 * otherwise.
2155 */
2156static int
2157add_service(const char *fmri, scf_handle_t *h, boolean_t *reboundp)
2158{
2159	scf_service_t *svc;
2160	scf_instance_t *inst;
2161	scf_iter_t *iter;
2162	char *inst_fmri;
2163	int ret, r;
2164
2165	*reboundp = B_FALSE;
2166
2167	svc = safe_scf_service_create(h);
2168	inst = safe_scf_instance_create(h);
2169	iter = safe_scf_iter_create(h);
2170	inst_fmri = startd_alloc(max_scf_fmri_size);
2171
2172rebound:
2173	if (scf_handle_decode_fmri(h, fmri, NULL, svc, NULL, NULL, NULL,
2174	    SCF_DECODE_FMRI_EXACT) != 0) {
2175		switch (scf_error()) {
2176		case SCF_ERROR_CONNECTION_BROKEN:
2177		default:
2178			libscf_handle_rebind(h);
2179			*reboundp = B_TRUE;
2180			goto rebound;
2181
2182		case SCF_ERROR_NOT_FOUND:
2183			ret = ENOENT;
2184			goto out;
2185
2186		case SCF_ERROR_INVALID_ARGUMENT:
2187		case SCF_ERROR_CONSTRAINT_VIOLATED:
2188		case SCF_ERROR_NOT_BOUND:
2189		case SCF_ERROR_HANDLE_MISMATCH:
2190			bad_error("scf_handle_decode_fmri", scf_error());
2191		}
2192	}
2193
2194	if (scf_iter_service_instances(iter, svc) != 0) {
2195		switch (scf_error()) {
2196		case SCF_ERROR_CONNECTION_BROKEN:
2197		default:
2198			libscf_handle_rebind(h);
2199			*reboundp = B_TRUE;
2200			goto rebound;
2201
2202		case SCF_ERROR_DELETED:
2203			ret = ENOENT;
2204			goto out;
2205
2206		case SCF_ERROR_HANDLE_MISMATCH:
2207		case SCF_ERROR_NOT_BOUND:
2208		case SCF_ERROR_NOT_SET:
2209			bad_error("scf_iter_service_instances", scf_error());
2210		}
2211	}
2212
2213	for (;;) {
2214		r = scf_iter_next_instance(iter, inst);
2215		if (r == 0)
2216			break;
2217		if (r != 1) {
2218			switch (scf_error()) {
2219			case SCF_ERROR_CONNECTION_BROKEN:
2220			default:
2221				libscf_handle_rebind(h);
2222				*reboundp = B_TRUE;
2223				goto rebound;
2224
2225			case SCF_ERROR_DELETED:
2226				ret = ENOENT;
2227				goto out;
2228
2229			case SCF_ERROR_HANDLE_MISMATCH:
2230			case SCF_ERROR_NOT_BOUND:
2231			case SCF_ERROR_NOT_SET:
2232			case SCF_ERROR_INVALID_ARGUMENT:
2233				bad_error("scf_iter_next_instance",
2234				    scf_error());
2235			}
2236		}
2237
2238		if (scf_instance_to_fmri(inst, inst_fmri, max_scf_fmri_size) <
2239		    0) {
2240			switch (scf_error()) {
2241			case SCF_ERROR_CONNECTION_BROKEN:
2242				libscf_handle_rebind(h);
2243				*reboundp = B_TRUE;
2244				goto rebound;
2245
2246			case SCF_ERROR_DELETED:
2247				continue;
2248
2249			case SCF_ERROR_NOT_BOUND:
2250			case SCF_ERROR_NOT_SET:
2251				bad_error("scf_instance_to_fmri", scf_error());
2252			}
2253		}
2254
2255		r = dgraph_add_instance(inst_fmri, inst, B_FALSE);
2256		switch (r) {
2257		case 0:
2258		case ECANCELED:
2259			break;
2260
2261		case EEXIST:
2262			continue;
2263
2264		case ECONNABORTED:
2265			libscf_handle_rebind(h);
2266			*reboundp = B_TRUE;
2267			goto rebound;
2268
2269		case EINVAL:
2270		default:
2271			bad_error("dgraph_add_instance", r);
2272		}
2273	}
2274
2275	ret = 0;
2276
2277out:
2278	startd_free(inst_fmri, max_scf_fmri_size);
2279	scf_iter_destroy(iter);
2280	scf_instance_destroy(inst);
2281	scf_service_destroy(svc);
2282	return (ret);
2283}
2284
2285struct depfmri_info {
2286	graph_vertex_t	*v;		/* GVT_GROUP vertex */
2287	gv_type_t	type;		/* type of dependency */
2288	const char	*inst_fmri;	/* FMRI of parental GVT_INST vert. */
2289	const char	*pg_name;	/* Name of dependency pg */
2290	scf_handle_t	*h;
2291	int		err;		/* return error code */
2292	int		**pathp;	/* return circular dependency path */
2293};
2294
2295/*
2296 * Find or create a vertex for fmri and make info->v depend on it.
2297 * Returns
2298 *   0 - success
2299 *   nonzero - failure
2300 *
2301 * On failure, sets info->err to
2302 *   EINVAL - fmri is invalid
2303 *	      fmri does not match info->type
2304 *   ELOOP - Adding the dependency creates a circular dependency.  *info->pathp
2305 *	     will point to an array of the ids of the members of the cycle.
2306 *   ECONNABORTED - repository connection was broken
2307 *   ECONNRESET - succeeded, but repository connection was reset
2308 */
2309static int
2310process_dependency_fmri(const char *fmri, struct depfmri_info *info)
2311{
2312	int err;
2313	graph_vertex_t *depgroup_v, *v;
2314	char *fmri_copy, *cfmri;
2315	size_t fmri_copy_sz;
2316	const char *scope, *service, *instance, *pg;
2317	scf_instance_t *inst;
2318	boolean_t rebound;
2319
2320	assert(MUTEX_HELD(&dgraph_lock));
2321
2322	/* Get or create vertex for FMRI */
2323	depgroup_v = info->v;
2324
2325	if (strncmp(fmri, "file:", sizeof ("file:") - 1) == 0) {
2326		if (info->type != GVT_FILE) {
2327			log_framework(LOG_NOTICE,
2328			    "FMRI \"%s\" is not allowed for the \"%s\" "
2329			    "dependency's type of instance %s.\n", fmri,
2330			    info->pg_name, info->inst_fmri);
2331			return (info->err = EINVAL);
2332		}
2333
2334		err = graph_insert_vertex_unconfigured(fmri, info->type, 0,
2335		    RERR_NONE, &v);
2336		switch (err) {
2337		case 0:
2338			break;
2339
2340		case EEXIST:
2341			assert(v->gv_type == GVT_FILE);
2342			break;
2343
2344		case EINVAL:		/* prevented above */
2345		default:
2346			bad_error("graph_insert_vertex_unconfigured", err);
2347		}
2348	} else {
2349		if (info->type != GVT_INST) {
2350			log_framework(LOG_NOTICE,
2351			    "FMRI \"%s\" is not allowed for the \"%s\" "
2352			    "dependency's type of instance %s.\n", fmri,
2353			    info->pg_name, info->inst_fmri);
2354			return (info->err = EINVAL);
2355		}
2356
2357		/*
2358		 * We must canonify fmri & add a vertex for it.
2359		 */
2360		fmri_copy_sz = strlen(fmri) + 1;
2361		fmri_copy = startd_alloc(fmri_copy_sz);
2362		(void) strcpy(fmri_copy, fmri);
2363
2364		/* Determine if the FMRI is a property group or instance */
2365		if (scf_parse_svc_fmri(fmri_copy, &scope, &service,
2366		    &instance, &pg, NULL) != 0) {
2367			startd_free(fmri_copy, fmri_copy_sz);
2368			log_framework(LOG_NOTICE,
2369			    "Dependency \"%s\" of %s has invalid FMRI "
2370			    "\"%s\".\n", info->pg_name, info->inst_fmri,
2371			    fmri);
2372			return (info->err = EINVAL);
2373		}
2374
2375		if (service == NULL || pg != NULL) {
2376			startd_free(fmri_copy, fmri_copy_sz);
2377			log_framework(LOG_NOTICE,
2378			    "Dependency \"%s\" of %s does not designate a "
2379			    "service or instance.\n", info->pg_name,
2380			    info->inst_fmri);
2381			return (info->err = EINVAL);
2382		}
2383
2384		if (scope == NULL || strcmp(scope, SCF_SCOPE_LOCAL) == 0) {
2385			cfmri = uu_msprintf("svc:/%s%s%s",
2386			    service, instance ? ":" : "", instance ? instance :
2387			    "");
2388		} else {
2389			cfmri = uu_msprintf("svc://%s/%s%s%s",
2390			    scope, service, instance ? ":" : "", instance ?
2391			    instance : "");
2392		}
2393
2394		startd_free(fmri_copy, fmri_copy_sz);
2395
2396		err = graph_insert_vertex_unconfigured(cfmri, instance ?
2397		    GVT_INST : GVT_SVC, instance ? 0 : DEPGRP_REQUIRE_ANY,
2398		    RERR_NONE, &v);
2399		uu_free(cfmri);
2400		switch (err) {
2401		case 0:
2402			break;
2403
2404		case EEXIST:
2405			/* Verify v. */
2406			if (instance != NULL)
2407				assert(v->gv_type == GVT_INST);
2408			else
2409				assert(v->gv_type == GVT_SVC);
2410			break;
2411
2412		default:
2413			bad_error("graph_insert_vertex_unconfigured", err);
2414		}
2415	}
2416
2417	/* Add dependency from depgroup_v to new vertex */
2418	info->err = graph_insert_dependency(depgroup_v, v, info->pathp);
2419	switch (info->err) {
2420	case 0:
2421		break;
2422
2423	case ELOOP:
2424		return (ELOOP);
2425
2426	default:
2427		bad_error("graph_insert_dependency", info->err);
2428	}
2429
2430	/* This must be after we insert the dependency, to avoid looping. */
2431	switch (v->gv_type) {
2432	case GVT_INST:
2433		if ((v->gv_flags & GV_CONFIGURED) != 0)
2434			break;
2435
2436		inst = safe_scf_instance_create(info->h);
2437
2438		rebound = B_FALSE;
2439
2440rebound:
2441		err = libscf_lookup_instance(v->gv_name, inst);
2442		switch (err) {
2443		case 0:
2444			err = configure_vertex(v, inst);
2445			switch (err) {
2446			case 0:
2447			case ECANCELED:
2448				break;
2449
2450			case ECONNABORTED:
2451				libscf_handle_rebind(info->h);
2452				rebound = B_TRUE;
2453				goto rebound;
2454
2455			default:
2456				bad_error("configure_vertex", err);
2457			}
2458			break;
2459
2460		case ENOENT:
2461			break;
2462
2463		case ECONNABORTED:
2464			libscf_handle_rebind(info->h);
2465			rebound = B_TRUE;
2466			goto rebound;
2467
2468		case EINVAL:
2469		case ENOTSUP:
2470		default:
2471			bad_error("libscf_fmri_get_instance", err);
2472		}
2473
2474		scf_instance_destroy(inst);
2475
2476		if (rebound)
2477			return (info->err = ECONNRESET);
2478		break;
2479
2480	case GVT_SVC:
2481		(void) add_service(v->gv_name, info->h, &rebound);
2482		if (rebound)
2483			return (info->err = ECONNRESET);
2484	}
2485
2486	return (0);
2487}
2488
2489struct deppg_info {
2490	graph_vertex_t	*v;		/* GVT_INST vertex */
2491	int		err;		/* return error */
2492	int		**pathp;	/* return circular dependency path */
2493};
2494
2495/*
2496 * Make info->v depend on a new GVT_GROUP node for this property group,
2497 * and then call process_dependency_fmri() for the values of the entity
2498 * property.  Return 0 on success, or if something goes wrong return nonzero
2499 * and set info->err to ECONNABORTED, EINVAL, or the error code returned by
2500 * process_dependency_fmri().
2501 */
2502static int
2503process_dependency_pg(scf_propertygroup_t *pg, struct deppg_info *info)
2504{
2505	scf_handle_t *h;
2506	depgroup_type_t deptype;
2507	restarter_error_t rerr;
2508	struct depfmri_info linfo;
2509	char *fmri, *pg_name;
2510	size_t fmri_sz;
2511	graph_vertex_t *depgrp;
2512	scf_property_t *prop;
2513	int err;
2514	int empty;
2515	scf_error_t scferr;
2516	ssize_t len;
2517
2518	assert(MUTEX_HELD(&dgraph_lock));
2519
2520	h = scf_pg_handle(pg);
2521
2522	pg_name = startd_alloc(max_scf_name_size);
2523
2524	len = scf_pg_get_name(pg, pg_name, max_scf_name_size);
2525	if (len < 0) {
2526		startd_free(pg_name, max_scf_name_size);
2527		switch (scf_error()) {
2528		case SCF_ERROR_CONNECTION_BROKEN:
2529		default:
2530			return (info->err = ECONNABORTED);
2531
2532		case SCF_ERROR_DELETED:
2533			return (info->err = 0);
2534
2535		case SCF_ERROR_NOT_SET:
2536			bad_error("scf_pg_get_name", scf_error());
2537		}
2538	}
2539
2540	/*
2541	 * Skip over empty dependency groups.  Since dependency property
2542	 * groups are updated atomically, they are either empty or
2543	 * fully populated.
2544	 */
2545	empty = depgroup_empty(h, pg);
2546	if (empty < 0) {
2547		log_error(LOG_INFO,
2548		    "Error reading dependency group \"%s\" of %s: %s\n",
2549		    pg_name, info->v->gv_name, scf_strerror(scf_error()));
2550		startd_free(pg_name, max_scf_name_size);
2551		return (info->err = EINVAL);
2552
2553	} else if (empty == 1) {
2554		log_framework(LOG_DEBUG,
2555		    "Ignoring empty dependency group \"%s\" of %s\n",
2556		    pg_name, info->v->gv_name);
2557		startd_free(pg_name, max_scf_name_size);
2558		return (info->err = 0);
2559	}
2560
2561	fmri_sz = strlen(info->v->gv_name) + 1 + len + 1;
2562	fmri = startd_alloc(fmri_sz);
2563
2564	(void) snprintf(fmri, fmri_sz, "%s>%s", info->v->gv_name,
2565	    pg_name);
2566
2567	/* Validate the pg before modifying the graph */
2568	deptype = depgroup_read_grouping(h, pg);
2569	if (deptype == DEPGRP_UNSUPPORTED) {
2570		log_error(LOG_INFO,
2571		    "Dependency \"%s\" of %s has an unknown grouping value.\n",
2572		    pg_name, info->v->gv_name);
2573		startd_free(fmri, fmri_sz);
2574		startd_free(pg_name, max_scf_name_size);
2575		return (info->err = EINVAL);
2576	}
2577
2578	rerr = depgroup_read_restart(h, pg);
2579	if (rerr == RERR_UNSUPPORTED) {
2580		log_error(LOG_INFO,
2581		    "Dependency \"%s\" of %s has an unknown restart_on value."
2582		    "\n", pg_name, info->v->gv_name);
2583		startd_free(fmri, fmri_sz);
2584		startd_free(pg_name, max_scf_name_size);
2585		return (info->err = EINVAL);
2586	}
2587
2588	prop = safe_scf_property_create(h);
2589
2590	if (scf_pg_get_property(pg, SCF_PROPERTY_ENTITIES, prop) != 0) {
2591		scferr = scf_error();
2592		scf_property_destroy(prop);
2593		if (scferr == SCF_ERROR_DELETED) {
2594			startd_free(fmri, fmri_sz);
2595			startd_free(pg_name, max_scf_name_size);
2596			return (info->err = 0);
2597		} else if (scferr != SCF_ERROR_NOT_FOUND) {
2598			startd_free(fmri, fmri_sz);
2599			startd_free(pg_name, max_scf_name_size);
2600			return (info->err = ECONNABORTED);
2601		}
2602
2603		log_error(LOG_INFO,
2604		    "Dependency \"%s\" of %s is missing a \"%s\" property.\n",
2605		    pg_name, info->v->gv_name, SCF_PROPERTY_ENTITIES);
2606
2607		startd_free(fmri, fmri_sz);
2608		startd_free(pg_name, max_scf_name_size);
2609
2610		return (info->err = EINVAL);
2611	}
2612
2613	/* Create depgroup vertex for pg */
2614	err = graph_insert_vertex_unconfigured(fmri, GVT_GROUP, deptype,
2615	    rerr, &depgrp);
2616	assert(err == 0);
2617	startd_free(fmri, fmri_sz);
2618
2619	/* Add dependency from inst vertex to new vertex */
2620	err = graph_insert_dependency(info->v, depgrp, info->pathp);
2621	/* ELOOP can't happen because this should be a new vertex */
2622	assert(err == 0);
2623
2624	linfo.v = depgrp;
2625	linfo.type = depgroup_read_scheme(h, pg);
2626	linfo.inst_fmri = info->v->gv_name;
2627	linfo.pg_name = pg_name;
2628	linfo.h = h;
2629	linfo.err = 0;
2630	linfo.pathp = info->pathp;
2631	err = walk_property_astrings(prop, (callback_t)process_dependency_fmri,
2632	    &linfo);
2633
2634	scf_property_destroy(prop);
2635	startd_free(pg_name, max_scf_name_size);
2636
2637	switch (err) {
2638	case 0:
2639	case EINTR:
2640		return (info->err = linfo.err);
2641
2642	case ECONNABORTED:
2643	case EINVAL:
2644		return (info->err = err);
2645
2646	case ECANCELED:
2647		return (info->err = 0);
2648
2649	case ECONNRESET:
2650		return (info->err = ECONNABORTED);
2651
2652	default:
2653		bad_error("walk_property_astrings", err);
2654		/* NOTREACHED */
2655	}
2656}
2657
2658/*
2659 * Build the dependency info for v from the repository.  Returns 0 on success,
2660 * ECONNABORTED on repository disconnection, EINVAL if the repository
2661 * configuration is invalid, and ELOOP if a dependency would cause a cycle.
2662 * In the last case, *pathp will point to a -1-terminated array of ids which
2663 * constitute the rest of the dependency cycle.
2664 */
2665static int
2666set_dependencies(graph_vertex_t *v, scf_instance_t *inst, int **pathp)
2667{
2668	struct deppg_info info;
2669	int err;
2670	uint_t old_configured;
2671
2672	assert(MUTEX_HELD(&dgraph_lock));
2673
2674	/*
2675	 * Mark the vertex as configured during dependency insertion to avoid
2676	 * dependency cycles (which can appear in the graph if one of the
2677	 * vertices is an exclusion-group).
2678	 */
2679	old_configured = v->gv_flags & GV_CONFIGURED;
2680	v->gv_flags |= GV_CONFIGURED;
2681
2682	info.err = 0;
2683	info.v = v;
2684	info.pathp = pathp;
2685
2686	err = walk_dependency_pgs(inst, (callback_t)process_dependency_pg,
2687	    &info);
2688
2689	if (!old_configured)
2690		v->gv_flags &= ~GV_CONFIGURED;
2691
2692	switch (err) {
2693	case 0:
2694	case EINTR:
2695		return (info.err);
2696
2697	case ECONNABORTED:
2698		return (ECONNABORTED);
2699
2700	case ECANCELED:
2701		/* Should get delete event, so return 0. */
2702		return (0);
2703
2704	default:
2705		bad_error("walk_dependency_pgs", err);
2706		/* NOTREACHED */
2707	}
2708}
2709
2710
2711static void
2712handle_cycle(const char *fmri, int *path)
2713{
2714	const char *cp;
2715	size_t sz;
2716
2717	assert(MUTEX_HELD(&dgraph_lock));
2718
2719	path_to_str(path, (char **)&cp, &sz);
2720
2721	log_error(LOG_ERR, "Transitioning %s to maintenance "
2722	    "because it completes a dependency cycle (see svcs -xv for "
2723	    "details):\n%s", fmri ? fmri : "?", cp);
2724
2725	startd_free((void *)cp, sz);
2726}
2727
2728/*
2729 * Increment the vertex's reference count to prevent the vertex removal
2730 * from the dgraph.
2731 */
2732static void
2733vertex_ref(graph_vertex_t *v)
2734{
2735	assert(MUTEX_HELD(&dgraph_lock));
2736
2737	v->gv_refs++;
2738}
2739
2740/*
2741 * Decrement the vertex's reference count and remove the vertex from
2742 * the dgraph when possible.
2743 *
2744 * Return VERTEX_REMOVED when the vertex has been removed otherwise
2745 * return VERTEX_INUSE.
2746 */
2747static int
2748vertex_unref(graph_vertex_t *v)
2749{
2750	assert(MUTEX_HELD(&dgraph_lock));
2751	assert(v->gv_refs > 0);
2752
2753	v->gv_refs--;
2754
2755	return (free_if_unrefed(v));
2756}
2757
2758/*
2759 * When run on the dependencies of a vertex, populates list with
2760 * graph_edge_t's which point to the service vertices or the instance
2761 * vertices (no GVT_GROUP nodes) on which the vertex depends.
2762 *
2763 * Increment the vertex's reference count once the vertex is inserted
2764 * in the list. The vertex won't be able to be deleted from the dgraph
2765 * while it is referenced.
2766 */
2767static int
2768append_svcs_or_insts(graph_edge_t *e, uu_list_t *list)
2769{
2770	graph_vertex_t *v = e->ge_vertex;
2771	graph_edge_t *new;
2772	int r;
2773
2774	switch (v->gv_type) {
2775	case GVT_INST:
2776	case GVT_SVC:
2777		break;
2778
2779	case GVT_GROUP:
2780		r = uu_list_walk(v->gv_dependencies,
2781		    (uu_walk_fn_t *)append_svcs_or_insts, list, 0);
2782		assert(r == 0);
2783		return (UU_WALK_NEXT);
2784
2785	case GVT_FILE:
2786		return (UU_WALK_NEXT);
2787
2788	default:
2789#ifndef NDEBUG
2790		uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
2791		    __LINE__, v->gv_type);
2792#endif
2793		abort();
2794	}
2795
2796	new = startd_alloc(sizeof (*new));
2797	new->ge_vertex = v;
2798	uu_list_node_init(new, &new->ge_link, graph_edge_pool);
2799	r = uu_list_insert_before(list, NULL, new);
2800	assert(r == 0);
2801
2802	/*
2803	 * Because we are inserting the vertex in a list, we don't want
2804	 * the vertex to be freed while the list is in use. In order to
2805	 * achieve that, increment the vertex's reference count.
2806	 */
2807	vertex_ref(v);
2808
2809	return (UU_WALK_NEXT);
2810}
2811
2812static boolean_t
2813should_be_in_subgraph(graph_vertex_t *v)
2814{
2815	graph_edge_t *e;
2816
2817	if (v == milestone)
2818		return (B_TRUE);
2819
2820	/*
2821	 * v is in the subgraph if any of its dependents are in the subgraph.
2822	 * Except for EXCLUDE_ALL dependents.  And OPTIONAL dependents only
2823	 * count if we're enabled.
2824	 */
2825	for (e = uu_list_first(v->gv_dependents);
2826	    e != NULL;
2827	    e = uu_list_next(v->gv_dependents, e)) {
2828		graph_vertex_t *dv = e->ge_vertex;
2829
2830		if (!(dv->gv_flags & GV_INSUBGRAPH))
2831			continue;
2832
2833		/*
2834		 * Don't include instances that are optional and disabled.
2835		 */
2836		if (v->gv_type == GVT_INST && dv->gv_type == GVT_SVC) {
2837
2838			int in = 0;
2839			graph_edge_t *ee;
2840
2841			for (ee = uu_list_first(dv->gv_dependents);
2842			    ee != NULL;
2843			    ee = uu_list_next(dv->gv_dependents, ee)) {
2844
2845				graph_vertex_t *ddv = e->ge_vertex;
2846
2847				if (ddv->gv_type == GVT_GROUP &&
2848				    ddv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2849					continue;
2850
2851				if (ddv->gv_type == GVT_GROUP &&
2852				    ddv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2853				    !(v->gv_flags & GV_ENBLD_NOOVR))
2854					continue;
2855
2856				in = 1;
2857			}
2858			if (!in)
2859				continue;
2860		}
2861		if (v->gv_type == GVT_INST &&
2862		    dv->gv_type == GVT_GROUP &&
2863		    dv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2864		    !(v->gv_flags & GV_ENBLD_NOOVR))
2865			continue;
2866
2867		/* Don't include excluded services and instances */
2868		if (dv->gv_type == GVT_GROUP &&
2869		    dv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2870			continue;
2871
2872		return (B_TRUE);
2873	}
2874
2875	return (B_FALSE);
2876}
2877
2878/*
2879 * Ensures that GV_INSUBGRAPH is set properly for v and its descendents.  If
2880 * any bits change, manipulate the repository appropriately.  Returns 0 or
2881 * ECONNABORTED.
2882 */
2883static int
2884eval_subgraph(graph_vertex_t *v, scf_handle_t *h)
2885{
2886	boolean_t old = (v->gv_flags & GV_INSUBGRAPH) != 0;
2887	boolean_t new;
2888	graph_edge_t *e;
2889	scf_instance_t *inst;
2890	int ret = 0, r;
2891
2892	assert(milestone != NULL && milestone != MILESTONE_NONE);
2893
2894	new = should_be_in_subgraph(v);
2895
2896	if (new == old)
2897		return (0);
2898
2899	log_framework(LOG_DEBUG, new ? "Adding %s to the subgraph.\n" :
2900	    "Removing %s from the subgraph.\n", v->gv_name);
2901
2902	v->gv_flags = (v->gv_flags & ~GV_INSUBGRAPH) |
2903	    (new ? GV_INSUBGRAPH : 0);
2904
2905	if (v->gv_type == GVT_INST && (v->gv_flags & GV_CONFIGURED)) {
2906		int err;
2907
2908get_inst:
2909		err = libscf_fmri_get_instance(h, v->gv_name, &inst);
2910		if (err != 0) {
2911			switch (err) {
2912			case ECONNABORTED:
2913				libscf_handle_rebind(h);
2914				ret = ECONNABORTED;
2915				goto get_inst;
2916
2917			case ENOENT:
2918				break;
2919
2920			case EINVAL:
2921			case ENOTSUP:
2922			default:
2923				bad_error("libscf_fmri_get_instance", err);
2924			}
2925		} else {
2926			const char *f;
2927
2928			if (new) {
2929				err = libscf_delete_enable_ovr(inst);
2930				f = "libscf_delete_enable_ovr";
2931			} else {
2932				err = libscf_set_enable_ovr(inst, 0);
2933				f = "libscf_set_enable_ovr";
2934			}
2935			scf_instance_destroy(inst);
2936			switch (err) {
2937			case 0:
2938			case ECANCELED:
2939				break;
2940
2941			case ECONNABORTED:
2942				libscf_handle_rebind(h);
2943				/*
2944				 * We must continue so the graph is updated,
2945				 * but we must return ECONNABORTED so any
2946				 * libscf state held by any callers is reset.
2947				 */
2948				ret = ECONNABORTED;
2949				goto get_inst;
2950
2951			case EROFS:
2952			case EPERM:
2953				log_error(LOG_WARNING,
2954				    "Could not set %s/%s for %s: %s.\n",
2955				    SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
2956				    v->gv_name, strerror(err));
2957				break;
2958
2959			default:
2960				bad_error(f, err);
2961			}
2962		}
2963	}
2964
2965	for (e = uu_list_first(v->gv_dependencies);
2966	    e != NULL;
2967	    e = uu_list_next(v->gv_dependencies, e)) {
2968		r = eval_subgraph(e->ge_vertex, h);
2969		if (r != 0) {
2970			assert(r == ECONNABORTED);
2971			ret = ECONNABORTED;
2972		}
2973	}
2974
2975	return (ret);
2976}
2977
2978/*
2979 * Delete the (property group) dependencies of v & create new ones based on
2980 * inst.  If doing so would create a cycle, log a message and put the instance
2981 * into maintenance.  Update GV_INSUBGRAPH flags as necessary.  Returns 0 or
2982 * ECONNABORTED.
2983 */
2984int
2985refresh_vertex(graph_vertex_t *v, scf_instance_t *inst)
2986{
2987	int err;
2988	int *path;
2989	char *fmri;
2990	int r;
2991	scf_handle_t *h = scf_instance_handle(inst);
2992	uu_list_t *old_deps;
2993	int ret = 0;
2994	graph_edge_t *e;
2995	graph_vertex_t *vv;
2996
2997	assert(MUTEX_HELD(&dgraph_lock));
2998	assert(v->gv_type == GVT_INST);
2999
3000	log_framework(LOG_DEBUG, "Graph engine: Refreshing %s.\n", v->gv_name);
3001
3002	if (milestone > MILESTONE_NONE) {
3003		/*
3004		 * In case some of v's dependencies are being deleted we must
3005		 * make a list of them now for GV_INSUBGRAPH-flag evaluation
3006		 * after the new dependencies are in place.
3007		 */
3008		old_deps = startd_list_create(graph_edge_pool, NULL, 0);
3009
3010		err = uu_list_walk(v->gv_dependencies,
3011		    (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
3012		assert(err == 0);
3013	}
3014
3015	delete_instance_dependencies(v, B_FALSE);
3016
3017	err = set_dependencies(v, inst, &path);
3018	switch (err) {
3019	case 0:
3020		break;
3021
3022	case ECONNABORTED:
3023		ret = err;
3024		goto out;
3025
3026	case EINVAL:
3027	case ELOOP:
3028		r = libscf_instance_get_fmri(inst, &fmri);
3029		switch (r) {
3030		case 0:
3031			break;
3032
3033		case ECONNABORTED:
3034			ret = ECONNABORTED;
3035			goto out;
3036
3037		case ECANCELED:
3038			ret = 0;
3039			goto out;
3040
3041		default:
3042			bad_error("libscf_instance_get_fmri", r);
3043		}
3044
3045		if (err == EINVAL) {
3046			log_error(LOG_ERR, "Transitioning %s "
3047			    "to maintenance due to misconfiguration.\n",
3048			    fmri ? fmri : "?");
3049			vertex_send_event(v,
3050			    RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY);
3051		} else {
3052			handle_cycle(fmri, path);
3053			vertex_send_event(v,
3054			    RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE);
3055		}
3056		startd_free(fmri, max_scf_fmri_size);
3057		ret = 0;
3058		goto out;
3059
3060	default:
3061		bad_error("set_dependencies", err);
3062	}
3063
3064	if (milestone > MILESTONE_NONE) {
3065		boolean_t aborted = B_FALSE;
3066
3067		for (e = uu_list_first(old_deps);
3068		    e != NULL;
3069		    e = uu_list_next(old_deps, e)) {
3070			vv = e->ge_vertex;
3071
3072			if (vertex_unref(vv) == VERTEX_INUSE &&
3073			    eval_subgraph(vv, h) == ECONNABORTED)
3074				aborted = B_TRUE;
3075		}
3076
3077		for (e = uu_list_first(v->gv_dependencies);
3078		    e != NULL;
3079		    e = uu_list_next(v->gv_dependencies, e)) {
3080			if (eval_subgraph(e->ge_vertex, h) ==
3081			    ECONNABORTED)
3082				aborted = B_TRUE;
3083		}
3084
3085		if (aborted) {
3086			ret = ECONNABORTED;
3087			goto out;
3088		}
3089	}
3090
3091	graph_start_if_satisfied(v);
3092
3093	ret = 0;
3094
3095out:
3096	if (milestone > MILESTONE_NONE) {
3097		void *cookie = NULL;
3098
3099		while ((e = uu_list_teardown(old_deps, &cookie)) != NULL)
3100			startd_free(e, sizeof (*e));
3101
3102		uu_list_destroy(old_deps);
3103	}
3104
3105	return (ret);
3106}
3107
3108/*
3109 * Set up v according to inst.  That is, make sure it depends on its
3110 * restarter and set up its dependencies.  Send the ADD_INSTANCE command to
3111 * the restarter, and send ENABLE or DISABLE as appropriate.
3112 *
3113 * Returns 0 on success, ECONNABORTED on repository disconnection, or
3114 * ECANCELED if inst is deleted.
3115 */
3116static int
3117configure_vertex(graph_vertex_t *v, scf_instance_t *inst)
3118{
3119	scf_handle_t *h;
3120	scf_propertygroup_t *pg;
3121	scf_snapshot_t *snap;
3122	char *restarter_fmri = startd_alloc(max_scf_value_size);
3123	int enabled, enabled_ovr;
3124	int err;
3125	int *path;
3126	int deathrow;
3127	int32_t tset;
3128
3129	restarter_fmri[0] = '\0';
3130
3131	assert(MUTEX_HELD(&dgraph_lock));
3132	assert(v->gv_type == GVT_INST);
3133	assert((v->gv_flags & GV_CONFIGURED) == 0);
3134
3135	/* GV_INSUBGRAPH should already be set properly. */
3136	assert(should_be_in_subgraph(v) ==
3137	    ((v->gv_flags & GV_INSUBGRAPH) != 0));
3138
3139	/*
3140	 * If the instance fmri is in the deathrow list then set the
3141	 * GV_DEATHROW flag on the vertex and create and set to true the
3142	 * SCF_PROPERTY_DEATHROW boolean property in the non-persistent
3143	 * repository for this instance fmri.
3144	 */
3145	if ((v->gv_flags & GV_DEATHROW) ||
3146	    (is_fmri_in_deathrow(v->gv_name) == B_TRUE)) {
3147		if ((v->gv_flags & GV_DEATHROW) == 0) {
3148			/*
3149			 * Set flag GV_DEATHROW, create and set to true
3150			 * the SCF_PROPERTY_DEATHROW property in the
3151			 * non-persistent repository for this instance fmri.
3152			 */
3153			v->gv_flags |= GV_DEATHROW;
3154
3155			switch (err = libscf_set_deathrow(inst, 1)) {
3156			case 0:
3157				break;
3158
3159			case ECONNABORTED:
3160			case ECANCELED:
3161				startd_free(restarter_fmri, max_scf_value_size);
3162				return (err);
3163
3164			case EROFS:
3165				log_error(LOG_WARNING, "Could not set %s/%s "
3166				    "for deathrow %s: %s.\n",
3167				    SCF_PG_DEATHROW, SCF_PROPERTY_DEATHROW,
3168				    v->gv_name, strerror(err));
3169				break;
3170
3171			case EPERM:
3172				uu_die("Permission denied.\n");
3173				/* NOTREACHED */
3174
3175			default:
3176				bad_error("libscf_set_deathrow", err);
3177			}
3178			log_framework(LOG_DEBUG, "Deathrow, graph set %s.\n",
3179			    v->gv_name);
3180		}
3181		startd_free(restarter_fmri, max_scf_value_size);
3182		return (0);
3183	}
3184
3185	h = scf_instance_handle(inst);
3186
3187	/*
3188	 * Using a temporary deathrow boolean property, set through
3189	 * libscf_set_deathrow(), only for fmris on deathrow, is necessary
3190	 * because deathrow_fini() may already have been called, and in case
3191	 * of a refresh, GV_DEATHROW may need to be set again.
3192	 * libscf_get_deathrow() sets deathrow to 1 only if this instance
3193	 * has a temporary boolean property named 'deathrow' valued true
3194	 * in a property group 'deathrow', -1 or 0 in all other cases.
3195	 */
3196	err = libscf_get_deathrow(h, inst, &deathrow);
3197	switch (err) {
3198	case 0:
3199		break;
3200
3201	case ECONNABORTED:
3202	case ECANCELED:
3203		startd_free(restarter_fmri, max_scf_value_size);
3204		return (err);
3205
3206	default:
3207		bad_error("libscf_get_deathrow", err);
3208	}
3209
3210	if (deathrow == 1) {
3211		v->gv_flags |= GV_DEATHROW;
3212		startd_free(restarter_fmri, max_scf_value_size);
3213		return (0);
3214	}
3215
3216	log_framework(LOG_DEBUG, "Graph adding %s.\n", v->gv_name);
3217
3218	/*
3219	 * If the instance does not have a restarter property group,
3220	 * initialize its state to uninitialized/none, in case the restarter
3221	 * is not enabled.
3222	 */
3223	pg = safe_scf_pg_create(h);
3224
3225	if (scf_instance_get_pg(inst, SCF_PG_RESTARTER, pg) != 0) {
3226		instance_data_t idata;
3227		uint_t count = 0, msecs = ALLOC_DELAY;
3228
3229		switch (scf_error()) {
3230		case SCF_ERROR_NOT_FOUND:
3231			break;
3232
3233		case SCF_ERROR_CONNECTION_BROKEN:
3234		default:
3235			scf_pg_destroy(pg);
3236			startd_free(restarter_fmri, max_scf_value_size);
3237			return (ECONNABORTED);
3238
3239		case SCF_ERROR_DELETED:
3240			scf_pg_destroy(pg);
3241			startd_free(restarter_fmri, max_scf_value_size);
3242			return (ECANCELED);
3243
3244		case SCF_ERROR_NOT_SET:
3245			bad_error("scf_instance_get_pg", scf_error());
3246		}
3247
3248		switch (err = libscf_instance_get_fmri(inst,
3249		    (char **)&idata.i_fmri)) {
3250		case 0:
3251			break;
3252
3253		case ECONNABORTED:
3254		case ECANCELED:
3255			scf_pg_destroy(pg);
3256			startd_free(restarter_fmri, max_scf_value_size);
3257			return (err);
3258
3259		default:
3260			bad_error("libscf_instance_get_fmri", err);
3261		}
3262
3263		idata.i_state = RESTARTER_STATE_NONE;
3264		idata.i_next_state = RESTARTER_STATE_NONE;
3265
3266init_state:
3267		switch (err = _restarter_commit_states(h, &idata,
3268		    RESTARTER_STATE_UNINIT, RESTARTER_STATE_NONE,
3269		    restarter_get_str_short(restarter_str_insert_in_graph))) {
3270		case 0:
3271			break;
3272
3273		case ENOMEM:
3274			++count;
3275			if (count < ALLOC_RETRY) {
3276				(void) poll(NULL, 0, msecs);
3277				msecs *= ALLOC_DELAY_MULT;
3278				goto init_state;
3279			}
3280
3281			uu_die("Insufficient memory.\n");
3282			/* NOTREACHED */
3283
3284		case ECONNABORTED:
3285			startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3286			scf_pg_destroy(pg);
3287			startd_free(restarter_fmri, max_scf_value_size);
3288			return (ECONNABORTED);
3289
3290		case ENOENT:
3291			startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3292			scf_pg_destroy(pg);
3293			startd_free(restarter_fmri, max_scf_value_size);
3294			return (ECANCELED);
3295
3296		case EPERM:
3297		case EACCES:
3298		case EROFS:
3299			log_error(LOG_NOTICE, "Could not initialize state for "
3300			    "%s: %s.\n", idata.i_fmri, strerror(err));
3301			break;
3302
3303		case EINVAL:
3304		default:
3305			bad_error("_restarter_commit_states", err);
3306		}
3307
3308		startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3309	}
3310
3311	scf_pg_destroy(pg);
3312
3313	if (milestone != NULL) {
3314		/*
3315		 * Make sure the enable-override is set properly before we
3316		 * read whether we should be enabled.
3317		 */
3318		if (milestone == MILESTONE_NONE ||
3319		    !(v->gv_flags & GV_INSUBGRAPH)) {
3320			/*
3321			 * This might seem unjustified after the milestone
3322			 * transition has completed (non_subgraph_svcs == 0),
3323			 * but it's important because when we boot to
3324			 * a milestone, we set the milestone before populating
3325			 * the graph, and all of the new non-subgraph services
3326			 * need to be disabled here.
3327			 */
3328			switch (err = libscf_set_enable_ovr(inst, 0)) {
3329			case 0:
3330				break;
3331
3332			case ECONNABORTED:
3333			case ECANCELED:
3334				startd_free(restarter_fmri, max_scf_value_size);
3335				return (err);
3336
3337			case EROFS:
3338				log_error(LOG_WARNING,
3339				    "Could not set %s/%s for %s: %s.\n",
3340				    SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
3341				    v->gv_name, strerror(err));
3342				break;
3343
3344			case EPERM:
3345				uu_die("Permission denied.\n");
3346				/* NOTREACHED */
3347
3348			default:
3349				bad_error("libscf_set_enable_ovr", err);
3350			}
3351		} else {
3352			assert(v->gv_flags & GV_INSUBGRAPH);
3353			switch (err = libscf_delete_enable_ovr(inst)) {
3354			case 0:
3355				break;
3356
3357			case ECONNABORTED:
3358			case ECANCELED:
3359				startd_free(restarter_fmri, max_scf_value_size);
3360				return (err);
3361
3362			case EPERM:
3363				uu_die("Permission denied.\n");
3364				/* NOTREACHED */
3365
3366			default:
3367				bad_error("libscf_delete_enable_ovr", err);
3368			}
3369		}
3370	}
3371
3372	err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
3373	    &enabled_ovr, &restarter_fmri);
3374	switch (err) {
3375	case 0:
3376		break;
3377
3378	case ECONNABORTED:
3379	case ECANCELED:
3380		startd_free(restarter_fmri, max_scf_value_size);
3381		return (err);
3382
3383	case ENOENT:
3384		log_framework(LOG_DEBUG,
3385		    "Ignoring %s because it has no general property group.\n",
3386		    v->gv_name);
3387		startd_free(restarter_fmri, max_scf_value_size);
3388		return (0);
3389
3390	default:
3391		bad_error("libscf_get_basic_instance_data", err);
3392	}
3393
3394	if ((tset = libscf_get_stn_tset(inst)) == -1) {
3395		log_framework(LOG_WARNING,
3396		    "Failed to get notification parameters for %s: %s\n",
3397		    v->gv_name, scf_strerror(scf_error()));
3398		v->gv_stn_tset = 0;
3399	} else {
3400		v->gv_stn_tset = tset;
3401	}
3402	if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
3403		stn_global = v->gv_stn_tset;
3404
3405	if (enabled == -1) {
3406		startd_free(restarter_fmri, max_scf_value_size);
3407		return (0);
3408	}
3409
3410	v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
3411	    (enabled ? GV_ENBLD_NOOVR : 0);
3412
3413	if (enabled_ovr != -1)
3414		enabled = enabled_ovr;
3415
3416	v->gv_state = RESTARTER_STATE_UNINIT;
3417
3418	snap = libscf_get_or_make_running_snapshot(inst, v->gv_name, B_TRUE);
3419	scf_snapshot_destroy(snap);
3420
3421	/* Set up the restarter. (Sends _ADD_INSTANCE on success.) */
3422	err = graph_change_restarter(v, restarter_fmri, h, &path);
3423	if (err != 0) {
3424		instance_data_t idata;
3425		uint_t count = 0, msecs = ALLOC_DELAY;
3426		restarter_str_t reason;
3427
3428		if (err == ECONNABORTED) {
3429			startd_free(restarter_fmri, max_scf_value_size);
3430			return (err);
3431		}
3432
3433		assert(err == EINVAL || err == ELOOP);
3434
3435		if (err == EINVAL) {
3436			log_framework(LOG_ERR, emsg_invalid_restarter,
3437			    v->gv_name, restarter_fmri);
3438			reason = restarter_str_invalid_restarter;
3439		} else {
3440			handle_cycle(v->gv_name, path);
3441			reason = restarter_str_dependency_cycle;
3442		}
3443
3444		startd_free(restarter_fmri, max_scf_value_size);
3445
3446		/*
3447		 * We didn't register the instance with the restarter, so we
3448		 * must set maintenance mode ourselves.
3449		 */
3450		err = libscf_instance_get_fmri(inst, (char **)&idata.i_fmri);
3451		if (err != 0) {
3452			assert(err == ECONNABORTED || err == ECANCELED);
3453			return (err);
3454		}
3455
3456		idata.i_state = RESTARTER_STATE_NONE;
3457		idata.i_next_state = RESTARTER_STATE_NONE;
3458
3459set_maint:
3460		switch (err = _restarter_commit_states(h, &idata,
3461		    RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE,
3462		    restarter_get_str_short(reason))) {
3463		case 0:
3464			break;
3465
3466		case ENOMEM:
3467			++count;
3468			if (count < ALLOC_RETRY) {
3469				(void) poll(NULL, 0, msecs);
3470				msecs *= ALLOC_DELAY_MULT;
3471				goto set_maint;
3472			}
3473
3474			uu_die("Insufficient memory.\n");
3475			/* NOTREACHED */
3476
3477		case ECONNABORTED:
3478			startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3479			return (ECONNABORTED);
3480
3481		case ENOENT:
3482			startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3483			return (ECANCELED);
3484
3485		case EPERM:
3486		case EACCES:
3487		case EROFS:
3488			log_error(LOG_NOTICE, "Could not initialize state for "
3489			    "%s: %s.\n", idata.i_fmri, strerror(err));
3490			break;
3491
3492		case EINVAL:
3493		default:
3494			bad_error("_restarter_commit_states", err);
3495		}
3496
3497		startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3498
3499		v->gv_state = RESTARTER_STATE_MAINT;
3500
3501		goto out;
3502	}
3503	startd_free(restarter_fmri, max_scf_value_size);
3504
3505	/* Add all the other dependencies. */
3506	err = refresh_vertex(v, inst);
3507	if (err != 0) {
3508		assert(err == ECONNABORTED);
3509		return (err);
3510	}
3511
3512out:
3513	v->gv_flags |= GV_CONFIGURED;
3514
3515	graph_enable_by_vertex(v, enabled, 0);
3516
3517	return (0);
3518}
3519
3520
3521static void
3522kill_user_procs(void)
3523{
3524	(void) fputs("svc.startd: Killing user processes.\n", stdout);
3525
3526	/*
3527	 * Despite its name, killall's role is to get select user processes--
3528	 * basically those representing terminal-based logins-- to die.  Victims
3529	 * are located by killall in the utmp database.  Since these are most
3530	 * often shell based logins, and many shells mask SIGTERM (but are
3531	 * responsive to SIGHUP) we first HUP and then shortly thereafter
3532	 * kill -9.
3533	 */
3534	(void) fork_with_timeout("/usr/sbin/killall HUP", 1, 5);
3535	(void) fork_with_timeout("/usr/sbin/killall KILL", 1, 5);
3536
3537	/*
3538	 * Note the selection of user id's 0, 1 and 15, subsequently
3539	 * inverted by -v.  15 is reserved for dladmd.  Yes, this is a
3540	 * kludge-- a better policy is needed.
3541	 *
3542	 * Note that fork_with_timeout will only wait out the 1 second
3543	 * "grace time" if pkill actually returns 0.  So if there are
3544	 * no matches, this will run to completion much more quickly.
3545	 */
3546	(void) fork_with_timeout("/usr/bin/pkill -TERM -v -u 0,1,15", 1, 5);
3547	(void) fork_with_timeout("/usr/bin/pkill -KILL -v -u 0,1,15", 1, 5);
3548}
3549
3550static void
3551do_uadmin(void)
3552{
3553	const char * const resetting = "/etc/svc/volatile/resetting";
3554	int fd;
3555	struct statvfs vfs;
3556	time_t now;
3557	struct tm nowtm;
3558	char down_buf[256], time_buf[256];
3559	uintptr_t mdep;
3560#if defined(__x86)
3561	char *fbarg = NULL;
3562#endif	/* __x86 */
3563
3564	mdep = 0;
3565	fd = creat(resetting, 0777);
3566	if (fd >= 0)
3567		startd_close(fd);
3568	else
3569		uu_warn("Could not create \"%s\"", resetting);
3570
3571	/* Kill dhcpagent if we're not using nfs for root */
3572	if ((statvfs("/", &vfs) == 0) &&
3573	    (strncmp(vfs.f_basetype, "nfs", sizeof ("nfs") - 1) != 0))
3574		fork_with_timeout("/usr/bin/pkill -x -u 0 dhcpagent", 0, 5);
3575
3576	/*
3577	 * Call sync(2) now, before we kill off user processes.  This takes
3578	 * advantage of the several seconds of pause we have before the
3579	 * killalls are done.  Time we can make good use of to get pages
3580	 * moving out to disk.
3581	 *
3582	 * Inside non-global zones, we don't bother, and it's better not to
3583	 * anyway, since sync(2) can have system-wide impact.
3584	 */
3585	if (getzoneid() == 0)
3586		sync();
3587
3588	kill_user_procs();
3589
3590	/*
3591	 * Note that this must come after the killing of user procs, since
3592	 * killall relies on utmpx, and this command affects the contents of
3593	 * said file.
3594	 */
3595	if (access("/usr/lib/acct/closewtmp", X_OK) == 0)
3596		fork_with_timeout("/usr/lib/acct/closewtmp", 0, 5);
3597
3598	/*
3599	 * For patches which may be installed as the system is shutting
3600	 * down, we need to ensure, one more time, that the boot archive
3601	 * really is up to date.
3602	 */
3603	if (getzoneid() == 0 && access("/usr/sbin/bootadm", X_OK) == 0)
3604		fork_with_timeout("/usr/sbin/bootadm -ea update_all", 0, 3600);
3605
3606	/*
3607	 * Right now, fast reboot is supported only on i386.
3608	 * scf_is_fastboot_default() should take care of it.
3609	 * If somehow we got there on unsupported platform -
3610	 * print warning and fall back to regular reboot.
3611	 */
3612	if (halting == AD_FASTREBOOT) {
3613#if defined(__x86)
3614		if (be_get_boot_args(&fbarg, BE_ENTRY_DEFAULT) == 0) {
3615			mdep = (uintptr_t)fbarg;
3616		} else {
3617			/*
3618			 * Failed to read BE info, fall back to normal reboot
3619			 */
3620			halting = AD_BOOT;
3621			uu_warn("Failed to get fast reboot arguments.\n"
3622			    "Falling back to regular reboot.\n");
3623		}
3624#else	/* __x86 */
3625		halting = AD_BOOT;
3626		uu_warn("Fast reboot configured, but not supported by "
3627		    "this ISA\n");
3628#endif	/* __x86 */
3629	}
3630
3631	fork_with_timeout("/sbin/umountall -l", 0, 5);
3632	fork_with_timeout("/sbin/umount /tmp /var/adm /var/run /var "
3633	    ">/dev/null 2>&1", 0, 5);
3634
3635	/*
3636	 * Try to get to consistency for whatever UFS filesystems are left.
3637	 * This is pretty expensive, so we save it for the end in the hopes of
3638	 * minimizing what it must do.  The other option would be to start in
3639	 * parallel with the killall's, but lockfs tends to throw out much more
3640	 * than is needed, and so subsequent commands (like umountall) take a
3641	 * long time to get going again.
3642	 *
3643	 * Inside of zones, we don't bother, since we're not about to terminate
3644	 * the whole OS instance.
3645	 *
3646	 * On systems using only ZFS, this call to lockfs -fa is a no-op.
3647	 */
3648	if (getzoneid() == 0) {
3649		if (access("/usr/sbin/lockfs", X_OK) == 0)
3650			fork_with_timeout("/usr/sbin/lockfs -fa", 0, 30);
3651
3652		sync();	/* once more, with feeling */
3653	}
3654
3655	fork_with_timeout("/sbin/umount /usr >/dev/null 2>&1", 0, 5);
3656
3657	/*
3658	 * Construct and emit the last words from userland:
3659	 * "<timestamp> The system is down.  Shutdown took <N> seconds."
3660	 *
3661	 * Normally we'd use syslog, but with /var and other things
3662	 * potentially gone, try to minimize the external dependencies.
3663	 */
3664	now = time(NULL);
3665	(void) localtime_r(&now, &nowtm);
3666
3667	if (strftime(down_buf, sizeof (down_buf),
3668	    "%b %e %T The system is down.", &nowtm) == 0) {
3669		(void) strlcpy(down_buf, "The system is down.",
3670		    sizeof (down_buf));
3671	}
3672
3673	if (halting_time != 0 && halting_time <= now) {
3674		(void) snprintf(time_buf, sizeof (time_buf),
3675		    "  Shutdown took %lu seconds.", now - halting_time);
3676	} else {
3677		time_buf[0] = '\0';
3678	}
3679	(void) printf("%s%s\n", down_buf, time_buf);
3680
3681	(void) uadmin(A_SHUTDOWN, halting, mdep);
3682	uu_warn("uadmin() failed");
3683
3684#if defined(__x86)
3685	if (halting == AD_FASTREBOOT)
3686		free(fbarg);
3687#endif	/* __x86 */
3688
3689	if (remove(resetting) != 0 && errno != ENOENT)
3690		uu_warn("Could not remove \"%s\"", resetting);
3691}
3692
3693/*
3694 * If any of the up_svcs[] are online or satisfiable, return true.  If they are
3695 * all missing, disabled, in maintenance, or unsatisfiable, return false.
3696 */
3697boolean_t
3698can_come_up(void)
3699{
3700	int i;
3701
3702	assert(MUTEX_HELD(&dgraph_lock));
3703
3704	/*
3705	 * If we are booting to single user (boot -s),
3706	 * SCF_MILESTONE_SINGLE_USER is needed to come up because startd
3707	 * spawns sulogin after single-user is online (see specials.c).
3708	 */
3709	i = (booting_to_single_user ? 0 : 1);
3710
3711	for (; up_svcs[i] != NULL; ++i) {
3712		if (up_svcs_p[i] == NULL) {
3713			up_svcs_p[i] = vertex_get_by_name(up_svcs[i]);
3714
3715			if (up_svcs_p[i] == NULL)
3716				continue;
3717		}
3718
3719		/*
3720		 * Ignore unconfigured services (the ones that have been
3721		 * mentioned in a dependency from other services, but do
3722		 * not exist in the repository).  Services which exist
3723		 * in the repository but don't have general/enabled
3724		 * property will be also ignored.
3725		 */
3726		if (!(up_svcs_p[i]->gv_flags & GV_CONFIGURED))
3727			continue;
3728
3729		switch (up_svcs_p[i]->gv_state) {
3730		case RESTARTER_STATE_ONLINE:
3731		case RESTARTER_STATE_DEGRADED:
3732			/*
3733			 * Deactivate verbose boot once a login service has been
3734			 * reached.
3735			 */
3736			st->st_log_login_reached = 1;
3737			/*FALLTHROUGH*/
3738		case RESTARTER_STATE_UNINIT:
3739			return (B_TRUE);
3740
3741		case RESTARTER_STATE_OFFLINE:
3742			if (instance_satisfied(up_svcs_p[i], B_TRUE) != -1)
3743				return (B_TRUE);
3744			log_framework(LOG_DEBUG,
3745			    "can_come_up(): %s is unsatisfiable.\n",
3746			    up_svcs_p[i]->gv_name);
3747			continue;
3748
3749		case RESTARTER_STATE_DISABLED:
3750		case RESTARTER_STATE_MAINT:
3751			log_framework(LOG_DEBUG,
3752			    "can_come_up(): %s is in state %s.\n",
3753			    up_svcs_p[i]->gv_name,
3754			    instance_state_str[up_svcs_p[i]->gv_state]);
3755			continue;
3756
3757		default:
3758#ifndef NDEBUG
3759			uu_warn("%s:%d: Unexpected vertex state %d.\n",
3760			    __FILE__, __LINE__, up_svcs_p[i]->gv_state);
3761#endif
3762			abort();
3763		}
3764	}
3765
3766	/*
3767	 * In the seed repository, console-login is unsatisfiable because
3768	 * services are missing.  To behave correctly in that case we don't want
3769	 * to return false until manifest-import is online.
3770	 */
3771
3772	if (manifest_import_p == NULL) {
3773		manifest_import_p = vertex_get_by_name(manifest_import);
3774
3775		if (manifest_import_p == NULL)
3776			return (B_FALSE);
3777	}
3778
3779	switch (manifest_import_p->gv_state) {
3780	case RESTARTER_STATE_ONLINE:
3781	case RESTARTER_STATE_DEGRADED:
3782	case RESTARTER_STATE_DISABLED:
3783	case RESTARTER_STATE_MAINT:
3784		break;
3785
3786	case RESTARTER_STATE_OFFLINE:
3787		if (instance_satisfied(manifest_import_p, B_TRUE) == -1)
3788			break;
3789		/* FALLTHROUGH */
3790
3791	case RESTARTER_STATE_UNINIT:
3792		return (B_TRUE);
3793	}
3794
3795	return (B_FALSE);
3796}
3797
3798/*
3799 * Runs sulogin.  Returns
3800 *   0 - success
3801 *   EALREADY - sulogin is already running
3802 *   EBUSY - console-login is running
3803 */
3804static int
3805run_sulogin(const char *msg)
3806{
3807	graph_vertex_t *v;
3808
3809	assert(MUTEX_HELD(&dgraph_lock));
3810
3811	if (sulogin_running)
3812		return (EALREADY);
3813
3814	v = vertex_get_by_name(console_login_fmri);
3815	if (v != NULL && inst_running(v))
3816		return (EBUSY);
3817
3818	sulogin_running = B_TRUE;
3819
3820	MUTEX_UNLOCK(&dgraph_lock);
3821
3822	fork_sulogin(B_FALSE, msg);
3823
3824	MUTEX_LOCK(&dgraph_lock);
3825
3826	sulogin_running = B_FALSE;
3827
3828	if (console_login_ready) {
3829		v = vertex_get_by_name(console_login_fmri);
3830
3831		if (v != NULL && v->gv_state == RESTARTER_STATE_OFFLINE) {
3832			if (v->gv_start_f == NULL)
3833				vertex_send_event(v,
3834				    RESTARTER_EVENT_TYPE_START);
3835			else
3836				v->gv_start_f(v);
3837		}
3838
3839		console_login_ready = B_FALSE;
3840	}
3841
3842	return (0);
3843}
3844
3845/*
3846 * The sulogin thread runs sulogin while can_come_up() is false.  run_sulogin()
3847 * keeps sulogin from stepping on console-login's toes.
3848 */
3849/* ARGSUSED */
3850static void *
3851sulogin_thread(void *unused)
3852{
3853	(void) pthread_setname_np(pthread_self(), "sulogin");
3854
3855	MUTEX_LOCK(&dgraph_lock);
3856
3857	assert(sulogin_thread_running);
3858
3859	do {
3860		(void) run_sulogin("Console login service(s) cannot run\n");
3861	} while (!can_come_up());
3862
3863	sulogin_thread_running = B_FALSE;
3864	MUTEX_UNLOCK(&dgraph_lock);
3865
3866	return (NULL);
3867}
3868
3869/* ARGSUSED */
3870void *
3871single_user_thread(void *unused)
3872{
3873	uint_t left;
3874	scf_handle_t *h;
3875	scf_instance_t *inst;
3876	scf_property_t *prop;
3877	scf_value_t *val;
3878	const char *msg;
3879	char *buf;
3880	int r;
3881
3882	(void) pthread_setname_np(pthread_self(), "single_user");
3883
3884	MUTEX_LOCK(&single_user_thread_lock);
3885	single_user_thread_count++;
3886
3887	if (!booting_to_single_user)
3888		kill_user_procs();
3889
3890	if (go_single_user_mode || booting_to_single_user) {
3891		msg = "SINGLE USER MODE\n";
3892	} else {
3893		assert(go_to_level1);
3894
3895		fork_rc_script('1', "start", B_TRUE);
3896
3897		uu_warn("The system is ready for administration.\n");
3898
3899		msg = "";
3900	}
3901
3902	MUTEX_UNLOCK(&single_user_thread_lock);
3903
3904	for (;;) {
3905		MUTEX_LOCK(&dgraph_lock);
3906		r = run_sulogin(msg);
3907		MUTEX_UNLOCK(&dgraph_lock);
3908		if (r == 0)
3909			break;
3910
3911		assert(r == EALREADY || r == EBUSY);
3912
3913		left = 3;
3914		while (left > 0)
3915			left = sleep(left);
3916	}
3917
3918	MUTEX_LOCK(&single_user_thread_lock);
3919
3920	/*
3921	 * If another single user thread has started, let it finish changing
3922	 * the run level.
3923	 */
3924	if (single_user_thread_count > 1) {
3925		single_user_thread_count--;
3926		MUTEX_UNLOCK(&single_user_thread_lock);
3927		return (NULL);
3928	}
3929
3930	h = libscf_handle_create_bound_loop();
3931	inst = scf_instance_create(h);
3932	prop = safe_scf_property_create(h);
3933	val = safe_scf_value_create(h);
3934	buf = startd_alloc(max_scf_fmri_size);
3935
3936lookup:
3937	if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
3938	    NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
3939		switch (scf_error()) {
3940		case SCF_ERROR_NOT_FOUND:
3941			r = libscf_create_self(h);
3942			if (r == 0)
3943				goto lookup;
3944			assert(r == ECONNABORTED);
3945			/* FALLTHROUGH */
3946
3947		case SCF_ERROR_CONNECTION_BROKEN:
3948			libscf_handle_rebind(h);
3949			goto lookup;
3950
3951		case SCF_ERROR_INVALID_ARGUMENT:
3952		case SCF_ERROR_CONSTRAINT_VIOLATED:
3953		case SCF_ERROR_NOT_BOUND:
3954		case SCF_ERROR_HANDLE_MISMATCH:
3955		default:
3956			bad_error("scf_handle_decode_fmri", scf_error());
3957		}
3958	}
3959
3960	MUTEX_LOCK(&dgraph_lock);
3961
3962	r = scf_instance_delete_prop(inst, SCF_PG_OPTIONS_OVR,
3963	    SCF_PROPERTY_MILESTONE);
3964	switch (r) {
3965	case 0:
3966	case ECANCELED:
3967		break;
3968
3969	case ECONNABORTED:
3970		MUTEX_UNLOCK(&dgraph_lock);
3971		libscf_handle_rebind(h);
3972		goto lookup;
3973
3974	case EPERM:
3975	case EACCES:
3976	case EROFS:
3977		log_error(LOG_WARNING, "Could not clear temporary milestone: "
3978		    "%s.\n", strerror(r));
3979		break;
3980
3981	default:
3982		bad_error("scf_instance_delete_prop", r);
3983	}
3984
3985	MUTEX_UNLOCK(&dgraph_lock);
3986
3987	r = libscf_get_milestone(inst, prop, val, buf, max_scf_fmri_size);
3988	switch (r) {
3989	case ECANCELED:
3990	case ENOENT:
3991	case EINVAL:
3992		(void) strcpy(buf, "all");
3993		/* FALLTHROUGH */
3994
3995	case 0:
3996		uu_warn("Returning to milestone %s.\n", buf);
3997		break;
3998
3999	case ECONNABORTED:
4000		libscf_handle_rebind(h);
4001		goto lookup;
4002
4003	default:
4004		bad_error("libscf_get_milestone", r);
4005	}
4006
4007	r = dgraph_set_milestone(buf, h, B_FALSE);
4008	switch (r) {
4009	case 0:
4010	case ECONNRESET:
4011	case EALREADY:
4012	case EINVAL:
4013	case ENOENT:
4014		break;
4015
4016	default:
4017		bad_error("dgraph_set_milestone", r);
4018	}
4019
4020	/*
4021	 * See graph_runlevel_changed().
4022	 */
4023	MUTEX_LOCK(&dgraph_lock);
4024	utmpx_set_runlevel(target_milestone_as_runlevel(), 'S', B_TRUE);
4025	MUTEX_UNLOCK(&dgraph_lock);
4026
4027	startd_free(buf, max_scf_fmri_size);
4028	scf_value_destroy(val);
4029	scf_property_destroy(prop);
4030	scf_instance_destroy(inst);
4031	scf_handle_destroy(h);
4032
4033	/*
4034	 * We'll give ourselves 3 seconds to respond to all of the enablings
4035	 * that setting the milestone should have created before checking
4036	 * whether to run sulogin.
4037	 */
4038	left = 3;
4039	while (left > 0)
4040		left = sleep(left);
4041
4042	MUTEX_LOCK(&dgraph_lock);
4043	/*
4044	 * Clearing these variables will allow the sulogin thread to run.  We
4045	 * check here in case there aren't any more state updates anytime soon.
4046	 */
4047	go_to_level1 = go_single_user_mode = booting_to_single_user = B_FALSE;
4048	if (!sulogin_thread_running && !can_come_up()) {
4049		(void) startd_thread_create(sulogin_thread, NULL);
4050		sulogin_thread_running = B_TRUE;
4051	}
4052	MUTEX_UNLOCK(&dgraph_lock);
4053	single_user_thread_count--;
4054	MUTEX_UNLOCK(&single_user_thread_lock);
4055	return (NULL);
4056}
4057
4058
4059/*
4060 * Dependency graph operations API.  These are handle-independent thread-safe
4061 * graph manipulation functions which are the entry points for the event
4062 * threads below.
4063 */
4064
4065/*
4066 * If a configured vertex exists for inst_fmri, return EEXIST.  If no vertex
4067 * exists for inst_fmri, add one.  Then fetch the restarter from inst, make
4068 * this vertex dependent on it, and send _ADD_INSTANCE to the restarter.
4069 * Fetch whether the instance should be enabled from inst and send _ENABLE or
4070 * _DISABLE as appropriate.  Finally rummage through inst's dependency
4071 * property groups and add vertices and edges as appropriate.  If anything
4072 * goes wrong after sending _ADD_INSTANCE, send _ADMIN_MAINT_ON to put the
4073 * instance in maintenance.  Don't send _START or _STOP until we get a state
4074 * update in case we're being restarted and the service is already running.
4075 *
4076 * To support booting to a milestone, we must also make sure all dependencies
4077 * encountered are configured, if they exist in the repository.
4078 *
4079 * Returns 0 on success, ECONNABORTED on repository disconnection, EINVAL if
4080 * inst_fmri is an invalid (or not canonical) FMRI, ECANCELED if inst is
4081 * deleted, or EEXIST if a configured vertex for inst_fmri already exists.
4082 */
4083int
4084dgraph_add_instance(const char *inst_fmri, scf_instance_t *inst,
4085    boolean_t lock_graph)
4086{
4087	graph_vertex_t *v;
4088	int err;
4089
4090	if (strcmp(inst_fmri, SCF_SERVICE_STARTD) == 0)
4091		return (0);
4092
4093	/* Check for a vertex for inst_fmri. */
4094	if (lock_graph) {
4095		MUTEX_LOCK(&dgraph_lock);
4096	} else {
4097		assert(MUTEX_HELD(&dgraph_lock));
4098	}
4099
4100	v = vertex_get_by_name(inst_fmri);
4101
4102	if (v != NULL) {
4103		assert(v->gv_type == GVT_INST);
4104
4105		if (v->gv_flags & GV_CONFIGURED) {
4106			if (lock_graph)
4107				MUTEX_UNLOCK(&dgraph_lock);
4108			return (EEXIST);
4109		}
4110	} else {
4111		/* Add the vertex. */
4112		err = graph_insert_vertex_unconfigured(inst_fmri, GVT_INST, 0,
4113		    RERR_NONE, &v);
4114		if (err != 0) {
4115			assert(err == EINVAL);
4116			if (lock_graph)
4117				MUTEX_UNLOCK(&dgraph_lock);
4118			return (EINVAL);
4119		}
4120	}
4121
4122	err = configure_vertex(v, inst);
4123
4124	if (lock_graph)
4125		MUTEX_UNLOCK(&dgraph_lock);
4126
4127	return (err);
4128}
4129
4130/*
4131 * Locate the vertex for this property group's instance.  If it doesn't exist
4132 * or is unconfigured, call dgraph_add_instance() & return.  Otherwise fetch
4133 * the restarter for the instance, and if it has changed, send
4134 * _REMOVE_INSTANCE to the old restarter, remove the dependency, make sure the
4135 * new restarter has a vertex, add a new dependency, and send _ADD_INSTANCE to
4136 * the new restarter.  Then fetch whether the instance should be enabled, and
4137 * if it is different from what we had, or if we changed the restarter, send
4138 * the appropriate _ENABLE or _DISABLE command.
4139 *
4140 * Returns 0 on success, ENOTSUP if the pg's parent is not an instance,
4141 * ECONNABORTED on repository disconnection, ECANCELED if the instance is
4142 * deleted, or -1 if the instance's general property group is deleted or if
4143 * its enabled property is misconfigured.
4144 */
4145static int
4146dgraph_update_general(scf_propertygroup_t *pg)
4147{
4148	scf_handle_t *h;
4149	scf_instance_t *inst;
4150	char *fmri;
4151	char *restarter_fmri;
4152	graph_vertex_t *v;
4153	int err;
4154	int enabled, enabled_ovr;
4155	int oldflags;
4156
4157	/* Find the vertex for this service */
4158	h = scf_pg_handle(pg);
4159
4160	inst = safe_scf_instance_create(h);
4161
4162	if (scf_pg_get_parent_instance(pg, inst) != 0) {
4163		switch (scf_error()) {
4164		case SCF_ERROR_CONSTRAINT_VIOLATED:
4165			return (ENOTSUP);
4166
4167		case SCF_ERROR_CONNECTION_BROKEN:
4168		default:
4169			return (ECONNABORTED);
4170
4171		case SCF_ERROR_DELETED:
4172			return (0);
4173
4174		case SCF_ERROR_NOT_SET:
4175			bad_error("scf_pg_get_parent_instance", scf_error());
4176		}
4177	}
4178
4179	err = libscf_instance_get_fmri(inst, &fmri);
4180	switch (err) {
4181	case 0:
4182		break;
4183
4184	case ECONNABORTED:
4185		scf_instance_destroy(inst);
4186		return (ECONNABORTED);
4187
4188	case ECANCELED:
4189		scf_instance_destroy(inst);
4190		return (0);
4191
4192	default:
4193		bad_error("libscf_instance_get_fmri", err);
4194	}
4195
4196	log_framework(LOG_DEBUG,
4197	    "Graph engine: Reloading general properties for %s.\n", fmri);
4198
4199	MUTEX_LOCK(&dgraph_lock);
4200
4201	v = vertex_get_by_name(fmri);
4202	if (v == NULL || !(v->gv_flags & GV_CONFIGURED)) {
4203		/* Will get the up-to-date properties. */
4204		MUTEX_UNLOCK(&dgraph_lock);
4205		err = dgraph_add_instance(fmri, inst, B_TRUE);
4206		startd_free(fmri, max_scf_fmri_size);
4207		scf_instance_destroy(inst);
4208		return (err == ECANCELED ? 0 : err);
4209	}
4210
4211	/* Read enabled & restarter from repository. */
4212	restarter_fmri = startd_alloc(max_scf_value_size);
4213	err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
4214	    &enabled_ovr, &restarter_fmri);
4215	if (err != 0 || enabled == -1) {
4216		MUTEX_UNLOCK(&dgraph_lock);
4217		scf_instance_destroy(inst);
4218		startd_free(fmri, max_scf_fmri_size);
4219
4220		switch (err) {
4221		case ENOENT:
4222		case 0:
4223			startd_free(restarter_fmri, max_scf_value_size);
4224			return (-1);
4225
4226		case ECONNABORTED:
4227		case ECANCELED:
4228			startd_free(restarter_fmri, max_scf_value_size);
4229			return (err);
4230
4231		default:
4232			bad_error("libscf_get_basic_instance_data", err);
4233		}
4234	}
4235
4236	oldflags = v->gv_flags;
4237	v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
4238	    (enabled ? GV_ENBLD_NOOVR : 0);
4239
4240	if (enabled_ovr != -1)
4241		enabled = enabled_ovr;
4242
4243	/*
4244	 * If GV_ENBLD_NOOVR has changed, then we need to re-evaluate the
4245	 * subgraph.
4246	 */
4247	if (milestone > MILESTONE_NONE && v->gv_flags != oldflags)
4248		(void) eval_subgraph(v, h);
4249
4250	scf_instance_destroy(inst);
4251
4252	/* Ignore restarter change for now. */
4253
4254	startd_free(restarter_fmri, max_scf_value_size);
4255	startd_free(fmri, max_scf_fmri_size);
4256
4257	/*
4258	 * Always send _ENABLE or _DISABLE.  We could avoid this if the
4259	 * restarter didn't change and the enabled value didn't change, but
4260	 * that's not easy to check and improbable anyway, so we'll just do
4261	 * this.
4262	 */
4263	graph_enable_by_vertex(v, enabled, 1);
4264
4265	MUTEX_UNLOCK(&dgraph_lock);
4266
4267	return (0);
4268}
4269
4270/*
4271 * Delete all of the property group dependencies of v, update inst's running
4272 * snapshot, and add the dependencies in the new snapshot.  If any of the new
4273 * dependencies would create a cycle, send _ADMIN_MAINT_ON.  Otherwise
4274 * reevaluate v's dependencies, send _START or _STOP as appropriate, and do
4275 * the same for v's dependents.
4276 *
4277 * Returns
4278 *   0 - success
4279 *   ECONNABORTED - repository connection broken
4280 *   ECANCELED - inst was deleted
4281 *   EINVAL - inst is invalid (e.g., missing general/enabled)
4282 *   -1 - libscf_snapshots_refresh() failed
4283 */
4284static int
4285dgraph_refresh_instance(graph_vertex_t *v, scf_instance_t *inst)
4286{
4287	int r;
4288	int enabled;
4289	int32_t tset;
4290
4291	assert(MUTEX_HELD(&dgraph_lock));
4292	assert(v->gv_type == GVT_INST);
4293
4294	/* Only refresh services with valid general/enabled properties. */
4295	r = libscf_get_basic_instance_data(scf_instance_handle(inst), inst,
4296	    v->gv_name, &enabled, NULL, NULL);
4297	switch (r) {
4298	case 0:
4299		break;
4300
4301	case ECONNABORTED:
4302	case ECANCELED:
4303		return (r);
4304
4305	case ENOENT:
4306		log_framework(LOG_DEBUG,
4307		    "Ignoring %s because it has no general property group.\n",
4308		    v->gv_name);
4309		return (EINVAL);
4310
4311	default:
4312		bad_error("libscf_get_basic_instance_data", r);
4313	}
4314
4315	if ((tset = libscf_get_stn_tset(inst)) == -1) {
4316		log_framework(LOG_WARNING,
4317		    "Failed to get notification parameters for %s: %s\n",
4318		    v->gv_name, scf_strerror(scf_error()));
4319		tset = 0;
4320	}
4321	v->gv_stn_tset = tset;
4322	if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
4323		stn_global = tset;
4324
4325	if (enabled == -1)
4326		return (EINVAL);
4327
4328	r = libscf_snapshots_refresh(inst, v->gv_name);
4329	if (r != 0) {
4330		if (r != -1)
4331			bad_error("libscf_snapshots_refresh", r);
4332
4333		/* error logged */
4334		return (r);
4335	}
4336
4337	r = refresh_vertex(v, inst);
4338	if (r != 0 && r != ECONNABORTED)
4339		bad_error("refresh_vertex", r);
4340	return (r);
4341}
4342
4343/*
4344 * Returns true only if none of this service's dependents are 'up' -- online
4345 * or degraded (offline is considered down in this situation). This function
4346 * is somehow similar to is_nonsubgraph_leaf() but works on subtrees.
4347 */
4348static boolean_t
4349insubtree_dependents_down(graph_vertex_t *v)
4350{
4351	graph_vertex_t *vv;
4352	graph_edge_t *e;
4353
4354	assert(MUTEX_HELD(&dgraph_lock));
4355
4356	for (e = uu_list_first(v->gv_dependents); e != NULL;
4357	    e = uu_list_next(v->gv_dependents, e)) {
4358		vv = e->ge_vertex;
4359		if (vv->gv_type == GVT_INST) {
4360			if ((vv->gv_flags & GV_CONFIGURED) == 0)
4361				continue;
4362
4363			if ((vv->gv_flags & GV_TOOFFLINE) == 0)
4364				continue;
4365
4366			if ((vv->gv_state == RESTARTER_STATE_ONLINE) ||
4367			    (vv->gv_state == RESTARTER_STATE_DEGRADED))
4368				return (B_FALSE);
4369		} else {
4370			/*
4371			 * Skip all excluded dependents and decide whether
4372			 * to offline the service based on the restart_on
4373			 * attribute.
4374			 */
4375			if (is_depgrp_bypassed(vv))
4376				continue;
4377
4378			/*
4379			 * For dependency groups or service vertices, keep
4380			 * traversing to see if instances are running.
4381			 */
4382			if (insubtree_dependents_down(vv) == B_FALSE)
4383				return (B_FALSE);
4384		}
4385	}
4386
4387	return (B_TRUE);
4388}
4389
4390/*
4391 * Returns true only if none of this service's dependents are 'up' -- online,
4392 * degraded, or offline.
4393 */
4394static int
4395is_nonsubgraph_leaf(graph_vertex_t *v)
4396{
4397	graph_vertex_t *vv;
4398	graph_edge_t *e;
4399
4400	assert(MUTEX_HELD(&dgraph_lock));
4401
4402	for (e = uu_list_first(v->gv_dependents);
4403	    e != NULL;
4404	    e = uu_list_next(v->gv_dependents, e)) {
4405
4406		vv = e->ge_vertex;
4407		if (vv->gv_type == GVT_INST) {
4408			if ((vv->gv_flags & GV_CONFIGURED) == 0)
4409				continue;
4410
4411			if (vv->gv_flags & GV_INSUBGRAPH)
4412				continue;
4413
4414			if (up_state(vv->gv_state))
4415				return (0);
4416		} else {
4417			/*
4418			 * For dependency group or service vertices, keep
4419			 * traversing to see if instances are running.
4420			 *
4421			 * We should skip exclude_all dependencies otherwise
4422			 * the vertex will never be considered as a leaf
4423			 * if the dependent is offline. The main reason for
4424			 * this is that disable_nonsubgraph_leaves() skips
4425			 * exclusion dependencies.
4426			 */
4427			if (vv->gv_type == GVT_GROUP &&
4428			    vv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4429				continue;
4430
4431			if (!is_nonsubgraph_leaf(vv))
4432				return (0);
4433		}
4434	}
4435
4436	return (1);
4437}
4438
4439/*
4440 * Disable v temporarily.  Attempt to do this by setting its enabled override
4441 * property in the repository.  If that fails, send a _DISABLE command.
4442 * Returns 0 on success and ECONNABORTED if the repository connection is
4443 * broken.
4444 */
4445static int
4446disable_service_temporarily(graph_vertex_t *v, scf_handle_t *h)
4447{
4448	const char * const emsg = "Could not temporarily disable %s because "
4449	    "%s.  Will stop service anyways.  Repository status for the "
4450	    "service may be inaccurate.\n";
4451	const char * const emsg_cbroken =
4452	    "the repository connection was broken";
4453
4454	scf_instance_t *inst;
4455	int r;
4456
4457	inst = scf_instance_create(h);
4458	if (inst == NULL) {
4459		char buf[100];
4460
4461		(void) snprintf(buf, sizeof (buf),
4462		    "scf_instance_create() failed (%s)",
4463		    scf_strerror(scf_error()));
4464		log_error(LOG_WARNING, emsg, v->gv_name, buf);
4465
4466		graph_enable_by_vertex(v, 0, 0);
4467		return (0);
4468	}
4469
4470	r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
4471	    NULL, NULL, SCF_DECODE_FMRI_EXACT);
4472	if (r != 0) {
4473		switch (scf_error()) {
4474		case SCF_ERROR_CONNECTION_BROKEN:
4475			log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4476			graph_enable_by_vertex(v, 0, 0);
4477			return (ECONNABORTED);
4478
4479		case SCF_ERROR_NOT_FOUND:
4480			return (0);
4481
4482		case SCF_ERROR_HANDLE_MISMATCH:
4483		case SCF_ERROR_INVALID_ARGUMENT:
4484		case SCF_ERROR_CONSTRAINT_VIOLATED:
4485		case SCF_ERROR_NOT_BOUND:
4486		default:
4487			bad_error("scf_handle_decode_fmri",
4488			    scf_error());
4489		}
4490	}
4491
4492	r = libscf_set_enable_ovr(inst, 0);
4493	switch (r) {
4494	case 0:
4495		scf_instance_destroy(inst);
4496		return (0);
4497
4498	case ECANCELED:
4499		scf_instance_destroy(inst);
4500		return (0);
4501
4502	case ECONNABORTED:
4503		log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4504		graph_enable_by_vertex(v, 0, 0);
4505		return (ECONNABORTED);
4506
4507	case EPERM:
4508		log_error(LOG_WARNING, emsg, v->gv_name,
4509		    "the repository denied permission");
4510		graph_enable_by_vertex(v, 0, 0);
4511		return (0);
4512
4513	case EROFS:
4514		log_error(LOG_WARNING, emsg, v->gv_name,
4515		    "the repository is read-only");
4516		graph_enable_by_vertex(v, 0, 0);
4517		return (0);
4518
4519	default:
4520		bad_error("libscf_set_enable_ovr", r);
4521		/* NOTREACHED */
4522	}
4523}
4524
4525/*
4526 * Of the transitive instance dependencies of v, offline those which are
4527 * in the subtree and which are leaves (i.e., have no dependents which are
4528 * "up").
4529 */
4530void
4531offline_subtree_leaves(graph_vertex_t *v, void *arg)
4532{
4533	assert(MUTEX_HELD(&dgraph_lock));
4534
4535	/* If v isn't an instance, recurse on its dependencies. */
4536	if (v->gv_type != GVT_INST) {
4537		graph_walk_dependencies(v, offline_subtree_leaves, arg);
4538		return;
4539	}
4540
4541	/*
4542	 * If v is not in the subtree, so should all of its dependencies,
4543	 * so do nothing.
4544	 */
4545	if ((v->gv_flags & GV_TOOFFLINE) == 0)
4546		return;
4547
4548	/* If v isn't a leaf because it's already down, recurse. */
4549	if (!up_state(v->gv_state)) {
4550		graph_walk_dependencies(v, offline_subtree_leaves, arg);
4551		return;
4552	}
4553
4554	/* if v is a leaf, offline it or disable it if it's the last one */
4555	if (insubtree_dependents_down(v) == B_TRUE) {
4556		if (v->gv_flags & GV_TODISABLE)
4557			vertex_send_event(v,
4558			    RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
4559		else
4560			offline_vertex(v);
4561	}
4562}
4563
4564void
4565graph_offline_subtree_leaves(graph_vertex_t *v, void *h)
4566{
4567	graph_walk_dependencies(v, offline_subtree_leaves, (void *)h);
4568}
4569
4570
4571/*
4572 * Of the transitive instance dependencies of v, disable those which are not
4573 * in the subgraph and which are leaves (i.e., have no dependents which are
4574 * "up").
4575 */
4576static void
4577disable_nonsubgraph_leaves(graph_vertex_t *v, void *arg)
4578{
4579	assert(MUTEX_HELD(&dgraph_lock));
4580
4581	/*
4582	 * We must skip exclusion dependencies because they are allowed to
4583	 * complete dependency cycles.  This is correct because A's exclusion
4584	 * dependency on B doesn't bear on the order in which they should be
4585	 * stopped.  Indeed, the exclusion dependency should guarantee that
4586	 * they are never online at the same time.
4587	 */
4588	if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4589		return;
4590
4591	/* If v isn't an instance, recurse on its dependencies. */
4592	if (v->gv_type != GVT_INST)
4593		goto recurse;
4594
4595	if ((v->gv_flags & GV_CONFIGURED) == 0)
4596		/*
4597		 * Unconfigured instances should have no dependencies, but in
4598		 * case they ever get them,
4599		 */
4600		goto recurse;
4601
4602	/*
4603	 * If v is in the subgraph, so should all of its dependencies, so do
4604	 * nothing.
4605	 */
4606	if (v->gv_flags & GV_INSUBGRAPH)
4607		return;
4608
4609	/* If v isn't a leaf because it's already down, recurse. */
4610	if (!up_state(v->gv_state))
4611		goto recurse;
4612
4613	/* If v is disabled but not down yet, be patient. */
4614	if ((v->gv_flags & GV_ENABLED) == 0)
4615		return;
4616
4617	/* If v is a leaf, disable it. */
4618	if (is_nonsubgraph_leaf(v))
4619		(void) disable_service_temporarily(v, (scf_handle_t *)arg);
4620
4621	return;
4622
4623recurse:
4624	graph_walk_dependencies(v, disable_nonsubgraph_leaves, arg);
4625}
4626
4627static int
4628stn_restarter_state(restarter_instance_state_t rstate)
4629{
4630	static const struct statemap {
4631		restarter_instance_state_t restarter_state;
4632		int scf_state;
4633	} map[] = {
4634		{ RESTARTER_STATE_UNINIT, SCF_STATE_UNINIT },
4635		{ RESTARTER_STATE_MAINT, SCF_STATE_MAINT },
4636		{ RESTARTER_STATE_OFFLINE, SCF_STATE_OFFLINE },
4637		{ RESTARTER_STATE_DISABLED, SCF_STATE_DISABLED },
4638		{ RESTARTER_STATE_ONLINE, SCF_STATE_ONLINE },
4639		{ RESTARTER_STATE_DEGRADED, SCF_STATE_DEGRADED }
4640	};
4641
4642	int i;
4643
4644	for (i = 0; i < sizeof (map) / sizeof (map[0]); i++) {
4645		if (rstate == map[i].restarter_state)
4646			return (map[i].scf_state);
4647	}
4648
4649	return (-1);
4650}
4651
4652/*
4653 * State transition counters
4654 * Not incremented atomically - indicative only
4655 */
4656static uint64_t stev_ct_maint;
4657static uint64_t stev_ct_hwerr;
4658static uint64_t stev_ct_service;
4659static uint64_t stev_ct_global;
4660static uint64_t stev_ct_noprefs;
4661static uint64_t stev_ct_from_uninit;
4662static uint64_t stev_ct_bad_state;
4663static uint64_t stev_ct_ovr_prefs;
4664
4665static void
4666dgraph_state_transition_notify(graph_vertex_t *v,
4667    restarter_instance_state_t old_state, restarter_str_t reason)
4668{
4669	restarter_instance_state_t new_state = v->gv_state;
4670	int stn_transition, maint;
4671	int from, to;
4672	nvlist_t *attr;
4673	fmev_pri_t pri = FMEV_LOPRI;
4674	int raise = 0;
4675
4676	if ((from = stn_restarter_state(old_state)) == -1 ||
4677	    (to = stn_restarter_state(new_state)) == -1) {
4678		stev_ct_bad_state++;
4679		return;
4680	}
4681
4682	stn_transition = from << 16 | to;
4683
4684	maint = (to == SCF_STATE_MAINT || from == SCF_STATE_MAINT);
4685
4686	if (maint) {
4687		/*
4688		 * All transitions to/from maintenance state must raise
4689		 * an event.
4690		 */
4691		raise++;
4692		pri = FMEV_HIPRI;
4693		stev_ct_maint++;
4694	} else if (reason == restarter_str_ct_ev_hwerr) {
4695		/*
4696		 * All transitions caused by hardware fault must raise
4697		 * an event
4698		 */
4699		raise++;
4700		pri = FMEV_HIPRI;
4701		stev_ct_hwerr++;
4702	} else if (stn_transition & v->gv_stn_tset) {
4703		/*
4704		 * Specifically enabled event.
4705		 */
4706		raise++;
4707		stev_ct_service++;
4708	} else if (from == SCF_STATE_UNINIT) {
4709		/*
4710		 * Only raise these if specifically selected above.
4711		 */
4712		stev_ct_from_uninit++;
4713	} else if (stn_transition & stn_global &&
4714	    (IS_ENABLED(v) == 1 || to == SCF_STATE_DISABLED)) {
4715		raise++;
4716		stev_ct_global++;
4717	} else {
4718		stev_ct_noprefs++;
4719	}
4720
4721	if (info_events_all) {
4722		stev_ct_ovr_prefs++;
4723		raise++;
4724	}
4725	if (!raise)
4726		return;
4727
4728	if (nvlist_alloc(&attr, NV_UNIQUE_NAME, 0) != 0 ||
4729	    nvlist_add_string(attr, "fmri", v->gv_name) != 0 ||
4730	    nvlist_add_uint32(attr, "reason-version",
4731	    restarter_str_version()) || nvlist_add_string(attr, "reason-short",
4732	    restarter_get_str_short(reason)) != 0 ||
4733	    nvlist_add_string(attr, "reason-long",
4734	    restarter_get_str_long(reason)) != 0 ||
4735	    nvlist_add_int32(attr, "transition", stn_transition) != 0) {
4736		log_framework(LOG_WARNING,
4737		    "FMEV: %s could not create nvlist for transition "
4738		    "event: %s\n", v->gv_name, strerror(errno));
4739		nvlist_free(attr);
4740		return;
4741	}
4742
4743	if (fmev_rspublish_nvl(FMEV_RULESET_SMF, "state-transition",
4744	    instance_state_str[new_state], pri, attr) != FMEV_SUCCESS) {
4745		log_framework(LOG_DEBUG,
4746		    "FMEV: %s failed to publish transition event: %s\n",
4747		    v->gv_name, fmev_strerror(fmev_errno));
4748		nvlist_free(attr);
4749	}
4750}
4751
4752/*
4753 * Find the vertex for inst_name.  If it doesn't exist, return ENOENT.
4754 * Otherwise set its state to state.  If the instance has entered a state
4755 * which requires automatic action, take it (Uninitialized: do
4756 * dgraph_refresh_instance() without the snapshot update.  Disabled: if the
4757 * instance should be enabled, send _ENABLE.  Offline: if the instance should
4758 * be disabled, send _DISABLE, and if its dependencies are satisfied, send
4759 * _START.  Online, Degraded: if the instance wasn't running, update its start
4760 * snapshot.  Maintenance: no action.)
4761 *
4762 * Also fails with ECONNABORTED, or EINVAL if state is invalid.
4763 */
4764static int
4765dgraph_set_instance_state(scf_handle_t *h, const char *inst_name,
4766    protocol_states_t *states)
4767{
4768	graph_vertex_t *v;
4769	int err = 0;
4770	restarter_instance_state_t old_state;
4771	restarter_instance_state_t state = states->ps_state;
4772	restarter_error_t serr = states->ps_err;
4773
4774	MUTEX_LOCK(&dgraph_lock);
4775
4776	v = vertex_get_by_name(inst_name);
4777	if (v == NULL) {
4778		MUTEX_UNLOCK(&dgraph_lock);
4779		return (ENOENT);
4780	}
4781
4782	assert(v->gv_type == GVT_INST);
4783
4784	switch (state) {
4785	case RESTARTER_STATE_UNINIT:
4786	case RESTARTER_STATE_DISABLED:
4787	case RESTARTER_STATE_OFFLINE:
4788	case RESTARTER_STATE_ONLINE:
4789	case RESTARTER_STATE_DEGRADED:
4790	case RESTARTER_STATE_MAINT:
4791		break;
4792
4793	default:
4794		MUTEX_UNLOCK(&dgraph_lock);
4795		return (EINVAL);
4796	}
4797
4798	log_framework(LOG_DEBUG, "Graph noting %s %s -> %s.\n", v->gv_name,
4799	    instance_state_str[v->gv_state], instance_state_str[state]);
4800
4801	old_state = v->gv_state;
4802	v->gv_state = state;
4803
4804	v->gv_reason = states->ps_reason;
4805	err = gt_transition(h, v, serr, old_state);
4806	if (err == 0 && v->gv_state != old_state) {
4807		dgraph_state_transition_notify(v, old_state, states->ps_reason);
4808	}
4809
4810	MUTEX_UNLOCK(&dgraph_lock);
4811	return (err);
4812}
4813
4814/*
4815 * Handle state changes during milestone shutdown.  See
4816 * dgraph_set_milestone().  If the repository connection is broken,
4817 * ECONNABORTED will be returned, though a _DISABLE command will be sent for
4818 * the vertex anyway.
4819 */
4820int
4821vertex_subgraph_dependencies_shutdown(scf_handle_t *h, graph_vertex_t *v,
4822    restarter_instance_state_t old_state)
4823{
4824	int was_up, now_up;
4825	int ret = 0;
4826
4827	assert(v->gv_type == GVT_INST);
4828
4829	/* Don't care if we're not going to a milestone. */
4830	if (milestone == NULL)
4831		return (0);
4832
4833	/* Don't care if we already finished coming down. */
4834	if (non_subgraph_svcs == 0)
4835		return (0);
4836
4837	/* Don't care if the service is in the subgraph. */
4838	if (v->gv_flags & GV_INSUBGRAPH)
4839		return (0);
4840
4841	/*
4842	 * Update non_subgraph_svcs.  It is the number of non-subgraph
4843	 * services which are in online, degraded, or offline.
4844	 */
4845
4846	was_up = up_state(old_state);
4847	now_up = up_state(v->gv_state);
4848
4849	if (!was_up && now_up) {
4850		++non_subgraph_svcs;
4851	} else if (was_up && !now_up) {
4852		--non_subgraph_svcs;
4853
4854		if (non_subgraph_svcs == 0) {
4855			if (halting != -1) {
4856				do_uadmin();
4857			} else if (go_single_user_mode || go_to_level1) {
4858				(void) startd_thread_create(single_user_thread,
4859				    NULL);
4860			}
4861			return (0);
4862		}
4863	}
4864
4865	/* If this service is a leaf, it should be disabled. */
4866	if ((v->gv_flags & GV_ENABLED) && is_nonsubgraph_leaf(v)) {
4867		int r;
4868
4869		r = disable_service_temporarily(v, h);
4870		switch (r) {
4871		case 0:
4872			break;
4873
4874		case ECONNABORTED:
4875			ret = ECONNABORTED;
4876			break;
4877
4878		default:
4879			bad_error("disable_service_temporarily", r);
4880		}
4881	}
4882
4883	/*
4884	 * If the service just came down, propagate the disable to the newly
4885	 * exposed leaves.
4886	 */
4887	if (was_up && !now_up)
4888		graph_walk_dependencies(v, disable_nonsubgraph_leaves,
4889		    (void *)h);
4890
4891	return (ret);
4892}
4893
4894/*
4895 * Decide whether to start up an sulogin thread after a service is
4896 * finished changing state.  Only need to do the full can_come_up()
4897 * evaluation if an instance is changing state, we're not halfway through
4898 * loading the thread, and we aren't shutting down or going to the single
4899 * user milestone.
4900 */
4901void
4902graph_transition_sulogin(restarter_instance_state_t state,
4903    restarter_instance_state_t old_state)
4904{
4905	assert(MUTEX_HELD(&dgraph_lock));
4906
4907	if (state != old_state && st->st_load_complete &&
4908	    !go_single_user_mode && !go_to_level1 &&
4909	    halting == -1) {
4910		if (!sulogin_thread_running && !can_come_up()) {
4911			(void) startd_thread_create(sulogin_thread, NULL);
4912			sulogin_thread_running = B_TRUE;
4913		}
4914	}
4915}
4916
4917/*
4918 * Propagate a start, stop event, or a satisfiability event.
4919 *
4920 * PROPAGATE_START and PROPAGATE_STOP simply propagate the transition event
4921 * to direct dependents.  PROPAGATE_SAT propagates a start then walks the
4922 * full dependent graph to check for newly satisfied nodes.  This is
4923 * necessary for cases when non-direct dependents may be effected but direct
4924 * dependents may not (e.g. for optional_all evaluations, see the
4925 * propagate_satbility() comments).
4926 *
4927 * PROPAGATE_SAT should be used whenever a non-running service moves into
4928 * a state which can satisfy optional dependencies, like disabled or
4929 * maintenance.
4930 */
4931void
4932graph_transition_propagate(graph_vertex_t *v, propagate_event_t type,
4933    restarter_error_t rerr)
4934{
4935	if (type == PROPAGATE_STOP) {
4936		graph_walk_dependents(v, propagate_stop, (void *)rerr);
4937	} else if (type == PROPAGATE_START || type == PROPAGATE_SAT) {
4938		graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
4939
4940		if (type == PROPAGATE_SAT)
4941			propagate_satbility(v);
4942	} else {
4943#ifndef NDEBUG
4944		uu_warn("%s:%d: Unexpected type value %d.\n",  __FILE__,
4945		    __LINE__, type);
4946#endif
4947		abort();
4948	}
4949}
4950
4951/*
4952 * If a vertex for fmri exists and it is enabled, send _DISABLE to the
4953 * restarter.  If it is running, send _STOP.  Send _REMOVE_INSTANCE.  Delete
4954 * all property group dependencies, and the dependency on the restarter,
4955 * disposing of vertices as appropriate.  If other vertices depend on this
4956 * one, mark it unconfigured and return.  Otherwise remove the vertex.  Always
4957 * returns 0.
4958 */
4959static int
4960dgraph_remove_instance(const char *fmri, scf_handle_t *h)
4961{
4962	graph_vertex_t *v;
4963	graph_edge_t *e;
4964	uu_list_t *old_deps;
4965	int err;
4966
4967	log_framework(LOG_DEBUG, "Graph engine: Removing %s.\n", fmri);
4968
4969	MUTEX_LOCK(&dgraph_lock);
4970
4971	v = vertex_get_by_name(fmri);
4972	if (v == NULL) {
4973		MUTEX_UNLOCK(&dgraph_lock);
4974		return (0);
4975	}
4976
4977	/* Send restarter delete event. */
4978	if (v->gv_flags & GV_CONFIGURED)
4979		graph_unset_restarter(v);
4980
4981	if (milestone > MILESTONE_NONE) {
4982		/*
4983		 * Make a list of v's current dependencies so we can
4984		 * reevaluate their GV_INSUBGRAPH flags after the dependencies
4985		 * are removed.
4986		 */
4987		old_deps = startd_list_create(graph_edge_pool, NULL, 0);
4988
4989		err = uu_list_walk(v->gv_dependencies,
4990		    (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
4991		assert(err == 0);
4992	}
4993
4994	delete_instance_dependencies(v, B_TRUE);
4995
4996	/*
4997	 * Deleting an instance can both satisfy and unsatisfy dependencies,
4998	 * depending on their type.  First propagate the stop as a RERR_RESTART
4999	 * event -- deletion isn't a fault, just a normal stop.  This gives
5000	 * dependent services the chance to do a clean shutdown.  Then, mark
5001	 * the service as unconfigured and propagate the start event for the
5002	 * optional_all dependencies that might have become satisfied.
5003	 */
5004	graph_walk_dependents(v, propagate_stop, (void *)RERR_RESTART);
5005
5006	v->gv_flags &= ~GV_CONFIGURED;
5007	v->gv_flags &= ~GV_DEATHROW;
5008
5009	graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
5010	propagate_satbility(v);
5011
5012	/*
5013	 * If there are no (non-service) dependents, the vertex can be
5014	 * completely removed.
5015	 */
5016	if (v != milestone && v->gv_refs == 0 &&
5017	    uu_list_numnodes(v->gv_dependents) == 1)
5018		remove_inst_vertex(v);
5019
5020	if (milestone > MILESTONE_NONE) {
5021		void *cookie = NULL;
5022
5023		while ((e = uu_list_teardown(old_deps, &cookie)) != NULL) {
5024			v = e->ge_vertex;
5025
5026			if (vertex_unref(v) == VERTEX_INUSE)
5027				while (eval_subgraph(v, h) == ECONNABORTED)
5028					libscf_handle_rebind(h);
5029
5030			startd_free(e, sizeof (*e));
5031		}
5032
5033		uu_list_destroy(old_deps);
5034	}
5035
5036	MUTEX_UNLOCK(&dgraph_lock);
5037
5038	return (0);
5039}
5040
5041/*
5042 * Return the eventual (maybe current) milestone in the form of a
5043 * legacy runlevel.
5044 */
5045static char
5046target_milestone_as_runlevel()
5047{
5048	assert(MUTEX_HELD(&dgraph_lock));
5049
5050	if (milestone == NULL)
5051		return ('3');
5052	else if (milestone == MILESTONE_NONE)
5053		return ('0');
5054
5055	if (strcmp(milestone->gv_name, multi_user_fmri) == 0)
5056		return ('2');
5057	else if (strcmp(milestone->gv_name, single_user_fmri) == 0)
5058		return ('S');
5059	else if (strcmp(milestone->gv_name, multi_user_svr_fmri) == 0)
5060		return ('3');
5061
5062#ifndef NDEBUG
5063	(void) fprintf(stderr, "%s:%d: Unknown milestone name \"%s\".\n",
5064	    __FILE__, __LINE__, milestone->gv_name);
5065#endif
5066	abort();
5067	/* NOTREACHED */
5068}
5069
5070static struct {
5071	char	rl;
5072	int	sig;
5073} init_sigs[] = {
5074	{ 'S', SIGBUS },
5075	{ '0', SIGINT },
5076	{ '1', SIGQUIT },
5077	{ '2', SIGILL },
5078	{ '3', SIGTRAP },
5079	{ '4', SIGIOT },
5080	{ '5', SIGEMT },
5081	{ '6', SIGFPE },
5082	{ 0, 0 }
5083};
5084
5085static void
5086signal_init(char rl)
5087{
5088	pid_t init_pid;
5089	int i;
5090
5091	assert(MUTEX_HELD(&dgraph_lock));
5092
5093	if (zone_getattr(getzoneid(), ZONE_ATTR_INITPID, &init_pid,
5094	    sizeof (init_pid)) != sizeof (init_pid)) {
5095		log_error(LOG_NOTICE, "Could not get pid to signal init.\n");
5096		return;
5097	}
5098
5099	for (i = 0; init_sigs[i].rl != 0; ++i)
5100		if (init_sigs[i].rl == rl)
5101			break;
5102
5103	if (init_sigs[i].rl != 0) {
5104		if (kill(init_pid, init_sigs[i].sig) != 0) {
5105			switch (errno) {
5106			case EPERM:
5107			case ESRCH:
5108				log_error(LOG_NOTICE, "Could not signal init: "
5109				    "%s.\n", strerror(errno));
5110				break;
5111
5112			case EINVAL:
5113			default:
5114				bad_error("kill", errno);
5115			}
5116		}
5117	}
5118}
5119
5120/*
5121 * This is called when one of the major milestones changes state, or when
5122 * init is signalled and tells us it was told to change runlevel.  We wait
5123 * to reach the milestone because this allows /etc/inittab entries to retain
5124 * some boot ordering: historically, entries could place themselves before/after
5125 * the running of /sbin/rcX scripts but we can no longer make the
5126 * distinction because the /sbin/rcX scripts no longer exist as punctuation
5127 * marks in /etc/inittab.
5128 *
5129 * Also, we only trigger an update when we reach the eventual target
5130 * milestone: without this, an /etc/inittab entry marked only for
5131 * runlevel 2 would be executed for runlevel 3, which is not how
5132 * /etc/inittab entries work.
5133 *
5134 * If we're single user coming online, then we set utmpx to the target
5135 * runlevel so that legacy scripts can work as expected.
5136 */
5137static void
5138graph_runlevel_changed(char rl, int online)
5139{
5140	char trl;
5141
5142	assert(MUTEX_HELD(&dgraph_lock));
5143
5144	trl = target_milestone_as_runlevel();
5145
5146	if (online) {
5147		if (rl == trl) {
5148			current_runlevel = trl;
5149			signal_init(trl);
5150		} else if (rl == 'S') {
5151			/*
5152			 * At boot, set the entry early for the benefit of the
5153			 * legacy init scripts.
5154			 */
5155			utmpx_set_runlevel(trl, 'S', B_FALSE);
5156		}
5157	} else {
5158		if (rl == '3' && trl == '2') {
5159			current_runlevel = trl;
5160			signal_init(trl);
5161		} else if (rl == '2' && trl == 'S') {
5162			current_runlevel = trl;
5163			signal_init(trl);
5164		}
5165	}
5166}
5167
5168/*
5169 * Move to a backwards-compatible runlevel by executing the appropriate
5170 * /etc/rc?.d/K* scripts and/or setting the milestone.
5171 *
5172 * Returns
5173 *   0 - success
5174 *   ECONNRESET - success, but handle was reset
5175 *   ECONNABORTED - repository connection broken
5176 *   ECANCELED - pg was deleted
5177 */
5178static int
5179dgraph_set_runlevel(scf_propertygroup_t *pg, scf_property_t *prop)
5180{
5181	char rl;
5182	scf_handle_t *h;
5183	int r;
5184	const char *ms = NULL;	/* what to commit as options/milestone */
5185	boolean_t rebound = B_FALSE;
5186	int mark_rl = 0;
5187
5188	const char * const stop = "stop";
5189
5190	r = libscf_extract_runlevel(prop, &rl);
5191	switch (r) {
5192	case 0:
5193		break;
5194
5195	case ECONNABORTED:
5196	case ECANCELED:
5197		return (r);
5198
5199	case EINVAL:
5200	case ENOENT:
5201		log_error(LOG_WARNING, "runlevel property is misconfigured; "
5202		    "ignoring.\n");
5203		/* delete the bad property */
5204		goto nolock_out;
5205
5206	default:
5207		bad_error("libscf_extract_runlevel", r);
5208	}
5209
5210	switch (rl) {
5211	case 's':
5212		rl = 'S';
5213		/* FALLTHROUGH */
5214
5215	case 'S':
5216	case '2':
5217	case '3':
5218		/*
5219		 * These cases cause a milestone change, so
5220		 * graph_runlevel_changed() will eventually deal with
5221		 * signalling init.
5222		 */
5223		break;
5224
5225	case '0':
5226	case '1':
5227	case '4':
5228	case '5':
5229	case '6':
5230		mark_rl = 1;
5231		break;
5232
5233	default:
5234		log_framework(LOG_NOTICE, "Unknown runlevel '%c'.\n", rl);
5235		ms = NULL;
5236		goto nolock_out;
5237	}
5238
5239	h = scf_pg_handle(pg);
5240
5241	MUTEX_LOCK(&dgraph_lock);
5242
5243	/*
5244	 * Since this triggers no milestone changes, force it by hand.
5245	 */
5246	if (current_runlevel == '4' && rl == '3')
5247		mark_rl = 1;
5248
5249	/*
5250	 * 1. If we are here after an "init X":
5251	 *
5252	 * init X
5253	 *	init/lscf_set_runlevel()
5254	 *		process_pg_event()
5255	 *		dgraph_set_runlevel()
5256	 *
5257	 * then we haven't passed through graph_runlevel_changed() yet,
5258	 * therefore 'current_runlevel' has not changed for sure but 'rl' has.
5259	 * In consequence, if 'rl' is lower than 'current_runlevel', we change
5260	 * the system runlevel and execute the appropriate /etc/rc?.d/K* scripts
5261	 * past this test.
5262	 *
5263	 * 2. On the other hand, if we are here after a "svcadm milestone":
5264	 *
5265	 * svcadm milestone X
5266	 *	dgraph_set_milestone()
5267	 *		handle_graph_update_event()
5268	 *		dgraph_set_instance_state()
5269	 *		graph_post_X_[online|offline]()
5270	 *		graph_runlevel_changed()
5271	 *		signal_init()
5272	 *			init/lscf_set_runlevel()
5273	 *				process_pg_event()
5274	 *				dgraph_set_runlevel()
5275	 *
5276	 * then we already passed through graph_runlevel_changed() (by the way
5277	 * of dgraph_set_milestone()) and 'current_runlevel' may have changed
5278	 * and already be equal to 'rl' so we are going to return immediately
5279	 * from dgraph_set_runlevel() without changing the system runlevel and
5280	 * without executing the /etc/rc?.d/K* scripts.
5281	 */
5282	if (rl == current_runlevel) {
5283		ms = NULL;
5284		goto out;
5285	}
5286
5287	log_framework(LOG_DEBUG, "Changing to runlevel '%c'.\n", rl);
5288
5289	/*
5290	 * Make sure stop rc scripts see the new settings via who -r.
5291	 */
5292	utmpx_set_runlevel(rl, current_runlevel, B_TRUE);
5293
5294	/*
5295	 * Some run levels don't have a direct correspondence to any
5296	 * milestones, so we have to signal init directly.
5297	 */
5298	if (mark_rl) {
5299		current_runlevel = rl;
5300		signal_init(rl);
5301	}
5302
5303	switch (rl) {
5304	case 'S':
5305		uu_warn("The system is coming down for administration.  "
5306		    "Please wait.\n");
5307		fork_rc_script(rl, stop, B_FALSE);
5308		ms = single_user_fmri;
5309		go_single_user_mode = B_TRUE;
5310		break;
5311
5312	case '0':
5313		halting_time = time(NULL);
5314		fork_rc_script(rl, stop, B_TRUE);
5315		halting = AD_HALT;
5316		goto uadmin;
5317
5318	case '5':
5319		halting_time = time(NULL);
5320		fork_rc_script(rl, stop, B_TRUE);
5321		halting = AD_POWEROFF;
5322		goto uadmin;
5323
5324	case '6':
5325		halting_time = time(NULL);
5326		fork_rc_script(rl, stop, B_TRUE);
5327		if (scf_is_fastboot_default() && getzoneid() == GLOBAL_ZONEID)
5328			halting = AD_FASTREBOOT;
5329		else
5330			halting = AD_BOOT;
5331
5332uadmin:
5333		uu_warn("The system is coming down.  Please wait.\n");
5334		ms = "none";
5335
5336		/*
5337		 * We can't wait until all services are offline since this
5338		 * thread is responsible for taking them offline.  Instead we
5339		 * set halting to the second argument for uadmin() and call
5340		 * do_uadmin() from dgraph_set_instance_state() when
5341		 * appropriate.
5342		 */
5343		break;
5344
5345	case '1':
5346		if (current_runlevel != 'S') {
5347			uu_warn("Changing to state 1.\n");
5348			fork_rc_script(rl, stop, B_FALSE);
5349		} else {
5350			uu_warn("The system is coming up for administration.  "
5351			    "Please wait.\n");
5352		}
5353		ms = single_user_fmri;
5354		go_to_level1 = B_TRUE;
5355		break;
5356
5357	case '2':
5358		if (current_runlevel == '3' || current_runlevel == '4')
5359			fork_rc_script(rl, stop, B_FALSE);
5360		ms = multi_user_fmri;
5361		break;
5362
5363	case '3':
5364	case '4':
5365		ms = "all";
5366		break;
5367
5368	default:
5369#ifndef NDEBUG
5370		(void) fprintf(stderr, "%s:%d: Uncaught case %d ('%c').\n",
5371		    __FILE__, __LINE__, rl, rl);
5372#endif
5373		abort();
5374	}
5375
5376out:
5377	MUTEX_UNLOCK(&dgraph_lock);
5378
5379nolock_out:
5380	switch (r = libscf_clear_runlevel(pg, ms)) {
5381	case 0:
5382		break;
5383
5384	case ECONNABORTED:
5385		libscf_handle_rebind(h);
5386		rebound = B_TRUE;
5387		goto nolock_out;
5388
5389	case ECANCELED:
5390		break;
5391
5392	case EPERM:
5393	case EACCES:
5394	case EROFS:
5395		log_error(LOG_NOTICE, "Could not delete \"%s/%s\" property: "
5396		    "%s.\n", SCF_PG_OPTIONS, "runlevel", strerror(r));
5397		break;
5398
5399	default:
5400		bad_error("libscf_clear_runlevel", r);
5401	}
5402
5403	return (rebound ? ECONNRESET : 0);
5404}
5405
5406/*
5407 * mark_subtree walks the dependents and add the GV_TOOFFLINE flag
5408 * to the instances that are supposed to go offline during an
5409 * administrative disable operation.
5410 */
5411static int
5412mark_subtree(graph_edge_t *e, void *arg)
5413{
5414	graph_vertex_t *v;
5415	int r;
5416
5417	v = e->ge_vertex;
5418
5419	/* If it's already in the subgraph, skip. */
5420	if (v->gv_flags & GV_TOOFFLINE)
5421		return (UU_WALK_NEXT);
5422
5423	switch (v->gv_type) {
5424	case GVT_INST:
5425		/* If the instance is already offline, skip it. */
5426		if (!inst_running(v))
5427			return (UU_WALK_NEXT);
5428
5429		v->gv_flags |= GV_TOOFFLINE;
5430		log_framework(LOG_DEBUG, "%s added to subtree\n", v->gv_name);
5431		break;
5432	case GVT_GROUP:
5433		/*
5434		 * Skip all excluded dependents and decide whether to offline
5435		 * the service based on the restart_on attribute.
5436		 */
5437		if (is_depgrp_bypassed(v))
5438			return (UU_WALK_NEXT);
5439		break;
5440	}
5441
5442	r = uu_list_walk(v->gv_dependents, (uu_walk_fn_t *)mark_subtree, arg,
5443	    0);
5444	assert(r == 0);
5445	return (UU_WALK_NEXT);
5446}
5447
5448static int
5449mark_subgraph(graph_edge_t *e, void *arg)
5450{
5451	graph_vertex_t *v;
5452	int r;
5453	int optional = (int)arg;
5454
5455	v = e->ge_vertex;
5456
5457	/* If it's already in the subgraph, skip. */
5458	if (v->gv_flags & GV_INSUBGRAPH)
5459		return (UU_WALK_NEXT);
5460
5461	/*
5462	 * Keep track if walk has entered an optional dependency group
5463	 */
5464	if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_OPTIONAL_ALL) {
5465		optional = 1;
5466	}
5467	/*
5468	 * Quit if we are in an optional dependency group and the instance
5469	 * is disabled
5470	 */
5471	if (optional && (v->gv_type == GVT_INST) &&
5472	    (!(v->gv_flags & GV_ENBLD_NOOVR)))
5473		return (UU_WALK_NEXT);
5474
5475	v->gv_flags |= GV_INSUBGRAPH;
5476
5477	/* Skip all excluded dependencies. */
5478	if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
5479		return (UU_WALK_NEXT);
5480
5481	r = uu_list_walk(v->gv_dependencies, (uu_walk_fn_t *)mark_subgraph,
5482	    (void *)optional, 0);
5483	assert(r == 0);
5484	return (UU_WALK_NEXT);
5485}
5486
5487/*
5488 * Bring down all services which are not dependencies of fmri.  The
5489 * dependencies of fmri (direct & indirect) will constitute the "subgraph",
5490 * and will have the GV_INSUBGRAPH flag set.  The rest must be brought down,
5491 * which means the state is "disabled", "maintenance", or "uninitialized".  We
5492 * could consider "offline" to be down, and refrain from sending start
5493 * commands for such services, but that's not strictly necessary, so we'll
5494 * decline to intrude on the state machine.  It would probably confuse users
5495 * anyway.
5496 *
5497 * The services should be brought down in reverse-dependency order, so we
5498 * can't do it all at once here.  We initiate by override-disabling the leaves
5499 * of the dependency tree -- those services which are up but have no
5500 * dependents which are up.  When they come down,
5501 * vertex_subgraph_dependencies_shutdown() will override-disable the newly
5502 * exposed leaves.  Perseverance will ensure completion.
5503 *
5504 * Sometimes we need to take action when the transition is complete, like
5505 * start sulogin or halt the system.  To tell when we're done, we initialize
5506 * non_subgraph_svcs here to be the number of services which need to come
5507 * down.  As each does, we decrement the counter.  When it hits zero, we take
5508 * the appropriate action.  See vertex_subgraph_dependencies_shutdown().
5509 *
5510 * In case we're coming up, we also remove any enable-overrides for the
5511 * services which are dependencies of fmri.
5512 *
5513 * If norepository is true, the function will not change the repository.
5514 *
5515 * The decision to change the system run level in accordance with the milestone
5516 * is taken in dgraph_set_runlevel().
5517 *
5518 * Returns
5519 *   0 - success
5520 *   ECONNRESET - success, but handle was rebound
5521 *   EINVAL - fmri is invalid (error is logged)
5522 *   EALREADY - the milestone is already set to fmri
5523 *   ENOENT - a configured vertex does not exist for fmri (an error is logged)
5524 */
5525static int
5526dgraph_set_milestone(const char *fmri, scf_handle_t *h, boolean_t norepository)
5527{
5528	const char *cfmri, *fs;
5529	graph_vertex_t *nm, *v;
5530	int ret = 0, r;
5531	scf_instance_t *inst;
5532	boolean_t isall, isnone, rebound = B_FALSE;
5533
5534	/* Validate fmri */
5535	isall = (strcmp(fmri, "all") == 0);
5536	isnone = (strcmp(fmri, "none") == 0);
5537
5538	if (!isall && !isnone) {
5539		if (fmri_canonify(fmri, (char **)&cfmri, B_FALSE) == EINVAL)
5540			goto reject;
5541
5542		if (strcmp(cfmri, single_user_fmri) != 0 &&
5543		    strcmp(cfmri, multi_user_fmri) != 0 &&
5544		    strcmp(cfmri, multi_user_svr_fmri) != 0) {
5545			startd_free((void *)cfmri, max_scf_fmri_size);
5546reject:
5547			log_framework(LOG_WARNING,
5548			    "Rejecting request for invalid milestone \"%s\".\n",
5549			    fmri);
5550			return (EINVAL);
5551		}
5552	}
5553
5554	inst = safe_scf_instance_create(h);
5555
5556	MUTEX_LOCK(&dgraph_lock);
5557
5558	if (milestone == NULL) {
5559		if (isall) {
5560			log_framework(LOG_DEBUG,
5561			    "Milestone already set to all.\n");
5562			ret = EALREADY;
5563			goto out;
5564		}
5565	} else if (milestone == MILESTONE_NONE) {
5566		if (isnone) {
5567			log_framework(LOG_DEBUG,
5568			    "Milestone already set to none.\n");
5569			ret = EALREADY;
5570			goto out;
5571		}
5572	} else {
5573		if (!isall && !isnone &&
5574		    strcmp(cfmri, milestone->gv_name) == 0) {
5575			log_framework(LOG_DEBUG,
5576			    "Milestone already set to %s.\n", cfmri);
5577			ret = EALREADY;
5578			goto out;
5579		}
5580	}
5581
5582	if (!isall && !isnone) {
5583		nm = vertex_get_by_name(cfmri);
5584		if (nm == NULL || !(nm->gv_flags & GV_CONFIGURED)) {
5585			log_framework(LOG_WARNING, "Cannot set milestone to %s "
5586			    "because no such service exists.\n", cfmri);
5587			ret = ENOENT;
5588			goto out;
5589		}
5590	}
5591
5592	log_framework(LOG_DEBUG, "Changing milestone to %s.\n", fmri);
5593
5594	/*
5595	 * Set milestone, removing the old one if this was the last reference.
5596	 */
5597	if (milestone > MILESTONE_NONE)
5598		(void) vertex_unref(milestone);
5599
5600	if (isall)
5601		milestone = NULL;
5602	else if (isnone)
5603		milestone = MILESTONE_NONE;
5604	else {
5605		milestone = nm;
5606		/* milestone should count as a reference */
5607		vertex_ref(milestone);
5608	}
5609
5610	/* Clear all GV_INSUBGRAPH bits. */
5611	for (v = uu_list_first(dgraph); v != NULL; v = uu_list_next(dgraph, v))
5612		v->gv_flags &= ~GV_INSUBGRAPH;
5613
5614	if (!isall && !isnone) {
5615		/* Set GV_INSUBGRAPH for milestone & descendents. */
5616		milestone->gv_flags |= GV_INSUBGRAPH;
5617
5618		r = uu_list_walk(milestone->gv_dependencies,
5619		    (uu_walk_fn_t *)mark_subgraph, NULL, 0);
5620		assert(r == 0);
5621	}
5622
5623	/* Un-override services in the subgraph & override-disable the rest. */
5624	if (norepository)
5625		goto out;
5626
5627	non_subgraph_svcs = 0;
5628	for (v = uu_list_first(dgraph);
5629	    v != NULL;
5630	    v = uu_list_next(dgraph, v)) {
5631		if (v->gv_type != GVT_INST ||
5632		    (v->gv_flags & GV_CONFIGURED) == 0)
5633			continue;
5634
5635again:
5636		r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
5637		    NULL, NULL, SCF_DECODE_FMRI_EXACT);
5638		if (r != 0) {
5639			switch (scf_error()) {
5640			case SCF_ERROR_CONNECTION_BROKEN:
5641			default:
5642				libscf_handle_rebind(h);
5643				rebound = B_TRUE;
5644				goto again;
5645
5646			case SCF_ERROR_NOT_FOUND:
5647				continue;
5648
5649			case SCF_ERROR_HANDLE_MISMATCH:
5650			case SCF_ERROR_INVALID_ARGUMENT:
5651			case SCF_ERROR_CONSTRAINT_VIOLATED:
5652			case SCF_ERROR_NOT_BOUND:
5653				bad_error("scf_handle_decode_fmri",
5654				    scf_error());
5655			}
5656		}
5657
5658		if (isall || (v->gv_flags & GV_INSUBGRAPH)) {
5659			r = libscf_delete_enable_ovr(inst);
5660			fs = "libscf_delete_enable_ovr";
5661		} else {
5662			assert(isnone || (v->gv_flags & GV_INSUBGRAPH) == 0);
5663
5664			/*
5665			 * Services which are up need to come down before
5666			 * we're done, but we can only disable the leaves
5667			 * here.
5668			 */
5669
5670			if (up_state(v->gv_state))
5671				++non_subgraph_svcs;
5672
5673			/* If it's already disabled, don't bother. */
5674			if ((v->gv_flags & GV_ENABLED) == 0)
5675				continue;
5676
5677			if (!is_nonsubgraph_leaf(v))
5678				continue;
5679
5680			r = libscf_set_enable_ovr(inst, 0);
5681			fs = "libscf_set_enable_ovr";
5682		}
5683		switch (r) {
5684		case 0:
5685		case ECANCELED:
5686			break;
5687
5688		case ECONNABORTED:
5689			libscf_handle_rebind(h);
5690			rebound = B_TRUE;
5691			goto again;
5692
5693		case EPERM:
5694		case EROFS:
5695			log_error(LOG_WARNING,
5696			    "Could not set %s/%s for %s: %s.\n",
5697			    SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
5698			    v->gv_name, strerror(r));
5699			break;
5700
5701		default:
5702			bad_error(fs, r);
5703		}
5704	}
5705
5706	if (halting != -1) {
5707		if (non_subgraph_svcs > 1)
5708			uu_warn("%d system services are now being stopped.\n",
5709			    non_subgraph_svcs);
5710		else if (non_subgraph_svcs == 1)
5711			uu_warn("One system service is now being stopped.\n");
5712		else if (non_subgraph_svcs == 0)
5713			do_uadmin();
5714	}
5715
5716	ret = rebound ? ECONNRESET : 0;
5717
5718out:
5719	MUTEX_UNLOCK(&dgraph_lock);
5720	if (!isall && !isnone)
5721		startd_free((void *)cfmri, max_scf_fmri_size);
5722	scf_instance_destroy(inst);
5723	return (ret);
5724}
5725
5726
5727/*
5728 * Returns 0, ECONNABORTED, or EINVAL.
5729 */
5730static int
5731handle_graph_update_event(scf_handle_t *h, graph_protocol_event_t *e)
5732{
5733	int r;
5734
5735	switch (e->gpe_type) {
5736	case GRAPH_UPDATE_RELOAD_GRAPH:
5737		log_error(LOG_WARNING,
5738		    "graph_event: reload graph unimplemented\n");
5739		break;
5740
5741	case GRAPH_UPDATE_STATE_CHANGE: {
5742		protocol_states_t *states = e->gpe_data;
5743
5744		switch (r = dgraph_set_instance_state(h, e->gpe_inst, states)) {
5745		case 0:
5746		case ENOENT:
5747			break;
5748
5749		case ECONNABORTED:
5750			return (ECONNABORTED);
5751
5752		case EINVAL:
5753		default:
5754#ifndef NDEBUG
5755			(void) fprintf(stderr, "dgraph_set_instance_state() "
5756			    "failed with unexpected error %d at %s:%d.\n", r,
5757			    __FILE__, __LINE__);
5758#endif
5759			abort();
5760		}
5761
5762		startd_free(states, sizeof (protocol_states_t));
5763		break;
5764	}
5765
5766	default:
5767		log_error(LOG_WARNING,
5768		    "graph_event_loop received an unknown event: %d\n",
5769		    e->gpe_type);
5770		break;
5771	}
5772
5773	return (0);
5774}
5775
5776/*
5777 * graph_event_thread()
5778 *    Wait for state changes from the restarters.
5779 */
5780/*ARGSUSED*/
5781void *
5782graph_event_thread(void *unused)
5783{
5784	scf_handle_t *h;
5785	int err;
5786
5787	(void) pthread_setname_np(pthread_self(), "graph_event");
5788
5789	h = libscf_handle_create_bound_loop();
5790
5791	/*CONSTCOND*/
5792	while (1) {
5793		graph_protocol_event_t *e;
5794
5795		MUTEX_LOCK(&gu->gu_lock);
5796
5797		while (gu->gu_wakeup == 0)
5798			(void) pthread_cond_wait(&gu->gu_cv, &gu->gu_lock);
5799
5800		gu->gu_wakeup = 0;
5801
5802		while ((e = graph_event_dequeue()) != NULL) {
5803			MUTEX_LOCK(&e->gpe_lock);
5804			MUTEX_UNLOCK(&gu->gu_lock);
5805
5806			while ((err = handle_graph_update_event(h, e)) ==
5807			    ECONNABORTED)
5808				libscf_handle_rebind(h);
5809
5810			if (err == 0)
5811				graph_event_release(e);
5812			else
5813				graph_event_requeue(e);
5814
5815			MUTEX_LOCK(&gu->gu_lock);
5816		}
5817
5818		MUTEX_UNLOCK(&gu->gu_lock);
5819	}
5820}
5821
5822static void
5823set_initial_milestone(scf_handle_t *h)
5824{
5825	scf_instance_t *inst;
5826	char *fmri, *cfmri;
5827	size_t sz;
5828	int r;
5829
5830	inst = safe_scf_instance_create(h);
5831	fmri = startd_alloc(max_scf_fmri_size);
5832
5833	/*
5834	 * If -m milestone= was specified, we want to set options_ovr/milestone
5835	 * to it.  Otherwise we want to read what the milestone should be set
5836	 * to.  Either way we need our inst.
5837	 */
5838get_self:
5839	if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
5840	    NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5841		switch (scf_error()) {
5842		case SCF_ERROR_CONNECTION_BROKEN:
5843			libscf_handle_rebind(h);
5844			goto get_self;
5845
5846		case SCF_ERROR_NOT_FOUND:
5847			if (st->st_subgraph != NULL &&
5848			    st->st_subgraph[0] != '\0') {
5849				sz = strlcpy(fmri, st->st_subgraph,
5850				    max_scf_fmri_size);
5851				assert(sz < max_scf_fmri_size);
5852			} else {
5853				fmri[0] = '\0';
5854			}
5855			break;
5856
5857		case SCF_ERROR_INVALID_ARGUMENT:
5858		case SCF_ERROR_CONSTRAINT_VIOLATED:
5859		case SCF_ERROR_HANDLE_MISMATCH:
5860		default:
5861			bad_error("scf_handle_decode_fmri", scf_error());
5862		}
5863	} else {
5864		if (st->st_subgraph != NULL && st->st_subgraph[0] != '\0') {
5865			scf_propertygroup_t *pg;
5866
5867			pg = safe_scf_pg_create(h);
5868
5869			sz = strlcpy(fmri, st->st_subgraph, max_scf_fmri_size);
5870			assert(sz < max_scf_fmri_size);
5871
5872			r = libscf_inst_get_or_add_pg(inst, SCF_PG_OPTIONS_OVR,
5873			    SCF_PG_OPTIONS_OVR_TYPE, SCF_PG_OPTIONS_OVR_FLAGS,
5874			    pg);
5875			switch (r) {
5876			case 0:
5877				break;
5878
5879			case ECONNABORTED:
5880				libscf_handle_rebind(h);
5881				goto get_self;
5882
5883			case EPERM:
5884			case EACCES:
5885			case EROFS:
5886				log_error(LOG_WARNING, "Could not set %s/%s: "
5887				    "%s.\n", SCF_PG_OPTIONS_OVR,
5888				    SCF_PROPERTY_MILESTONE, strerror(r));
5889				/* FALLTHROUGH */
5890
5891			case ECANCELED:
5892				sz = strlcpy(fmri, st->st_subgraph,
5893				    max_scf_fmri_size);
5894				assert(sz < max_scf_fmri_size);
5895				break;
5896
5897			default:
5898				bad_error("libscf_inst_get_or_add_pg", r);
5899			}
5900
5901			r = libscf_clear_runlevel(pg, fmri);
5902			switch (r) {
5903			case 0:
5904				break;
5905
5906			case ECONNABORTED:
5907				libscf_handle_rebind(h);
5908				goto get_self;
5909
5910			case EPERM:
5911			case EACCES:
5912			case EROFS:
5913				log_error(LOG_WARNING, "Could not set %s/%s: "
5914				    "%s.\n", SCF_PG_OPTIONS_OVR,
5915				    SCF_PROPERTY_MILESTONE, strerror(r));
5916				/* FALLTHROUGH */
5917
5918			case ECANCELED:
5919				sz = strlcpy(fmri, st->st_subgraph,
5920				    max_scf_fmri_size);
5921				assert(sz < max_scf_fmri_size);
5922				break;
5923
5924			default:
5925				bad_error("libscf_clear_runlevel", r);
5926			}
5927
5928			scf_pg_destroy(pg);
5929		} else {
5930			scf_property_t *prop;
5931			scf_value_t *val;
5932
5933			prop = safe_scf_property_create(h);
5934			val = safe_scf_value_create(h);
5935
5936			r = libscf_get_milestone(inst, prop, val, fmri,
5937			    max_scf_fmri_size);
5938			switch (r) {
5939			case 0:
5940				break;
5941
5942			case ECONNABORTED:
5943				libscf_handle_rebind(h);
5944				goto get_self;
5945
5946			case EINVAL:
5947				log_error(LOG_WARNING, "Milestone property is "
5948				    "misconfigured.  Defaulting to \"all\".\n");
5949				/* FALLTHROUGH */
5950
5951			case ECANCELED:
5952			case ENOENT:
5953				fmri[0] = '\0';
5954				break;
5955
5956			default:
5957				bad_error("libscf_get_milestone", r);
5958			}
5959
5960			scf_value_destroy(val);
5961			scf_property_destroy(prop);
5962		}
5963	}
5964
5965	if (fmri[0] == '\0' || strcmp(fmri, "all") == 0)
5966		goto out;
5967
5968	if (strcmp(fmri, "none") != 0) {
5969retry:
5970		if (scf_handle_decode_fmri(h, fmri, NULL, NULL, inst, NULL,
5971		    NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5972			switch (scf_error()) {
5973			case SCF_ERROR_INVALID_ARGUMENT:
5974				log_error(LOG_WARNING,
5975				    "Requested milestone \"%s\" is invalid.  "
5976				    "Reverting to \"all\".\n", fmri);
5977				goto out;
5978
5979			case SCF_ERROR_CONSTRAINT_VIOLATED:
5980				log_error(LOG_WARNING, "Requested milestone "
5981				    "\"%s\" does not specify an instance.  "
5982				    "Reverting to \"all\".\n", fmri);
5983				goto out;
5984
5985			case SCF_ERROR_CONNECTION_BROKEN:
5986				libscf_handle_rebind(h);
5987				goto retry;
5988
5989			case SCF_ERROR_NOT_FOUND:
5990				log_error(LOG_WARNING, "Requested milestone "
5991				    "\"%s\" not in repository.  Reverting to "
5992				    "\"all\".\n", fmri);
5993				goto out;
5994
5995			case SCF_ERROR_HANDLE_MISMATCH:
5996			default:
5997				bad_error("scf_handle_decode_fmri",
5998				    scf_error());
5999			}
6000		}
6001
6002		r = fmri_canonify(fmri, &cfmri, B_FALSE);
6003		assert(r == 0);
6004
6005		r = dgraph_add_instance(cfmri, inst, B_TRUE);
6006		startd_free(cfmri, max_scf_fmri_size);
6007		switch (r) {
6008		case 0:
6009			break;
6010
6011		case ECONNABORTED:
6012			goto retry;
6013
6014		case EINVAL:
6015			log_error(LOG_WARNING,
6016			    "Requested milestone \"%s\" is invalid.  "
6017			    "Reverting to \"all\".\n", fmri);
6018			goto out;
6019
6020		case ECANCELED:
6021			log_error(LOG_WARNING,
6022			    "Requested milestone \"%s\" not "
6023			    "in repository.  Reverting to \"all\".\n",
6024			    fmri);
6025			goto out;
6026
6027		case EEXIST:
6028		default:
6029			bad_error("dgraph_add_instance", r);
6030		}
6031	}
6032
6033	log_console(LOG_INFO, "Booting to milestone \"%s\".\n", fmri);
6034
6035	r = dgraph_set_milestone(fmri, h, B_FALSE);
6036	switch (r) {
6037	case 0:
6038	case ECONNRESET:
6039	case EALREADY:
6040		break;
6041
6042	case EINVAL:
6043	case ENOENT:
6044	default:
6045		bad_error("dgraph_set_milestone", r);
6046	}
6047
6048out:
6049	startd_free(fmri, max_scf_fmri_size);
6050	scf_instance_destroy(inst);
6051}
6052
6053void
6054set_restart_milestone(scf_handle_t *h)
6055{
6056	scf_instance_t *inst;
6057	scf_property_t *prop;
6058	scf_value_t *val;
6059	char *fmri;
6060	int r;
6061
6062	inst = safe_scf_instance_create(h);
6063
6064get_self:
6065	if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL,
6066	    inst, NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
6067		switch (scf_error()) {
6068		case SCF_ERROR_CONNECTION_BROKEN:
6069			libscf_handle_rebind(h);
6070			goto get_self;
6071
6072		case SCF_ERROR_NOT_FOUND:
6073			break;
6074
6075		case SCF_ERROR_INVALID_ARGUMENT:
6076		case SCF_ERROR_CONSTRAINT_VIOLATED:
6077		case SCF_ERROR_HANDLE_MISMATCH:
6078		default:
6079			bad_error("scf_handle_decode_fmri", scf_error());
6080		}
6081
6082		scf_instance_destroy(inst);
6083		return;
6084	}
6085
6086	prop = safe_scf_property_create(h);
6087	val = safe_scf_value_create(h);
6088	fmri = startd_alloc(max_scf_fmri_size);
6089
6090	r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6091	switch (r) {
6092	case 0:
6093		break;
6094
6095	case ECONNABORTED:
6096		libscf_handle_rebind(h);
6097		goto get_self;
6098
6099	case ECANCELED:
6100	case ENOENT:
6101	case EINVAL:
6102		goto out;
6103
6104	default:
6105		bad_error("libscf_get_milestone", r);
6106	}
6107
6108	r = dgraph_set_milestone(fmri, h, B_TRUE);
6109	switch (r) {
6110	case 0:
6111	case ECONNRESET:
6112	case EALREADY:
6113	case EINVAL:
6114	case ENOENT:
6115		break;
6116
6117	default:
6118		bad_error("dgraph_set_milestone", r);
6119	}
6120
6121out:
6122	startd_free(fmri, max_scf_fmri_size);
6123	scf_value_destroy(val);
6124	scf_property_destroy(prop);
6125	scf_instance_destroy(inst);
6126}
6127
6128/*
6129 * void *graph_thread(void *)
6130 *
6131 * Graph management thread.
6132 */
6133/*ARGSUSED*/
6134void *
6135graph_thread(void *arg)
6136{
6137	scf_handle_t *h;
6138	int err;
6139
6140	(void) pthread_setname_np(pthread_self(), "graph");
6141
6142	h = libscf_handle_create_bound_loop();
6143
6144	if (st->st_initial)
6145		set_initial_milestone(h);
6146
6147	MUTEX_LOCK(&dgraph_lock);
6148	initial_milestone_set = B_TRUE;
6149	err = pthread_cond_broadcast(&initial_milestone_cv);
6150	assert(err == 0);
6151	MUTEX_UNLOCK(&dgraph_lock);
6152
6153	libscf_populate_graph(h);
6154
6155	if (!st->st_initial)
6156		set_restart_milestone(h);
6157
6158	MUTEX_LOCK(&st->st_load_lock);
6159	st->st_load_complete = 1;
6160	(void) pthread_cond_broadcast(&st->st_load_cv);
6161	MUTEX_UNLOCK(&st->st_load_lock);
6162
6163	MUTEX_LOCK(&dgraph_lock);
6164	/*
6165	 * Now that we've set st_load_complete we need to check can_come_up()
6166	 * since if we booted to a milestone, then there won't be any more
6167	 * state updates.
6168	 */
6169	if (!go_single_user_mode && !go_to_level1 &&
6170	    halting == -1) {
6171		if (!sulogin_thread_running && !can_come_up()) {
6172			(void) startd_thread_create(sulogin_thread, NULL);
6173			sulogin_thread_running = B_TRUE;
6174		}
6175	}
6176	MUTEX_UNLOCK(&dgraph_lock);
6177
6178	(void) pthread_mutex_lock(&gu->gu_freeze_lock);
6179
6180	/*CONSTCOND*/
6181	while (1) {
6182		(void) pthread_cond_wait(&gu->gu_freeze_cv,
6183		    &gu->gu_freeze_lock);
6184	}
6185}
6186
6187
6188/*
6189 * int next_action()
6190 *   Given an array of timestamps 'a' with 'num' elements, find the
6191 *   lowest non-zero timestamp and return its index. If there are no
6192 *   non-zero elements, return -1.
6193 */
6194static int
6195next_action(hrtime_t *a, int num)
6196{
6197	hrtime_t t = 0;
6198	int i = 0, smallest = -1;
6199
6200	for (i = 0; i < num; i++) {
6201		if (t == 0) {
6202			t = a[i];
6203			smallest = i;
6204		} else if (a[i] != 0 && a[i] < t) {
6205			t = a[i];
6206			smallest = i;
6207		}
6208	}
6209
6210	if (t == 0)
6211		return (-1);
6212	else
6213		return (smallest);
6214}
6215
6216/*
6217 * void process_actions()
6218 *   Process actions requested by the administrator. Possibilities include:
6219 *   refresh, restart, maintenance mode off, maintenance mode on,
6220 *   maintenance mode immediate, and degraded.
6221 *
6222 *   The set of pending actions is represented in the repository as a
6223 *   per-instance property group, with each action being a single property
6224 *   in that group.  This property group is converted to an array, with each
6225 *   action type having an array slot.  The actions in the array at the
6226 *   time process_actions() is called are acted on in the order of the
6227 *   timestamp (which is the value stored in the slot).  A value of zero
6228 *   indicates that there is no pending action of the type associated with
6229 *   a particular slot.
6230 *
6231 *   Sending an action event multiple times before the restarter has a
6232 *   chance to process that action will force it to be run at the last
6233 *   timestamp where it appears in the ordering.
6234 *
6235 *   Turning maintenance mode on trumps all other actions.
6236 *
6237 *   Returns 0 or ECONNABORTED.
6238 */
6239static int
6240process_actions(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst)
6241{
6242	scf_property_t *prop = NULL;
6243	scf_value_t *val = NULL;
6244	scf_type_t type;
6245	graph_vertex_t *vertex;
6246	admin_action_t a;
6247	int i, ret = 0, r;
6248	hrtime_t action_ts[NACTIONS];
6249	char *inst_name;
6250
6251	r = libscf_instance_get_fmri(inst, &inst_name);
6252	switch (r) {
6253	case 0:
6254		break;
6255
6256	case ECONNABORTED:
6257		return (ECONNABORTED);
6258
6259	case ECANCELED:
6260		return (0);
6261
6262	default:
6263		bad_error("libscf_instance_get_fmri", r);
6264	}
6265
6266	MUTEX_LOCK(&dgraph_lock);
6267
6268	vertex = vertex_get_by_name(inst_name);
6269	if (vertex == NULL) {
6270		MUTEX_UNLOCK(&dgraph_lock);
6271		log_framework(LOG_DEBUG, "%s: Can't find graph vertex. "
6272		    "The instance must have been removed.\n", inst_name);
6273		startd_free(inst_name, max_scf_fmri_size);
6274		return (0);
6275	}
6276
6277	prop = safe_scf_property_create(h);
6278	val = safe_scf_value_create(h);
6279
6280	for (i = 0; i < NACTIONS; i++) {
6281		if (scf_pg_get_property(pg, admin_actions[i], prop) != 0) {
6282			switch (scf_error()) {
6283			case SCF_ERROR_CONNECTION_BROKEN:
6284			default:
6285				ret = ECONNABORTED;
6286				goto out;
6287
6288			case SCF_ERROR_DELETED:
6289				goto out;
6290
6291			case SCF_ERROR_NOT_FOUND:
6292				action_ts[i] = 0;
6293				continue;
6294
6295			case SCF_ERROR_HANDLE_MISMATCH:
6296			case SCF_ERROR_INVALID_ARGUMENT:
6297			case SCF_ERROR_NOT_SET:
6298				bad_error("scf_pg_get_property", scf_error());
6299			}
6300		}
6301
6302		if (scf_property_type(prop, &type) != 0) {
6303			switch (scf_error()) {
6304			case SCF_ERROR_CONNECTION_BROKEN:
6305			default:
6306				ret = ECONNABORTED;
6307				goto out;
6308
6309			case SCF_ERROR_DELETED:
6310				action_ts[i] = 0;
6311				continue;
6312
6313			case SCF_ERROR_NOT_SET:
6314				bad_error("scf_property_type", scf_error());
6315			}
6316		}
6317
6318		if (type != SCF_TYPE_INTEGER) {
6319			action_ts[i] = 0;
6320			continue;
6321		}
6322
6323		if (scf_property_get_value(prop, val) != 0) {
6324			switch (scf_error()) {
6325			case SCF_ERROR_CONNECTION_BROKEN:
6326			default:
6327				ret = ECONNABORTED;
6328				goto out;
6329
6330			case SCF_ERROR_DELETED:
6331				goto out;
6332
6333			case SCF_ERROR_NOT_FOUND:
6334			case SCF_ERROR_CONSTRAINT_VIOLATED:
6335				action_ts[i] = 0;
6336				continue;
6337
6338			case SCF_ERROR_NOT_SET:
6339			case SCF_ERROR_PERMISSION_DENIED:
6340				bad_error("scf_property_get_value",
6341				    scf_error());
6342			}
6343		}
6344
6345		r = scf_value_get_integer(val, &action_ts[i]);
6346		assert(r == 0);
6347	}
6348
6349	a = ADMIN_EVENT_MAINT_ON_IMMEDIATE;
6350	if (action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ||
6351	    action_ts[ADMIN_EVENT_MAINT_ON]) {
6352		a = action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ?
6353		    ADMIN_EVENT_MAINT_ON_IMMEDIATE : ADMIN_EVENT_MAINT_ON;
6354
6355		vertex_send_event(vertex, admin_events[a]);
6356		r = libscf_unset_action(h, pg, a, action_ts[a]);
6357		switch (r) {
6358		case 0:
6359		case EACCES:
6360			break;
6361
6362		case ECONNABORTED:
6363			ret = ECONNABORTED;
6364			goto out;
6365
6366		case EPERM:
6367			uu_die("Insufficient privilege.\n");
6368			/* NOTREACHED */
6369
6370		default:
6371			bad_error("libscf_unset_action", r);
6372		}
6373	}
6374
6375	while ((a = next_action(action_ts, NACTIONS)) != -1) {
6376		log_framework(LOG_DEBUG,
6377		    "Graph: processing %s action for %s.\n", admin_actions[a],
6378		    inst_name);
6379
6380		if (a == ADMIN_EVENT_REFRESH) {
6381			r = dgraph_refresh_instance(vertex, inst);
6382			switch (r) {
6383			case 0:
6384			case ECANCELED:
6385			case EINVAL:
6386			case -1:
6387				break;
6388
6389			case ECONNABORTED:
6390				/* pg & inst are reset now, so just return. */
6391				ret = ECONNABORTED;
6392				goto out;
6393
6394			default:
6395				bad_error("dgraph_refresh_instance", r);
6396			}
6397		}
6398
6399		vertex_send_event(vertex, admin_events[a]);
6400
6401		r = libscf_unset_action(h, pg, a, action_ts[a]);
6402		switch (r) {
6403		case 0:
6404		case EACCES:
6405			break;
6406
6407		case ECONNABORTED:
6408			ret = ECONNABORTED;
6409			goto out;
6410
6411		case EPERM:
6412			uu_die("Insufficient privilege.\n");
6413			/* NOTREACHED */
6414
6415		default:
6416			bad_error("libscf_unset_action", r);
6417		}
6418
6419		action_ts[a] = 0;
6420	}
6421
6422out:
6423	MUTEX_UNLOCK(&dgraph_lock);
6424
6425	scf_property_destroy(prop);
6426	scf_value_destroy(val);
6427	startd_free(inst_name, max_scf_fmri_size);
6428	return (ret);
6429}
6430
6431/*
6432 * inst and pg_name are scratch space, and are unset on entry.
6433 * Returns
6434 *   0 - success
6435 *   ECONNRESET - success, but repository handle rebound
6436 *   ECONNABORTED - repository connection broken
6437 */
6438static int
6439process_pg_event(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst,
6440    char *pg_name)
6441{
6442	int r;
6443	scf_property_t *prop;
6444	scf_value_t *val;
6445	char *fmri;
6446	boolean_t rebound = B_FALSE, rebind_inst = B_FALSE;
6447
6448	if (scf_pg_get_name(pg, pg_name, max_scf_value_size) < 0) {
6449		switch (scf_error()) {
6450		case SCF_ERROR_CONNECTION_BROKEN:
6451		default:
6452			return (ECONNABORTED);
6453
6454		case SCF_ERROR_DELETED:
6455			return (0);
6456
6457		case SCF_ERROR_NOT_SET:
6458			bad_error("scf_pg_get_name", scf_error());
6459		}
6460	}
6461
6462	if (strcmp(pg_name, SCF_PG_GENERAL) == 0 ||
6463	    strcmp(pg_name, SCF_PG_GENERAL_OVR) == 0) {
6464		r = dgraph_update_general(pg);
6465		switch (r) {
6466		case 0:
6467		case ENOTSUP:
6468		case ECANCELED:
6469			return (0);
6470
6471		case ECONNABORTED:
6472			return (ECONNABORTED);
6473
6474		case -1:
6475			/* Error should have been logged. */
6476			return (0);
6477
6478		default:
6479			bad_error("dgraph_update_general", r);
6480		}
6481	} else if (strcmp(pg_name, SCF_PG_RESTARTER_ACTIONS) == 0) {
6482		if (scf_pg_get_parent_instance(pg, inst) != 0) {
6483			switch (scf_error()) {
6484			case SCF_ERROR_CONNECTION_BROKEN:
6485				return (ECONNABORTED);
6486
6487			case SCF_ERROR_DELETED:
6488			case SCF_ERROR_CONSTRAINT_VIOLATED:
6489				/* Ignore commands on services. */
6490				return (0);
6491
6492			case SCF_ERROR_NOT_BOUND:
6493			case SCF_ERROR_HANDLE_MISMATCH:
6494			case SCF_ERROR_NOT_SET:
6495			default:
6496				bad_error("scf_pg_get_parent_instance",
6497				    scf_error());
6498			}
6499		}
6500
6501		return (process_actions(h, pg, inst));
6502	}
6503
6504	if (strcmp(pg_name, SCF_PG_OPTIONS) != 0 &&
6505	    strcmp(pg_name, SCF_PG_OPTIONS_OVR) != 0)
6506		return (0);
6507
6508	/*
6509	 * We only care about the options[_ovr] property groups of our own
6510	 * instance, so get the fmri and compare.  Plus, once we know it's
6511	 * correct, if the repository connection is broken we know exactly what
6512	 * property group we were operating on, and can look it up again.
6513	 */
6514	if (scf_pg_get_parent_instance(pg, inst) != 0) {
6515		switch (scf_error()) {
6516		case SCF_ERROR_CONNECTION_BROKEN:
6517			return (ECONNABORTED);
6518
6519		case SCF_ERROR_DELETED:
6520		case SCF_ERROR_CONSTRAINT_VIOLATED:
6521			return (0);
6522
6523		case SCF_ERROR_HANDLE_MISMATCH:
6524		case SCF_ERROR_NOT_BOUND:
6525		case SCF_ERROR_NOT_SET:
6526		default:
6527			bad_error("scf_pg_get_parent_instance",
6528			    scf_error());
6529		}
6530	}
6531
6532	switch (r = libscf_instance_get_fmri(inst, &fmri)) {
6533	case 0:
6534		break;
6535
6536	case ECONNABORTED:
6537		return (ECONNABORTED);
6538
6539	case ECANCELED:
6540		return (0);
6541
6542	default:
6543		bad_error("libscf_instance_get_fmri", r);
6544	}
6545
6546	if (strcmp(fmri, SCF_SERVICE_STARTD) != 0) {
6547		startd_free(fmri, max_scf_fmri_size);
6548		return (0);
6549	}
6550
6551	/*
6552	 * update the information events flag
6553	 */
6554	if (strcmp(pg_name, SCF_PG_OPTIONS) == 0)
6555		info_events_all = libscf_get_info_events_all(pg);
6556
6557	prop = safe_scf_property_create(h);
6558	val = safe_scf_value_create(h);
6559
6560	if (strcmp(pg_name, SCF_PG_OPTIONS_OVR) == 0) {
6561		/* See if we need to set the runlevel. */
6562		/* CONSTCOND */
6563		if (0) {
6564rebind_pg:
6565			libscf_handle_rebind(h);
6566			rebound = B_TRUE;
6567
6568			r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6569			switch (r) {
6570			case 0:
6571				break;
6572
6573			case ECONNABORTED:
6574				goto rebind_pg;
6575
6576			case ENOENT:
6577				goto out;
6578
6579			case EINVAL:
6580			case ENOTSUP:
6581				bad_error("libscf_lookup_instance", r);
6582			}
6583
6584			if (scf_instance_get_pg(inst, pg_name, pg) != 0) {
6585				switch (scf_error()) {
6586				case SCF_ERROR_DELETED:
6587				case SCF_ERROR_NOT_FOUND:
6588					goto out;
6589
6590				case SCF_ERROR_CONNECTION_BROKEN:
6591					goto rebind_pg;
6592
6593				case SCF_ERROR_HANDLE_MISMATCH:
6594				case SCF_ERROR_NOT_BOUND:
6595				case SCF_ERROR_NOT_SET:
6596				case SCF_ERROR_INVALID_ARGUMENT:
6597				default:
6598					bad_error("scf_instance_get_pg",
6599					    scf_error());
6600				}
6601			}
6602		}
6603
6604		if (scf_pg_get_property(pg, "runlevel", prop) == 0) {
6605			r = dgraph_set_runlevel(pg, prop);
6606			switch (r) {
6607			case ECONNRESET:
6608				rebound = B_TRUE;
6609				rebind_inst = B_TRUE;
6610				/* FALLTHROUGH */
6611
6612			case 0:
6613				break;
6614
6615			case ECONNABORTED:
6616				goto rebind_pg;
6617
6618			case ECANCELED:
6619				goto out;
6620
6621			default:
6622				bad_error("dgraph_set_runlevel", r);
6623			}
6624		} else {
6625			switch (scf_error()) {
6626			case SCF_ERROR_CONNECTION_BROKEN:
6627			default:
6628				goto rebind_pg;
6629
6630			case SCF_ERROR_DELETED:
6631				goto out;
6632
6633			case SCF_ERROR_NOT_FOUND:
6634				break;
6635
6636			case SCF_ERROR_INVALID_ARGUMENT:
6637			case SCF_ERROR_HANDLE_MISMATCH:
6638			case SCF_ERROR_NOT_BOUND:
6639			case SCF_ERROR_NOT_SET:
6640				bad_error("scf_pg_get_property", scf_error());
6641			}
6642		}
6643	}
6644
6645	if (rebind_inst) {
6646lookup_inst:
6647		r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6648		switch (r) {
6649		case 0:
6650			break;
6651
6652		case ECONNABORTED:
6653			libscf_handle_rebind(h);
6654			rebound = B_TRUE;
6655			goto lookup_inst;
6656
6657		case ENOENT:
6658			goto out;
6659
6660		case EINVAL:
6661		case ENOTSUP:
6662			bad_error("libscf_lookup_instance", r);
6663		}
6664	}
6665
6666	r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6667	switch (r) {
6668	case 0:
6669		break;
6670
6671	case ECONNABORTED:
6672		libscf_handle_rebind(h);
6673		rebound = B_TRUE;
6674		goto lookup_inst;
6675
6676	case EINVAL:
6677		log_error(LOG_NOTICE,
6678		    "%s/%s property of %s is misconfigured.\n", pg_name,
6679		    SCF_PROPERTY_MILESTONE, SCF_SERVICE_STARTD);
6680		/* FALLTHROUGH */
6681
6682	case ECANCELED:
6683	case ENOENT:
6684		(void) strcpy(fmri, "all");
6685		break;
6686
6687	default:
6688		bad_error("libscf_get_milestone", r);
6689	}
6690
6691	r = dgraph_set_milestone(fmri, h, B_FALSE);
6692	switch (r) {
6693	case 0:
6694	case ECONNRESET:
6695	case EALREADY:
6696		break;
6697
6698	case EINVAL:
6699		log_error(LOG_WARNING, "Milestone %s is invalid.\n", fmri);
6700		break;
6701
6702	case ENOENT:
6703		log_error(LOG_WARNING, "Milestone %s does not exist.\n", fmri);
6704		break;
6705
6706	default:
6707		bad_error("dgraph_set_milestone", r);
6708	}
6709
6710out:
6711	startd_free(fmri, max_scf_fmri_size);
6712	scf_value_destroy(val);
6713	scf_property_destroy(prop);
6714
6715	return (rebound ? ECONNRESET : 0);
6716}
6717
6718/*
6719 * process_delete() deletes an instance from the dgraph if 'fmri' is an
6720 * instance fmri or if 'fmri' matches the 'general' property group of an
6721 * instance (or the 'general/enabled' property).
6722 *
6723 * 'fmri' may be overwritten and cannot be trusted on return by the caller.
6724 */
6725static void
6726process_delete(char *fmri, scf_handle_t *h)
6727{
6728	char *lfmri, *end_inst_fmri;
6729	const char *inst_name = NULL;
6730	const char *pg_name = NULL;
6731	const char *prop_name = NULL;
6732
6733	lfmri = safe_strdup(fmri);
6734
6735	/* Determine if the FMRI is a property group or instance */
6736	if (scf_parse_svc_fmri(lfmri, NULL, NULL, &inst_name, &pg_name,
6737	    &prop_name) != SCF_SUCCESS) {
6738		log_error(LOG_WARNING,
6739		    "Received invalid FMRI \"%s\" from repository server.\n",
6740		    fmri);
6741	} else if (inst_name != NULL && pg_name == NULL) {
6742		(void) dgraph_remove_instance(fmri, h);
6743	} else if (inst_name != NULL && pg_name != NULL) {
6744		/*
6745		 * If we're deleting the 'general' property group or
6746		 * 'general/enabled' property then the whole instance
6747		 * must be removed from the dgraph.
6748		 */
6749		if (strcmp(pg_name, SCF_PG_GENERAL) != 0) {
6750			free(lfmri);
6751			return;
6752		}
6753
6754		if (prop_name != NULL &&
6755		    strcmp(prop_name, SCF_PROPERTY_ENABLED) != 0) {
6756			free(lfmri);
6757			return;
6758		}
6759
6760		/*
6761		 * Because the instance has already been deleted from the
6762		 * repository, we cannot use any scf_ functions to retrieve
6763		 * the instance FMRI however we can easily reconstruct it
6764		 * manually.
6765		 */
6766		end_inst_fmri = strstr(fmri, SCF_FMRI_PROPERTYGRP_PREFIX);
6767		if (end_inst_fmri == NULL)
6768			bad_error("process_delete", 0);
6769
6770		end_inst_fmri[0] = '\0';
6771
6772		(void) dgraph_remove_instance(fmri, h);
6773	}
6774
6775	free(lfmri);
6776}
6777
6778/*ARGSUSED*/
6779void *
6780repository_event_thread(void *unused)
6781{
6782	scf_handle_t *h;
6783	scf_propertygroup_t *pg;
6784	scf_instance_t *inst;
6785	char *fmri = startd_alloc(max_scf_fmri_size);
6786	char *pg_name = startd_alloc(max_scf_value_size);
6787	int r;
6788
6789	(void) pthread_setname_np(pthread_self(), "repository_event");
6790
6791	h = libscf_handle_create_bound_loop();
6792
6793	pg = safe_scf_pg_create(h);
6794	inst = safe_scf_instance_create(h);
6795
6796retry:
6797	if (_scf_notify_add_pgtype(h, SCF_GROUP_FRAMEWORK) != SCF_SUCCESS) {
6798		if (scf_error() == SCF_ERROR_CONNECTION_BROKEN) {
6799			libscf_handle_rebind(h);
6800		} else {
6801			log_error(LOG_WARNING,
6802			    "Couldn't set up repository notification "
6803			    "for property group type %s: %s\n",
6804			    SCF_GROUP_FRAMEWORK, scf_strerror(scf_error()));
6805
6806			(void) sleep(1);
6807		}
6808
6809		goto retry;
6810	}
6811
6812	/*CONSTCOND*/
6813	while (1) {
6814		ssize_t res;
6815
6816		/* Note: fmri is only set on delete events. */
6817		res = _scf_notify_wait(pg, fmri, max_scf_fmri_size);
6818		if (res < 0) {
6819			libscf_handle_rebind(h);
6820			goto retry;
6821		} else if (res == 0) {
6822			/*
6823			 * property group modified.  inst and pg_name are
6824			 * pre-allocated scratch space.
6825			 */
6826			if (scf_pg_update(pg) < 0) {
6827				switch (scf_error()) {
6828				case SCF_ERROR_DELETED:
6829					continue;
6830
6831				case SCF_ERROR_CONNECTION_BROKEN:
6832					log_error(LOG_WARNING,
6833					    "Lost repository event due to "
6834					    "disconnection.\n");
6835					libscf_handle_rebind(h);
6836					goto retry;
6837
6838				case SCF_ERROR_NOT_BOUND:
6839				case SCF_ERROR_NOT_SET:
6840				default:
6841					bad_error("scf_pg_update", scf_error());
6842				}
6843			}
6844
6845			r = process_pg_event(h, pg, inst, pg_name);
6846			switch (r) {
6847			case 0:
6848				break;
6849
6850			case ECONNABORTED:
6851				log_error(LOG_WARNING, "Lost repository event "
6852				    "due to disconnection.\n");
6853				libscf_handle_rebind(h);
6854				/* FALLTHROUGH */
6855
6856			case ECONNRESET:
6857				goto retry;
6858
6859			default:
6860				bad_error("process_pg_event", r);
6861			}
6862		} else {
6863			/*
6864			 * Service, instance, or pg deleted.
6865			 * Don't trust fmri on return.
6866			 */
6867			process_delete(fmri, h);
6868		}
6869	}
6870
6871	/*NOTREACHED*/
6872	return (NULL);
6873}
6874
6875void
6876graph_engine_start()
6877{
6878	int err;
6879
6880	(void) startd_thread_create(graph_thread, NULL);
6881
6882	MUTEX_LOCK(&dgraph_lock);
6883	while (!initial_milestone_set) {
6884		err = pthread_cond_wait(&initial_milestone_cv, &dgraph_lock);
6885		assert(err == 0);
6886	}
6887	MUTEX_UNLOCK(&dgraph_lock);
6888
6889	(void) startd_thread_create(repository_event_thread, NULL);
6890	(void) startd_thread_create(graph_event_thread, NULL);
6891}
6892