1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright 2012 Milan Jurik. All rights reserved.
25  * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
26  * Copyright 2020 Joyent, Inc.
27  */
28 
29 #include <alloca.h>
30 #include <dirent.h>
31 #include <devid.h>
32 #include <fm/libdiskstatus.h>
33 #include <inttypes.h>
34 #include <pthread.h>
35 #include <strings.h>
36 #include <string.h>
37 #include <unistd.h>
38 #include <sys/dkio.h>
39 #include <sys/fm/protocol.h>
40 #include <sys/libdevid.h>
41 #include <sys/scsi/scsi_types.h>
42 #include <sys/byteorder.h>
43 #include <pthread.h>
44 #include <signal.h>
45 #include <fcntl.h>
46 #include <sys/ctfs.h>
47 #include <libcontract.h>
48 #include <poll.h>
49 #include <sys/contract/device.h>
50 #include <libsysevent.h>
51 #include <sys/sysevent/eventdefs.h>
52 #include <scsi/plugins/ses/vendor/sun.h>
53 
54 #include "disk.h"
55 #include "ses.h"
56 
57 #define	SES_VERSION	1
58 
59 #define	SES_STARTING_SUBCHASSIS 256	/* valid subchassis IDs are uint8_t */
60 #define	NO_SUBCHASSIS	((uint64_t)-1)
61 
62 static int ses_snap_freq = 250;		/* in milliseconds */
63 
64 #define	SES_STATUS_UNAVAIL(s)	\
65 	((s) == SES_ESC_UNSUPPORTED || (s) >= SES_ESC_NOT_INSTALLED)
66 
67 #define	HR_SECOND   1000000000
68 
69 /*
70  * Because multiple SES targets can be part of a single chassis, we construct
71  * our own hierarchy that takes this into account.  These SES targets may refer
72  * to the same devices (multiple paths) or to different devices (managing
73  * different portions of the space).  We arrange things into a
74  * ses_enum_enclosure_t, which contains a set of ses targets, and a list of all
75  * nodes found so far.
76  */
77 typedef struct ses_alt_node {
78 	topo_list_t		san_link;
79 	ses_node_t		*san_node;
80 } ses_alt_node_t;
81 
82 typedef struct ses_enum_node {
83 	topo_list_t		sen_link;
84 	ses_node_t		*sen_node;
85 	topo_list_t		sen_alt_nodes;
86 	uint64_t		sen_type;
87 	uint64_t		sen_instance;
88 	ses_enum_target_t	*sen_target;
89 } ses_enum_node_t;
90 
91 typedef struct ses_enum_chassis {
92 	topo_list_t		sec_link;
93 	topo_list_t		sec_subchassis;
94 	topo_list_t		sec_nodes;
95 	topo_list_t		sec_targets;
96 	const char		*sec_csn;
97 	ses_node_t		*sec_enclosure;
98 	ses_enum_target_t	*sec_target;
99 	topo_instance_t		sec_instance;
100 	topo_instance_t		sec_scinstance;
101 	topo_instance_t		sec_maxinstance;
102 	boolean_t		sec_hasdev;
103 	boolean_t		sec_internal;
104 } ses_enum_chassis_t;
105 
106 typedef struct ses_enum_data {
107 	topo_list_t		sed_devs;
108 	topo_list_t		sed_chassis;
109 	ses_enum_chassis_t	*sed_current;
110 	ses_enum_target_t	*sed_target;
111 	int			sed_errno;
112 	char			*sed_name;
113 	topo_mod_t		*sed_mod;
114 	topo_instance_t		sed_instance;
115 } ses_enum_data_t;
116 
117 typedef struct sas_connector_phy_data {
118 	uint64_t    scpd_index;
119 	uint64_t    scpd_pm;
120 } sas_connector_phy_data_t;
121 
122 typedef struct sas_connector_type {
123 	uint64_t    sct_type;
124 	char	    *sct_name;
125 } sas_connector_type_t;
126 
127 static const sas_connector_type_t sas_connector_type_list[] = {
128 	{   0x0, "Information unknown"  },
129 	{   0x1, "External SAS 4x receptacle (see SAS-2 and SFF-8470)"	},
130 	{   0x2, "Exteranl Mini SAS 4x receptacle (see SAS-2 and SFF-8088)" },
131 	{   0x3, "QSFP+ receptacle (see SAS-2.1 and SFF-8436)" },
132 	{   0x4, "Mini SAS 4x active receptacle (see SAS-2.1 and SFF-8088)" },
133 	{   0x5, "Mini SAS HD 4x receptacle (see SAS-2.1 and SFF-8644)" },
134 	{   0x6, "Mini SAS HD 8x receptacle (see SAS-2.1 and SFF-8644)" },
135 	{   0x7, "Mini SAS HD 16x receptacle (see SAS-2.1 and SFF-8644)" },
136 	{   0xF, "Vendor-specific external connector"	},
137 	{   0x10, "Internal wide SAS 4i plug (see SAS-2 and SFF-8484)"	},
138 	{   0x11,
139 	"Internal wide Mini SAS 4i receptacle (see SAS-2 and SFF-8087)"	},
140 	{   0x12, "Mini SAS HD 4i receptacle (see SAS-2.1 and SFF-8643)" },
141 	{   0x20, "Internal SAS Drive receptacle (see SAS-2 and SFF-8482)" },
142 	{   0x21, "Internal SATA host plug (see SAS-2 and SATA-2)"	},
143 	{   0x22, "Internal SAS Drive plug (see SAS-2 and SFF-8482)"	},
144 	{   0x23, "Internal SATA device plug (see SAS-2 and SATA-2)"	},
145 	{   0x24, "Micro SAS receptacle (see SAS-2.14)"	},
146 	{   0x25, "Micro SATA device plug (see SAS-2.1 and SATA)"	},
147 	{   0x26, "Micro SAS plug (see SAS-2.1 and SFF-8486)"	},
148 	{   0x27, "Micro SAS/SATA plug (see SAS-2.1 and SFF-8486)"	},
149 	{   0x28,
150 	"12 Gb/s SAS Drive backplane receptacle (see SAS-34 and SFF-8680)" },
151 	{   0x29, "12Gb/s SAS Drive Plug (see SAS-3 and SFF-8680)"	},
152 	{   0x2A, "Multifunction 12 Gb/s 6x Unshielded receptacle connector "
153 		"receptacle (see SAS-3 and SFF-8639)"	},
154 	{   0x2B, "Multifunction 12 Gb/s 6x Unshielded receptable connector "
155 		"plug (see SAS-3 and SFF-8639)"	},
156 	{   0x2F, "Internal SAS virtual connector"  },
157 	{   0x3F, "Vendor-specific internal connector"	},
158 	{   0x70, "Other Vendor-specific connector"	},
159 	{   0x71, "Other Vendor-specific connector"	},
160 	{   0x72, "Other Vendor-specific connector"	},
161 	{   0x73, "Other Vendor-specific connector"	},
162 	{   0x74, "Other Vendor-specific connector"	},
163 	{   0x75, "Other Vendor-specific connector"	},
164 	{   0x76, "Other Vendor-specific connector"	},
165 	{   0x77, "Other Vendor-specific connector"	},
166 	{   0x78, "Other Vendor-specific connector"	},
167 	{   0x79, "Other Vendor-specific connector"	},
168 	{   0x7A, "Other Vendor-specific connector"	},
169 	{   0x7B, "Other Vendor-specific connector"	},
170 	{   0x7C, "Other Vendor-specific connector"	},
171 	{   0x7D, "Other Vendor-specific connector"	},
172 	{   0x7E, "Other Vendor-specific connector"	},
173 	{   0x7F, "Other Vendor-specific connector"	},
174 	{   0x80, "Not Defined"	}
175 };
176 
177 #define	SAS_CONNECTOR_TYPE_CODE_NOT_DEFINED  0x80
178 #define	SAS_CONNECTOR_TYPE_NOT_DEFINED \
179 	"Connector type not defined by SES-2 standard"
180 #define	SAS_CONNECTOR_TYPE_RESERVED \
181 	"Connector type reserved by SES-2 standard"
182 
183 typedef struct phys_enum_type {
184 	uint64_t    pet_type;
185 	char	    *pet_nodename;
186 	char	    *pet_defaultlabel;
187 	boolean_t   pet_dorange;
188 } phys_enum_type_t;
189 
190 static const phys_enum_type_t phys_enum_type_list[] = {
191 	{   SES_ET_ARRAY_DEVICE, BAY, "BAY", B_TRUE  },
192 	{   SES_ET_COOLING, FAN, "FAN", B_TRUE  },
193 	{   SES_ET_DEVICE, BAY, "BAY", B_TRUE  },
194 	{   SES_ET_ESC_ELECTRONICS, CONTROLLER, "CONTROLLER", B_TRUE  },
195 	{   SES_ET_POWER_SUPPLY, PSU, "PSU", B_TRUE  },
196 	{   SES_ET_SUNW_FANBOARD, FANBOARD, "FANBOARD", B_TRUE  },
197 	{   SES_ET_SUNW_FANMODULE, FANMODULE, "FANMODULE", B_TRUE  },
198 	{   SES_ET_SUNW_POWERBOARD, POWERBOARD, "POWERBOARD", B_TRUE  },
199 	{   SES_ET_SUNW_POWERMODULE, POWERMODULE, "POWERMODULE", B_TRUE  }
200 };
201 
202 #define	N_PHYS_ENUM_TYPES (sizeof (phys_enum_type_list) / \
203 	sizeof (phys_enum_type_list[0]))
204 
205 /*
206  * Structure for the hierarchical tree for element nodes.
207  */
208 typedef struct ses_phys_tree {
209     ses_node_t	*spt_snode;
210     ses_enum_node_t	*spt_senumnode;
211     boolean_t	spt_isfru;
212     uint64_t	spt_eonlyindex;
213     uint64_t	spt_cindex;
214     uint64_t	spt_pindex;
215     uint64_t	spt_maxinst;
216     struct ses_phys_tree    *spt_parent;
217     struct ses_phys_tree    *spt_child;
218     struct ses_phys_tree    *spt_sibling;
219     tnode_t	*spt_tnode;
220 } ses_phys_tree_t;
221 
222 typedef enum {
223 	SES_NEW_CHASSIS		= 0x1,
224 	SES_NEW_SUBCHASSIS	= 0x2,
225 	SES_DUP_CHASSIS		= 0x4,
226 	SES_DUP_SUBCHASSIS	= 0x8
227 } ses_chassis_type_e;
228 
229 static const topo_pgroup_info_t smp_pgroup = {
230 	TOPO_PGROUP_SMP,
231 	TOPO_STABILITY_PRIVATE,
232 	TOPO_STABILITY_PRIVATE,
233 	1
234 };
235 
236 static const topo_pgroup_info_t ses_pgroup = {
237 	TOPO_PGROUP_SES,
238 	TOPO_STABILITY_PRIVATE,
239 	TOPO_STABILITY_PRIVATE,
240 	1
241 };
242 
243 static int ses_present(topo_mod_t *, tnode_t *, topo_version_t, nvlist_t *,
244     nvlist_t **);
245 static int ses_contains(topo_mod_t *, tnode_t *, topo_version_t, nvlist_t *,
246     nvlist_t **);
247 
248 static const topo_method_t ses_component_methods[] = {
249 	{ TOPO_METH_PRESENT, TOPO_METH_PRESENT_DESC,
250 	    TOPO_METH_PRESENT_VERSION0, TOPO_STABILITY_INTERNAL, ses_present },
251 	{ TOPO_METH_FAC_ENUM, TOPO_METH_FAC_ENUM_DESC, 0,
252 	    TOPO_STABILITY_INTERNAL, ses_node_enum_facility },
253 	{ TOPO_METH_SENSOR_FAILURE, TOPO_METH_SENSOR_FAILURE_DESC,
254 	    TOPO_METH_SENSOR_FAILURE_VERSION, TOPO_STABILITY_INTERNAL,
255 	    topo_method_sensor_failure },
256 	{ NULL }
257 };
258 
259 #define	TOPO_METH_SMCI_4U36_LABEL		"smci_4u36_bay_label"
260 #define	TOPO_METH_SMCI_4U36_LABEL_DESC	\
261 	"compute bay labels on SMCI 4U36 storage platform variants"
262 #define	TOPO_METH_SMCI_4U36_LABEL_VERSION	0
263 static int smci_4u36_bay_label(topo_mod_t *, tnode_t *, topo_version_t,
264     nvlist_t *, nvlist_t **);
265 
266 static const topo_method_t ses_bay_methods[] = {
267 	{ TOPO_METH_FAC_ENUM, TOPO_METH_FAC_ENUM_DESC, 0,
268 	    TOPO_STABILITY_INTERNAL, ses_node_enum_facility },
269 	{ TOPO_METH_OCCUPIED, TOPO_METH_OCCUPIED_DESC,
270 	    TOPO_METH_OCCUPIED_VERSION, TOPO_STABILITY_INTERNAL,
271 	    topo_mod_hc_occupied },
272 	{ TOPO_METH_SMCI_4U36_LABEL, TOPO_METH_SMCI_4U36_LABEL_DESC,
273 	    TOPO_METH_SMCI_4U36_LABEL_VERSION, TOPO_STABILITY_INTERNAL,
274 	    smci_4u36_bay_label },
275 	{ NULL }
276 };
277 
278 static const topo_method_t ses_recep_methods[] = {
279 	{ TOPO_METH_OCCUPIED, TOPO_METH_OCCUPIED_DESC,
280 	    TOPO_METH_OCCUPIED_VERSION, TOPO_STABILITY_INTERNAL,
281 	    topo_mod_hc_occupied },
282 	{ NULL }
283 };
284 
285 static const topo_method_t ses_enclosure_methods[] = {
286 	{ TOPO_METH_CONTAINS, TOPO_METH_CONTAINS_DESC,
287 	    TOPO_METH_CONTAINS_VERSION, TOPO_STABILITY_INTERNAL, ses_contains },
288 	{ TOPO_METH_FAC_ENUM, TOPO_METH_FAC_ENUM_DESC, 0,
289 	    TOPO_STABILITY_INTERNAL, ses_enc_enum_facility },
290 	{ NULL }
291 };
292 
293 /*
294  * The bay_label_overrides table can be used to map a server product ID to a
295  * topo method that will be invoked to override the value of the label property
296  * for all bay nodes.  By default the property value is static, derived from
297  * the corresponding SES array device element's descriptor string.
298  */
299 typedef struct ses_label_overrides {
300 	const char *slbl_product;
301 	const char *slbl_mname;
302 } ses_label_overrides_t;
303 
304 /*
305  * This table covers three generations of SMCI's 4U 36-bay storage server
306  * (and the Joyent-branded versions).  There was also an Ivy Bridge variant
307  * which has been omitted due to an inability to find one to test on.
308  */
309 static const ses_label_overrides_t bay_label_overrides[] = {
310 	/* Sandy Bridge variant */
311 	{ "SSG-6047R-E1R36L", TOPO_METH_SMCI_4U36_LABEL },
312 	{ "Joyent-Storage-Platform-5001", TOPO_METH_SMCI_4U36_LABEL },
313 
314 	/* Broadwell variant */
315 	{ "SSG-6048R-E1CR36L", TOPO_METH_SMCI_4U36_LABEL },
316 	{ "Joyent-Storage-Platform-7001", TOPO_METH_SMCI_4U36_LABEL },
317 
318 	/* Skylake variant */
319 	{ "SSG-6049P-E1CR36L", TOPO_METH_SMCI_4U36_LABEL },
320 	{ "Joyent-S10G5", TOPO_METH_SMCI_4U36_LABEL }
321 };
322 
323 #define	N_BAY_LBL_OVERRIDES (sizeof (bay_label_overrides) / \
324 	sizeof (bay_label_overrides[0]))
325 
326 /*
327  * Functions for tracking ses devices which we were unable to open. We retry
328  * these at regular intervals using ses_recheck_dir() and if we find that we
329  * can now open any of them then we send a sysevent to indicate that a new topo
330  * snapshot should be taken.
331  */
332 typedef struct ses_open_fail_list {
333 	struct ses_open_fail_list	*sof_next;
334 	char				*sof_path;
335 } ses_open_fail_list_t;
336 
337 static ses_open_fail_list_t *ses_sofh;
338 static pthread_mutex_t ses_sofmt;
339 static void ses_ct_print(char *ptr);
340 
341 static void
342 ses_recheck_dir()
343 {
344 	ses_target_t *target;
345 	sysevent_id_t eid;
346 	char buf[80];
347 	ses_open_fail_list_t *sof;
348 
349 	/*
350 	 * check list of "unable to open" devices
351 	 */
352 	(void) pthread_mutex_lock(&ses_sofmt);
353 	for (sof = ses_sofh; sof != NULL; sof = sof->sof_next) {
354 		/*
355 		 * see if we can open it now
356 		 */
357 		if ((target = ses_open(LIBSES_VERSION,
358 		    sof->sof_path)) == NULL) {
359 			(void) snprintf(buf, sizeof (buf),
360 			    "recheck_dir - still can't open %s", sof->sof_path);
361 			ses_ct_print(buf);
362 			continue;
363 		}
364 
365 		/*
366 		 * ok - better force a new snapshot
367 		 */
368 		(void) snprintf(buf, sizeof (buf),
369 		    "recheck_dir - can now open %s", sof->sof_path);
370 		ses_ct_print(buf);
371 		(void) sysevent_post_event(EC_PLATFORM, ESC_PLATFORM_SP_RESET,
372 		    SUNW_VENDOR, "fmd", NULL, &eid);
373 		ses_close(target);
374 		break;
375 	}
376 	(void) pthread_mutex_unlock(&ses_sofmt);
377 }
378 
379 static void
380 ses_sof_alloc(topo_mod_t *mod, char *path)
381 {
382 	ses_open_fail_list_t *sof;
383 
384 	(void) pthread_mutex_lock(&ses_sofmt);
385 	sof = topo_mod_zalloc(mod, sizeof (*sof));
386 	topo_mod_dprintf(mod, "sof_alloc %s", path);
387 	sof->sof_path = path;
388 	sof->sof_next = ses_sofh;
389 	ses_sofh = sof;
390 	(void) pthread_mutex_unlock(&ses_sofmt);
391 }
392 
393 static void
394 ses_sof_freeall(topo_mod_t *mod)
395 {
396 	ses_open_fail_list_t *sof, *next_sof;
397 
398 	(void) pthread_mutex_lock(&ses_sofmt);
399 	for (sof = ses_sofh; sof != NULL; sof = next_sof) {
400 		next_sof = sof->sof_next;
401 		topo_mod_dprintf(mod, "sof_freeall %s", sof->sof_path);
402 		topo_mod_strfree(mod, sof->sof_path);
403 		topo_mod_free(mod, sof, sizeof (*sof));
404 	}
405 	ses_sofh = NULL;
406 	(void) pthread_mutex_unlock(&ses_sofmt);
407 }
408 
409 /*
410  * functions for verifying that the ses_enum_target_t held in a device
411  * contract's cookie field is still valid (it may have been freed by
412  * ses_release()).
413  */
414 typedef struct ses_stp_list {
415 	struct ses_stp_list	*ssl_next;
416 	ses_enum_target_t	*ssl_tgt;
417 } ses_stp_list_t;
418 
419 static ses_stp_list_t *ses_sslh;
420 static pthread_mutex_t ses_sslmt;
421 
422 static void
423 ses_ssl_alloc(topo_mod_t *mod, ses_enum_target_t *stp)
424 {
425 	ses_stp_list_t *ssl;
426 
427 	(void) pthread_mutex_lock(&ses_sslmt);
428 	ssl = topo_mod_zalloc(mod, sizeof (*ssl));
429 	topo_mod_dprintf(mod, "ssl_alloc %p", stp);
430 	ssl->ssl_tgt = stp;
431 	ssl->ssl_next = ses_sslh;
432 	ses_sslh = ssl;
433 	(void) pthread_mutex_unlock(&ses_sslmt);
434 }
435 
436 static void
437 ses_ssl_free(topo_mod_t *mod, ses_enum_target_t *stp)
438 {
439 	ses_stp_list_t *ssl, *prev_ssl;
440 
441 	(void) pthread_mutex_lock(&ses_sslmt);
442 	prev_ssl = NULL;
443 	for (ssl = ses_sslh; ssl != NULL; ssl = ssl->ssl_next) {
444 		if (ssl->ssl_tgt == stp) {
445 			topo_mod_dprintf(mod, "ssl_free %p", ssl->ssl_tgt);
446 			if (prev_ssl == NULL)
447 				ses_sslh = ssl->ssl_next;
448 			else
449 				prev_ssl->ssl_next = ssl->ssl_next;
450 			topo_mod_free(mod, ssl, sizeof (*ssl));
451 			break;
452 		}
453 		prev_ssl = ssl;
454 	}
455 	(void) pthread_mutex_unlock(&ses_sslmt);
456 }
457 
458 static int
459 ses_ssl_valid(ses_enum_target_t *stp)
460 {
461 	ses_stp_list_t *ssl;
462 
463 	for (ssl = ses_sslh; ssl != NULL; ssl = ssl->ssl_next)
464 		if (ssl->ssl_tgt == stp)
465 			return (1);
466 	return (0);
467 }
468 
469 /*
470  * Functions for creating and destroying a background thread
471  * (ses_contract_thread) used for detecting when ses devices have been
472  * retired/unretired.
473  */
474 static struct ses_thread_s {
475 	pthread_mutex_t mt;
476 	pthread_t tid;
477 	int thr_sig;
478 	int doexit;
479 	int count;
480 } sesthread = {
481 	PTHREAD_MUTEX_INITIALIZER,
482 	0,
483 	SIGTERM,
484 	0,
485 	0
486 };
487 
488 typedef struct ses_mod_list {
489 	struct ses_mod_list	*smod_next;
490 	topo_mod_t		*smod_mod;
491 } ses_mod_list_t;
492 
493 static ses_mod_list_t *ses_smod;
494 
495 static void
496 ses_ct_print(char *ptr)
497 {
498 	(void) pthread_mutex_lock(&sesthread.mt);
499 	if (ses_smod != NULL && ses_smod->smod_mod != NULL)
500 		topo_mod_dprintf(ses_smod->smod_mod, ptr);
501 	(void) pthread_mutex_unlock(&sesthread.mt);
502 }
503 
504 /*ARGSUSED*/
505 static void *
506 ses_contract_thread(void *arg)
507 {
508 	int efd, ctlfd, statfd;
509 	ct_evthdl_t ev;
510 	ctevid_t evid;
511 	uint_t event;
512 	char path[PATH_MAX];
513 	char buf[80];
514 	ses_enum_target_t *stp;
515 	ct_stathdl_t stathdl;
516 	ctid_t ctid;
517 	struct pollfd fds;
518 	int pollret;
519 	sigset_t sigset;
520 
521 	ses_ct_print("start contract event thread");
522 	efd = open64(CTFS_ROOT "/device/pbundle", O_RDONLY);
523 	fds.fd = efd;
524 	fds.events = POLLIN;
525 	fds.revents = 0;
526 	(void) sigaddset(&sigset, sesthread.thr_sig);
527 	(void) pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);
528 	for (;;) {
529 		/* check if we've been asked to exit */
530 		(void) pthread_mutex_lock(&sesthread.mt);
531 		if (sesthread.doexit) {
532 			(void) pthread_mutex_unlock(&sesthread.mt);
533 			break;
534 		}
535 		(void) pthread_mutex_unlock(&sesthread.mt);
536 
537 		/* poll until an event arrives */
538 		if ((pollret = poll(&fds, 1, 10000)) <= 0) {
539 			if (pollret == 0)
540 				ses_recheck_dir();
541 			continue;
542 		}
543 
544 		/* read the event */
545 		(void) pthread_mutex_lock(&ses_sslmt);
546 		ses_ct_print("read contract event");
547 		if (ct_event_read(efd, &ev) != 0) {
548 			(void) pthread_mutex_unlock(&ses_sslmt);
549 			continue;
550 		}
551 
552 		/* see if it is an event we are expecting */
553 		ctid = ct_event_get_ctid(ev);
554 		(void) snprintf(buf, sizeof (buf),
555 		    "got contract event ctid=%d", ctid);
556 		ses_ct_print(buf);
557 		event = ct_event_get_type(ev);
558 		if (event != CT_DEV_EV_OFFLINE && event != CT_EV_NEGEND) {
559 			(void) snprintf(buf, sizeof (buf),
560 			    "bad contract event %x", event);
561 			ses_ct_print(buf);
562 			ct_event_free(ev);
563 			(void) pthread_mutex_unlock(&ses_sslmt);
564 			continue;
565 		}
566 
567 		/* find target pointer saved in cookie */
568 		evid = ct_event_get_evid(ev);
569 		(void) snprintf(path, PATH_MAX, CTFS_ROOT "/device/%ld/status",
570 		    ctid);
571 		statfd = open64(path, O_RDONLY);
572 		(void) ct_status_read(statfd, CTD_COMMON, &stathdl);
573 		stp = (ses_enum_target_t *)(uintptr_t)
574 		    ct_status_get_cookie(stathdl);
575 		ct_status_free(stathdl);
576 		(void) close(statfd);
577 
578 		/* check if target pointer is still valid */
579 		if (ses_ssl_valid(stp) == 0) {
580 			(void) snprintf(buf, sizeof (buf),
581 			    "contract already abandoned %x", event);
582 			ses_ct_print(buf);
583 			(void) snprintf(path, PATH_MAX,
584 			    CTFS_ROOT "/device/%ld/ctl", ctid);
585 			ctlfd = open64(path, O_WRONLY);
586 			if (event != CT_EV_NEGEND)
587 				(void) ct_ctl_ack(ctlfd, evid);
588 			else
589 				(void) ct_ctl_abandon(ctlfd);
590 			(void) close(ctlfd);
591 			ct_event_free(ev);
592 			(void) pthread_mutex_unlock(&ses_sslmt);
593 			continue;
594 		}
595 
596 		/* find control device for ack/abandon */
597 		(void) pthread_mutex_lock(&stp->set_lock);
598 		(void) snprintf(path, PATH_MAX, CTFS_ROOT "/device/%ld/ctl",
599 		    ctid);
600 		ctlfd = open64(path, O_WRONLY);
601 		if (event != CT_EV_NEGEND) {
602 			/* if this is an offline event, do the offline */
603 			ses_ct_print("got contract offline event");
604 			if (stp->set_target) {
605 				ses_ct_print("contract thread rele");
606 				ses_snap_rele(stp->set_snap);
607 				ses_close(stp->set_target);
608 				stp->set_target = NULL;
609 			}
610 			(void) ct_ctl_ack(ctlfd, evid);
611 		} else {
612 			/* if this is the negend, then abandon the contract */
613 			ses_ct_print("got contract negend");
614 			if (stp->set_ctid) {
615 				(void) snprintf(buf, sizeof (buf),
616 				    "abandon old contract %d", stp->set_ctid);
617 				ses_ct_print(buf);
618 				stp->set_ctid = 0;
619 			}
620 			(void) ct_ctl_abandon(ctlfd);
621 		}
622 		(void) close(ctlfd);
623 		(void) pthread_mutex_unlock(&stp->set_lock);
624 		ct_event_free(ev);
625 		(void) pthread_mutex_unlock(&ses_sslmt);
626 	}
627 	(void) close(efd);
628 	return (NULL);
629 }
630 
631 int
632 find_thr_sig(void)
633 {
634 	int i;
635 	sigset_t oset, rset;
636 	int sig[] = {SIGTERM, SIGUSR1, SIGUSR2};
637 	int sig_sz = sizeof (sig) / sizeof (int);
638 	int rc = SIGTERM;
639 
640 	/* prefered set of signals that are likely used to terminate threads */
641 	(void) sigemptyset(&oset);
642 	(void) pthread_sigmask(SIG_SETMASK, NULL, &oset);
643 	for (i = 0; i < sig_sz; i++) {
644 		if (sigismember(&oset, sig[i]) == 0) {
645 			return (sig[i]);
646 		}
647 	}
648 
649 	/* reserved set of signals that are not allowed to terminate thread */
650 	(void) sigemptyset(&rset);
651 	(void) sigaddset(&rset, SIGABRT);
652 	(void) sigaddset(&rset, SIGKILL);
653 	(void) sigaddset(&rset, SIGSTOP);
654 	(void) sigaddset(&rset, SIGCANCEL);
655 
656 	/* Find signal that is not masked and not in the reserved list. */
657 	for (i = 1; i < MAXSIG; i++) {
658 		if (sigismember(&rset, i) == 1) {
659 			continue;
660 		}
661 		if (sigismember(&oset, i) == 0) {
662 			return (i);
663 		}
664 	}
665 
666 	return (rc);
667 }
668 
669 /*ARGSUSED*/
670 static void
671 ses_handler(int sig)
672 {
673 }
674 
675 static void
676 ses_thread_init(topo_mod_t *mod)
677 {
678 	pthread_attr_t *attr = NULL;
679 	struct sigaction act;
680 	ses_mod_list_t *smod;
681 
682 	(void) pthread_mutex_lock(&sesthread.mt);
683 	sesthread.count++;
684 	smod = topo_mod_zalloc(mod, sizeof (*smod));
685 	smod->smod_mod = mod;
686 	smod->smod_next = ses_smod;
687 	ses_smod = smod;
688 	if (sesthread.tid == 0) {
689 		/* find a suitable signal to use for killing the thread below */
690 		sesthread.thr_sig = find_thr_sig();
691 
692 		/* if don't have a handler for this signal, create one */
693 		(void) sigaction(sesthread.thr_sig, NULL, &act);
694 		if (act.sa_handler == SIG_DFL || act.sa_handler == SIG_IGN)
695 			act.sa_handler = ses_handler;
696 		(void) sigaction(sesthread.thr_sig, &act, NULL);
697 
698 		/* create a thread to listen for offline events */
699 		(void) pthread_create(&sesthread.tid,
700 		    attr, ses_contract_thread, NULL);
701 	}
702 	(void) pthread_mutex_unlock(&sesthread.mt);
703 }
704 
705 static void
706 ses_thread_fini(topo_mod_t *mod)
707 {
708 	ses_mod_list_t *smod, *prev_smod;
709 
710 	(void) pthread_mutex_lock(&sesthread.mt);
711 	prev_smod = NULL;
712 	for (smod = ses_smod; smod != NULL; smod = smod->smod_next) {
713 		if (smod->smod_mod == mod) {
714 			if (prev_smod == NULL)
715 				ses_smod = smod->smod_next;
716 			else
717 				prev_smod->smod_next = smod->smod_next;
718 			topo_mod_free(mod, smod, sizeof (*smod));
719 			break;
720 		}
721 		prev_smod = smod;
722 	}
723 	if (--sesthread.count > 0) {
724 		(void) pthread_mutex_unlock(&sesthread.mt);
725 		return;
726 	}
727 	sesthread.doexit = 1;
728 	(void) pthread_mutex_unlock(&sesthread.mt);
729 	(void) pthread_kill(sesthread.tid, sesthread.thr_sig);
730 	(void) pthread_join(sesthread.tid, NULL);
731 	sesthread.tid = 0;
732 }
733 
734 static void
735 ses_create_contract(topo_mod_t *mod, ses_enum_target_t *stp)
736 {
737 	int tfd, len, rval;
738 	char link_path[PATH_MAX];
739 
740 	stp->set_ctid = 0;
741 
742 	/* convert "/dev" path into "/devices" path */
743 	if ((len = readlink(stp->set_devpath, link_path, PATH_MAX)) < 0) {
744 		topo_mod_dprintf(mod, "readlink failed");
745 		return;
746 	}
747 	link_path[len] = '\0';
748 
749 	/* set up template to create new contract */
750 	tfd = open64(CTFS_ROOT "/device/template", O_RDWR);
751 	(void) ct_tmpl_set_critical(tfd, CT_DEV_EV_OFFLINE);
752 	(void) ct_tmpl_set_cookie(tfd, (uint64_t)(uintptr_t)stp);
753 
754 	/* strip "../../devices" off the front and create the contract */
755 	if ((rval = ct_dev_tmpl_set_minor(tfd, &link_path[13])) != 0)
756 		topo_mod_dprintf(mod, "failed to set minor %s rval = %d",
757 		    &link_path[13], rval);
758 	else if ((rval = ct_tmpl_create(tfd, &stp->set_ctid)) != 0)
759 		topo_mod_dprintf(mod, "failed to create ctid rval = %d", rval);
760 	else
761 		topo_mod_dprintf(mod, "created ctid=%d", stp->set_ctid);
762 	(void) close(tfd);
763 }
764 
765 static void
766 ses_target_free(topo_mod_t *mod, ses_enum_target_t *stp)
767 {
768 	if (--stp->set_refcount == 0) {
769 		/* check if already closed due to contract offline request */
770 		(void) pthread_mutex_lock(&stp->set_lock);
771 		if (stp->set_target) {
772 			ses_snap_rele(stp->set_snap);
773 			ses_close(stp->set_target);
774 			stp->set_target = NULL;
775 		}
776 		if (stp->set_ctid) {
777 			int ctlfd;
778 			char path[PATH_MAX];
779 
780 			topo_mod_dprintf(mod, "abandon old contract %d",
781 			    stp->set_ctid);
782 			(void) snprintf(path, PATH_MAX,
783 			    CTFS_ROOT "/device/%ld/ctl", stp->set_ctid);
784 			ctlfd = open64(path, O_WRONLY);
785 			(void) ct_ctl_abandon(ctlfd);
786 			(void) close(ctlfd);
787 			stp->set_ctid = 0;
788 		}
789 		(void) pthread_mutex_unlock(&stp->set_lock);
790 		ses_ssl_free(mod, stp);
791 		topo_mod_strfree(mod, stp->set_devpath);
792 		topo_mod_free(mod, stp, sizeof (ses_enum_target_t));
793 	}
794 }
795 
796 static void
797 ses_data_free(ses_enum_data_t *sdp, ses_enum_chassis_t *pcp)
798 {
799 	topo_mod_t *mod = sdp->sed_mod;
800 	ses_enum_chassis_t *cp;
801 	ses_enum_node_t *np;
802 	ses_enum_target_t *tp;
803 	ses_alt_node_t *ap;
804 	topo_list_t *cpl;
805 
806 
807 	if (pcp != NULL)
808 		cpl = &pcp->sec_subchassis;
809 	else
810 		cpl = &sdp->sed_chassis;
811 
812 	while ((cp = topo_list_next(cpl)) != NULL) {
813 		topo_list_delete(cpl, cp);
814 
815 		while ((np = topo_list_next(&cp->sec_nodes)) != NULL) {
816 			while ((ap = topo_list_next(&np->sen_alt_nodes)) !=
817 			    NULL) {
818 				topo_list_delete(&np->sen_alt_nodes, ap);
819 				topo_mod_free(mod, ap, sizeof (ses_alt_node_t));
820 			}
821 			topo_list_delete(&cp->sec_nodes, np);
822 			topo_mod_free(mod, np, sizeof (ses_enum_node_t));
823 		}
824 
825 		while ((tp = topo_list_next(&cp->sec_targets)) != NULL) {
826 			topo_list_delete(&cp->sec_targets, tp);
827 			ses_target_free(mod, tp);
828 		}
829 
830 		topo_mod_free(mod, cp, sizeof (ses_enum_chassis_t));
831 	}
832 
833 	if (pcp == NULL) {
834 		dev_list_free(mod, &sdp->sed_devs);
835 		topo_mod_free(mod, sdp, sizeof (ses_enum_data_t));
836 	}
837 }
838 
839 /*
840  * For enclosure nodes, we have a special contains method.  By default, the hc
841  * walker will compare the node name and instance number to determine if an
842  * FMRI matches.  For enclosures where the enumeration order is impossible to
843  * predict, we instead use the chassis-id as a unique identifier, and ignore
844  * the instance number.
845  */
846 static int
847 fmri_contains(topo_mod_t *mod, nvlist_t *nv1, nvlist_t *nv2)
848 {
849 	uint8_t v1, v2;
850 	nvlist_t **hcp1, **hcp2;
851 	int err, i;
852 	uint_t nhcp1, nhcp2;
853 	nvlist_t *a1, *a2;
854 	char *c1, *c2;
855 	int mindepth;
856 
857 	if (nvlist_lookup_uint8(nv1, FM_VERSION, &v1) != 0 ||
858 	    nvlist_lookup_uint8(nv2, FM_VERSION, &v2) != 0 ||
859 	    v1 > FM_HC_SCHEME_VERSION || v2 > FM_HC_SCHEME_VERSION)
860 		return (topo_mod_seterrno(mod, EMOD_FMRI_VERSION));
861 
862 	err = nvlist_lookup_nvlist_array(nv1, FM_FMRI_HC_LIST, &hcp1, &nhcp1);
863 	err |= nvlist_lookup_nvlist_array(nv2, FM_FMRI_HC_LIST, &hcp2, &nhcp2);
864 	if (err != 0)
865 		return (topo_mod_seterrno(mod, EMOD_FMRI_NVL));
866 
867 	/*
868 	 * If the chassis-id doesn't match, then these FMRIs are not
869 	 * equivalent.  If one of the FMRIs doesn't have a chassis ID, then we
870 	 * have no choice but to fall back to the instance ID.
871 	 */
872 	if (nvlist_lookup_nvlist(nv1, FM_FMRI_AUTHORITY, &a1) == 0 &&
873 	    nvlist_lookup_nvlist(nv2, FM_FMRI_AUTHORITY, &a2) == 0 &&
874 	    nvlist_lookup_string(a1, FM_FMRI_AUTH_CHASSIS, &c1) == 0 &&
875 	    nvlist_lookup_string(a2, FM_FMRI_AUTH_CHASSIS, &c2) == 0) {
876 		if (strcmp(c1, c2) != 0)
877 			return (0);
878 
879 		mindepth = 1;
880 	} else {
881 		mindepth = 0;
882 	}
883 
884 	if (nhcp2 < nhcp1)
885 		return (0);
886 
887 	for (i = 0; i < nhcp1; i++) {
888 		char *nm1 = NULL;
889 		char *nm2 = NULL;
890 		char *id1 = NULL;
891 		char *id2 = NULL;
892 
893 		(void) nvlist_lookup_string(hcp1[i], FM_FMRI_HC_NAME, &nm1);
894 		(void) nvlist_lookup_string(hcp2[i], FM_FMRI_HC_NAME, &nm2);
895 		(void) nvlist_lookup_string(hcp1[i], FM_FMRI_HC_ID, &id1);
896 		(void) nvlist_lookup_string(hcp2[i], FM_FMRI_HC_ID, &id2);
897 		if (nm1 == NULL || nm2 == NULL || id1 == NULL || id2 == NULL)
898 			return (topo_mod_seterrno(mod, EMOD_FMRI_NVL));
899 
900 		if (strcmp(nm1, nm2) == 0 &&
901 		    (i < mindepth || strcmp(id1, id2) == 0))
902 			continue;
903 
904 		return (0);
905 	}
906 
907 	return (1);
908 }
909 
910 /*ARGSUSED*/
911 static int
912 ses_contains(topo_mod_t *mod, tnode_t *tn, topo_version_t version,
913     nvlist_t *in, nvlist_t **out)
914 {
915 	int ret;
916 	nvlist_t *nv1, *nv2;
917 
918 	if (version > TOPO_METH_CONTAINS_VERSION)
919 		return (topo_mod_seterrno(mod, EMOD_VER_NEW));
920 
921 	if (nvlist_lookup_nvlist(in, TOPO_METH_FMRI_ARG_FMRI, &nv1) != 0 ||
922 	    nvlist_lookup_nvlist(in, TOPO_METH_FMRI_ARG_SUBFMRI, &nv2) != 0)
923 		return (topo_mod_seterrno(mod, EMOD_METHOD_INVAL));
924 
925 	ret = fmri_contains(mod, nv1, nv2);
926 	if (ret < 0)
927 		return (-1);
928 
929 	if (topo_mod_nvalloc(mod, out, NV_UNIQUE_NAME) == 0) {
930 		if (nvlist_add_uint32(*out, TOPO_METH_CONTAINS_RET,
931 		    ret) == 0)
932 			return (0);
933 		else
934 			nvlist_free(*out);
935 	}
936 
937 	return (-1);
938 
939 }
940 
941 /*
942  * Return a current instance of the node.  This is somewhat complicated because
943  * we need to take a new snapshot in order to get the new data, but we don't
944  * want to be constantly taking SES snapshots if the consumer is going to do a
945  * series of queries.  So we adopt the strategy of assuming that the SES state
946  * is not going to be rapidly changing, and limit our snapshot frequency to
947  * some defined bounds.
948  */
949 ses_node_t *
950 ses_node_lock(topo_mod_t *mod, tnode_t *tn)
951 {
952 	ses_enum_target_t *tp = topo_node_getspecific(tn);
953 	hrtime_t now;
954 	ses_snap_t *snap;
955 	int err;
956 	uint64_t nodeid;
957 	ses_node_t *np;
958 
959 	if (tp == NULL) {
960 		(void) topo_mod_seterrno(mod, EMOD_METHOD_NOTSUP);
961 		return (NULL);
962 	}
963 
964 	(void) pthread_mutex_lock(&tp->set_lock);
965 
966 	/*
967 	 * Determine if we need to take a new snapshot.
968 	 */
969 	now = gethrtime();
970 
971 	if (tp->set_target == NULL) {
972 		/*
973 		 * We may have closed the device but not yet abandoned the
974 		 * contract (ie we've had the offline event but not yet the
975 		 * negend). If so, just return failure.
976 		 */
977 		if (tp->set_ctid != 0) {
978 			(void) topo_mod_seterrno(mod, EMOD_METHOD_NOTSUP);
979 			(void) pthread_mutex_unlock(&tp->set_lock);
980 			return (NULL);
981 		}
982 
983 		/*
984 		 * The device has been closed due to a contract offline
985 		 * request, then we need to reopen it and create a new contract.
986 		 */
987 		if ((tp->set_target =
988 		    ses_open(LIBSES_VERSION, tp->set_devpath)) == NULL) {
989 			sysevent_id_t eid;
990 
991 			(void) topo_mod_seterrno(mod, EMOD_METHOD_NOTSUP);
992 			(void) pthread_mutex_unlock(&tp->set_lock);
993 			topo_mod_dprintf(mod, "recheck_dir - "
994 			    "can no longer open %s", tp->set_devpath);
995 			(void) sysevent_post_event(EC_PLATFORM,
996 			    ESC_PLATFORM_SP_RESET, SUNW_VENDOR, "fmd", NULL,
997 			    &eid);
998 			return (NULL);
999 		}
1000 		topo_mod_dprintf(mod, "reopen contract");
1001 		ses_create_contract(mod, tp);
1002 		tp->set_snap = ses_snap_hold(tp->set_target);
1003 		tp->set_snaptime = gethrtime();
1004 	} else if (now - tp->set_snaptime > (ses_snap_freq * 1000 * 1000) &&
1005 	    (snap = ses_snap_new(tp->set_target)) != NULL) {
1006 		if (ses_snap_generation(snap) !=
1007 		    ses_snap_generation(tp->set_snap)) {
1008 			/*
1009 			 * If we find ourselves in this situation, we're in
1010 			 * trouble.  The generation count has changed, which
1011 			 * indicates that our current topology is out of date.
1012 			 * But we need to consult the new topology in order to
1013 			 * determine presence at this moment in time.  We can't
1014 			 * go back and change the topo snapshot in situ, so
1015 			 * we'll just have to fail the call in this unlikely
1016 			 * scenario.
1017 			 */
1018 			ses_snap_rele(snap);
1019 			(void) topo_mod_seterrno(mod, EMOD_METHOD_NOTSUP);
1020 			(void) pthread_mutex_unlock(&tp->set_lock);
1021 			return (NULL);
1022 		} else {
1023 			ses_snap_rele(tp->set_snap);
1024 			tp->set_snap = snap;
1025 		}
1026 		tp->set_snaptime = gethrtime();
1027 	}
1028 
1029 	snap = tp->set_snap;
1030 
1031 	verify(topo_prop_get_uint64(tn, TOPO_PGROUP_SES,
1032 	    TOPO_PROP_NODE_ID, &nodeid, &err) == 0);
1033 	verify((np = ses_node_lookup(snap, nodeid)) != NULL);
1034 
1035 	return (np);
1036 }
1037 
1038 /*ARGSUSED*/
1039 void
1040 ses_node_unlock(topo_mod_t *mod, tnode_t *tn)
1041 {
1042 	ses_enum_target_t *tp = topo_node_getspecific(tn);
1043 
1044 	verify(tp != NULL);
1045 
1046 	(void) pthread_mutex_unlock(&tp->set_lock);
1047 }
1048 
1049 /*
1050  * Determine if the element is present.
1051  */
1052 /*ARGSUSED*/
1053 static int
1054 ses_present(topo_mod_t *mod, tnode_t *tn, topo_version_t version,
1055     nvlist_t *in, nvlist_t **out)
1056 {
1057 	boolean_t present;
1058 	ses_node_t *np;
1059 	nvlist_t *props, *nvl;
1060 	uint64_t status;
1061 
1062 	if ((np = ses_node_lock(mod, tn)) == NULL)
1063 		return (-1);
1064 
1065 	verify((props = ses_node_props(np)) != NULL);
1066 	verify(nvlist_lookup_uint64(props,
1067 	    SES_PROP_STATUS_CODE, &status) == 0);
1068 
1069 	ses_node_unlock(mod, tn);
1070 
1071 	present = (status != SES_ESC_NOT_INSTALLED);
1072 
1073 	if (topo_mod_nvalloc(mod, &nvl, NV_UNIQUE_NAME) != 0)
1074 		return (topo_mod_seterrno(mod, EMOD_FMRI_NVL));
1075 
1076 	if (nvlist_add_uint32(nvl, TOPO_METH_PRESENT_RET,
1077 	    present) != 0) {
1078 		nvlist_free(nvl);
1079 		return (topo_mod_seterrno(mod, EMOD_FMRI_NVL));
1080 	}
1081 
1082 	*out = nvl;
1083 
1084 	return (0);
1085 }
1086 
1087 /*
1088  * Sets standard properties for a ses node (enclosure, bay, controller
1089  * or expander).
1090  * This includes setting the FRU, as well as setting the
1091  * authority information.  When  the fru topo node(frutn) is not NULL
1092  * its resouce should be used as FRU.
1093  */
1094 static int
1095 ses_set_standard_props(topo_mod_t *mod, tnode_t *frutn, tnode_t *tn,
1096     nvlist_t *auth, uint64_t nodeid, const char *path)
1097 {
1098 	int err;
1099 	char *product, *chassis;
1100 	nvlist_t *fmri;
1101 
1102 	/*
1103 	 * Set the authority explicitly if specified.
1104 	 */
1105 	if (auth) {
1106 		verify(nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT,
1107 		    &product) == 0);
1108 		verify(nvlist_lookup_string(auth, FM_FMRI_AUTH_CHASSIS,
1109 		    &chassis) == 0);
1110 		if (topo_prop_set_string(tn, FM_FMRI_AUTHORITY,
1111 		    FM_FMRI_AUTH_PRODUCT, TOPO_PROP_IMMUTABLE, product,
1112 		    &err) != 0 ||
1113 		    topo_prop_set_string(tn, FM_FMRI_AUTHORITY,
1114 		    FM_FMRI_AUTH_CHASSIS, TOPO_PROP_IMMUTABLE, chassis,
1115 		    &err) != 0 ||
1116 		    topo_prop_set_string(tn, FM_FMRI_AUTHORITY,
1117 		    FM_FMRI_AUTH_SERVER, TOPO_PROP_IMMUTABLE, "",
1118 		    &err) != 0) {
1119 			topo_mod_dprintf(mod, "failed to add authority "
1120 			    "properties: %s\n", topo_strerror(err));
1121 			return (topo_mod_seterrno(mod, err));
1122 		}
1123 	}
1124 
1125 	/*
1126 	 * Copy the resource and set that as the FRU.
1127 	 */
1128 	if (frutn != NULL) {
1129 		if (topo_node_resource(frutn, &fmri, &err) != 0) {
1130 			topo_mod_dprintf(mod,
1131 			    "topo_node_resource() failed : %s\n",
1132 			    topo_strerror(err));
1133 			return (topo_mod_seterrno(mod, err));
1134 		}
1135 	} else {
1136 		if (topo_node_resource(tn, &fmri, &err) != 0) {
1137 			topo_mod_dprintf(mod,
1138 			    "topo_node_resource() failed : %s\n",
1139 			    topo_strerror(err));
1140 			return (topo_mod_seterrno(mod, err));
1141 		}
1142 	}
1143 
1144 	if (topo_node_fru_set(tn, fmri, 0, &err) != 0) {
1145 		topo_mod_dprintf(mod,
1146 		    "topo_node_fru_set() failed : %s\n",
1147 		    topo_strerror(err));
1148 		nvlist_free(fmri);
1149 		return (topo_mod_seterrno(mod, err));
1150 	}
1151 
1152 	nvlist_free(fmri);
1153 
1154 	/*
1155 	 * Set the SES-specific properties so that consumers can query
1156 	 * additional information about the particular SES element.
1157 	 */
1158 	if (topo_pgroup_create(tn, &ses_pgroup, &err) != 0) {
1159 		topo_mod_dprintf(mod, "failed to create propgroup "
1160 		    "%s: %s\n", TOPO_PGROUP_SES, topo_strerror(err));
1161 		return (-1);
1162 	}
1163 
1164 	if (topo_prop_set_uint64(tn, TOPO_PGROUP_SES,
1165 	    TOPO_PROP_NODE_ID, TOPO_PROP_IMMUTABLE,
1166 	    nodeid, &err) != 0) {
1167 		topo_mod_dprintf(mod,
1168 		    "failed to create property %s: %s\n",
1169 		    TOPO_PROP_NODE_ID, topo_strerror(err));
1170 		return (-1);
1171 	}
1172 
1173 	if (topo_prop_set_string(tn, TOPO_PGROUP_SES,
1174 	    TOPO_PROP_TARGET_PATH, TOPO_PROP_IMMUTABLE,
1175 	    path, &err) != 0) {
1176 		topo_mod_dprintf(mod,
1177 		    "failed to create property %s: %s\n",
1178 		    TOPO_PROP_TARGET_PATH, topo_strerror(err));
1179 		return (-1);
1180 	}
1181 
1182 	return (0);
1183 }
1184 
1185 /*
1186  * Iterate over the SES phy information. If any of the ports indicates that it's
1187  * a SATA device and we haven't matched any disk devices yet, that means
1188  * that the HBA was able to create a WWN for the SATA device based on its GUID,
1189  * which is good. However, SES includes the WWN for the device's STP bridge. In
1190  * theory, if the driver includes the WWN based on the SATA guid then it should
1191  * also set the bridge-port property indicating the WWN that should match the
1192  * SATA device.
1193  */
1194 static int
1195 ses_create_disk_bridge(ses_enum_data_t *sdp, tnode_t *pnode, nvlist_t *props,
1196     tnode_t **child)
1197 {
1198 	nvlist_t **phys;
1199 	uint_t i, n_phys;
1200 	topo_mod_t *mod = sdp->sed_mod;
1201 
1202 	if (nvlist_lookup_nvlist_array(props, SES_SAS_PROP_PHYS, &phys,
1203 	    &n_phys) != 0)
1204 		return (1);
1205 
1206 	for (i = 0; i < n_phys; i++) {
1207 		uint64_t wwn;
1208 		boolean_t sata;
1209 		char wwnstr[64];
1210 
1211 		if (nvlist_lookup_uint64(phys[i], SES_SAS_PROP_ADDR,
1212 		    &wwn) != 0 || wwn == 0) {
1213 			continue;
1214 		}
1215 
1216 		if (nvlist_lookup_boolean_value(phys[i],
1217 		    SES_SAS_PROP_SATA_DEVICE, &sata) != 0 || !sata) {
1218 			continue;
1219 		}
1220 
1221 		if (scsi_wwn_to_wwnstr(wwn, 0, wwnstr) == NULL)
1222 			continue;
1223 
1224 		if (disk_declare_bridge(mod, pnode, &sdp->sed_devs,
1225 		    wwnstr, child) == 0) {
1226 			return (0);
1227 		}
1228 
1229 	}
1230 
1231 	return (1);
1232 }
1233 
1234 /*
1235  * Callback to add a disk to a given bay.  We first check the status-code to
1236  * determine if a disk is present, ignoring those that aren't in an appropriate
1237  * state.  We then scan the parent bay node's SAS address array to determine
1238  * possible attached SAS addresses.  We create a disk node if the disk is not
1239  * SAS or the SES target does not support the necessary pages for this; if we
1240  * find the SAS address, we create a disk node and also correlate it with
1241  * the corresponding Solaris device node to fill in the rest of the data.
1242  */
1243 static int
1244 ses_create_disk(ses_enum_data_t *sdp, tnode_t *pnode, nvlist_t *props)
1245 {
1246 	topo_mod_t *mod = sdp->sed_mod;
1247 	uint64_t status;
1248 	uint_t s, nsas;
1249 	char **paths;
1250 	int err, ret;
1251 	tnode_t *child = NULL;
1252 
1253 	/*
1254 	 * Skip devices that are not in a present (and possibly damaged) state.
1255 	 * Also, skip devices that this expander is either not fully wired to,
1256 	 * or are hidden due to SAS zoning, as indicated by the
1257 	 * SES_ESC_NO_ACCESS state.
1258 	 */
1259 	if (nvlist_lookup_uint64(props, SES_PROP_STATUS_CODE, &status) != 0)
1260 		return (0);
1261 
1262 	if (status != SES_ESC_UNSUPPORTED &&
1263 	    status != SES_ESC_OK &&
1264 	    status != SES_ESC_CRITICAL &&
1265 	    status != SES_ESC_NONCRITICAL &&
1266 	    status != SES_ESC_UNRECOVERABLE &&
1267 	    status != SES_ESC_UNKNOWN)
1268 		return (0);
1269 
1270 	topo_mod_dprintf(mod, "found attached disk");
1271 
1272 	/*
1273 	 * Create the disk range.
1274 	 */
1275 	if (topo_node_range_create(mod, pnode, DISK, 0, 0) != 0) {
1276 		topo_mod_dprintf(mod,
1277 		    "topo_node_create_range() failed: %s",
1278 		    topo_mod_errmsg(mod));
1279 		return (-1);
1280 	}
1281 
1282 	/*
1283 	 * Look through all SAS addresses and attempt to correlate them to a
1284 	 * known Solaris device.  If we don't find a matching node, then we
1285 	 * don't enumerate the disk node.
1286 	 * Note that TOPO_PROP_SAS_ADDR prop includes SAS address from
1287 	 * alternate elements that represent the same device.
1288 	 */
1289 	if (topo_prop_get_string_array(pnode, TOPO_PGROUP_SES,
1290 	    TOPO_PROP_SAS_ADDR, &paths, &nsas, &err) != 0)
1291 		return (0);
1292 
1293 	err = 0;
1294 
1295 	for (s = 0; s < nsas; s++) {
1296 		ret = disk_declare_addr(mod, pnode, &sdp->sed_devs, paths[s],
1297 		    &child);
1298 		if (ret == 0) {
1299 			break;
1300 		} else if (ret < 0) {
1301 			err = -1;
1302 			break;
1303 		}
1304 	}
1305 
1306 	/*
1307 	 * We need to take another pass through the properties for this bay by
1308 	 * iterating over the phys and noting if any of these are SATA. Note,
1309 	 * this information isn't commonly part of the topo tree at this time,
1310 	 * hence why we end up going back and iterating over the properties
1311 	 * ourselves.
1312 	 */
1313 	if (s == nsas) {
1314 		if (ses_create_disk_bridge(sdp, pnode, props, &child) != 0)
1315 			(void) disk_declare_non_enumerated(mod, pnode, &child);
1316 	}
1317 
1318 	/* copy sas_addresses (target-ports) from parent (with 'w'added) */
1319 	if (child != NULL) {
1320 		int i;
1321 		char **tports;
1322 		uint64_t wwn;
1323 
1324 		tports = topo_mod_zalloc(mod, sizeof (char *) * nsas);
1325 		if (tports != NULL) {
1326 			for (i = 0; i < nsas; i++) {
1327 				if (scsi_wwnstr_to_wwn(paths[i], &wwn) !=
1328 				    DDI_SUCCESS)
1329 					break;
1330 				tports[i] = scsi_wwn_to_wwnstr(wwn, 1, NULL);
1331 				if (tports[i] == NULL)
1332 					break;
1333 			}
1334 			/* if they all worked then create the property */
1335 			if (i == nsas)
1336 				(void) topo_prop_set_string_array(child,
1337 				    TOPO_PGROUP_STORAGE,
1338 				    TOPO_STORAGE_TARGET_PORT_L0IDS,
1339 				    TOPO_PROP_IMMUTABLE, (const char **)tports,
1340 				    nsas, &err);
1341 
1342 			for (i = 0; i < nsas; i++)
1343 				if (tports[i] != NULL)
1344 					scsi_free_wwnstr(tports[i]);
1345 			topo_mod_free(mod, tports, sizeof (char *) * nsas);
1346 		}
1347 	}
1348 
1349 	for (s = 0; s < nsas; s++)
1350 		topo_mod_free(mod, paths[s], strlen(paths[s]) + 1);
1351 	topo_mod_free(mod, paths, nsas * sizeof (char *));
1352 
1353 	return (err);
1354 }
1355 
1356 static int
1357 ses_add_bay_props(topo_mod_t *mod, tnode_t *tn, ses_enum_node_t *snp)
1358 {
1359 	ses_alt_node_t *ap;
1360 	ses_node_t *np;
1361 	nvlist_t *props;
1362 
1363 	nvlist_t **phys;
1364 	uint_t i, j, n_phys, all_phys = 0;
1365 	char **paths;
1366 	uint64_t addr;
1367 	size_t len;
1368 	int terr, err = -1;
1369 
1370 	for (ap = topo_list_next(&snp->sen_alt_nodes); ap != NULL;
1371 	    ap = topo_list_next(ap)) {
1372 		np = ap->san_node;
1373 		props = ses_node_props(np);
1374 
1375 		if (nvlist_lookup_nvlist_array(props, SES_SAS_PROP_PHYS,
1376 		    &phys, &n_phys) != 0)
1377 			continue;
1378 
1379 		all_phys += n_phys;
1380 	}
1381 
1382 	if (all_phys == 0)
1383 		return (0);
1384 
1385 	if ((paths = topo_mod_zalloc(mod, all_phys * sizeof (char *))) == NULL)
1386 		return (-1);
1387 
1388 	for (i = 0, ap = topo_list_next(&snp->sen_alt_nodes); ap != NULL;
1389 	    ap = topo_list_next(ap)) {
1390 		np = ap->san_node;
1391 		props = ses_node_props(np);
1392 
1393 		if (nvlist_lookup_nvlist_array(props, SES_SAS_PROP_PHYS,
1394 		    &phys, &n_phys) != 0)
1395 			continue;
1396 
1397 		for (j = 0; j < n_phys; j++) {
1398 			if (nvlist_lookup_uint64(phys[j], SES_SAS_PROP_ADDR,
1399 			    &addr) != 0)
1400 				continue;
1401 
1402 			len = snprintf(NULL, 0, "%016llx", addr) + 1;
1403 			if ((paths[i] = topo_mod_alloc(mod, len)) == NULL)
1404 				goto error;
1405 
1406 			(void) snprintf(paths[i], len, "%016llx", addr);
1407 
1408 			++i;
1409 		}
1410 	}
1411 
1412 	err = topo_prop_set_string_array(tn, TOPO_PGROUP_SES,
1413 	    TOPO_PROP_SAS_ADDR, TOPO_PROP_IMMUTABLE,
1414 	    (const char **)paths, i, &terr);
1415 	if (err != 0)
1416 		err = topo_mod_seterrno(mod, terr);
1417 
1418 error:
1419 	for (i = 0; i < all_phys && paths[i] != NULL; i++)
1420 		topo_mod_free(mod, paths[i], strlen(paths[i]) + 1);
1421 	topo_mod_free(mod, paths, all_phys * sizeof (char *));
1422 
1423 	return (err);
1424 }
1425 
1426 static const char *
1427 lookup_bay_override(const char *product_id)
1428 {
1429 	for (uint_t i = 0; i < N_BAY_LBL_OVERRIDES; i++) {
1430 		if (strcmp(product_id,
1431 		    bay_label_overrides[i].slbl_product) == 0) {
1432 			return (bay_label_overrides[i].slbl_mname);
1433 		}
1434 	}
1435 	return (NULL);
1436 }
1437 
1438 /*
1439  * Callback to create a basic node (bay, psu, fan, or controller and expander).
1440  */
1441 static int
1442 ses_create_generic(ses_enum_data_t *sdp, ses_enum_node_t *snp, tnode_t *pnode,
1443     tnode_t *frutn, const char *nodename, const char *labelname,
1444     tnode_t **node)
1445 {
1446 	ses_node_t *np = snp->sen_node;
1447 	ses_node_t *parent;
1448 	uint64_t instance = snp->sen_instance;
1449 	topo_mod_t *mod = sdp->sed_mod;
1450 	nvlist_t *props, *aprops;
1451 	nvlist_t *auth = NULL, *fmri = NULL;
1452 	tnode_t *tn = NULL;
1453 	char *clean_label = NULL, label[128];
1454 	int err;
1455 	char *part = NULL, *serial = NULL, *revision = NULL;
1456 	char *desc;
1457 	boolean_t report;
1458 
1459 	props = ses_node_props(np);
1460 
1461 	(void) nvlist_lookup_string(props, LIBSES_PROP_PART, &part);
1462 	(void) nvlist_lookup_string(props, LIBSES_PROP_SERIAL, &serial);
1463 
1464 	topo_mod_dprintf(mod, "adding %s %llu", nodename, instance);
1465 
1466 	/*
1467 	 * Create the node.  The interesting information is all copied from the
1468 	 * parent enclosure node, so there is not much to do.
1469 	 */
1470 	if ((auth = topo_mod_auth(mod, pnode)) == NULL)
1471 		goto error;
1472 
1473 	/*
1474 	 * We want to report revision information for the controller nodes, but
1475 	 * we do not get per-element revision information.  However, we do have
1476 	 * revision information for the entire enclosure, and we can use the
1477 	 * 'reported-via' property to know that this controller corresponds to
1478 	 * the given revision information.  This means we cannot get revision
1479 	 * information for targets we are not explicitly connected to, but
1480 	 * there is little we can do about the situation.
1481 	 */
1482 	if (strcmp(nodename, CONTROLLER) == 0 &&
1483 	    nvlist_lookup_boolean_value(props, SES_PROP_REPORT, &report) == 0 &&
1484 	    report) {
1485 		for (parent = ses_node_parent(np); parent != NULL;
1486 		    parent = ses_node_parent(parent)) {
1487 			if (ses_node_type(parent) == SES_NODE_ENCLOSURE) {
1488 				(void) nvlist_lookup_string(
1489 				    ses_node_props(parent),
1490 				    SES_EN_PROP_REV, &revision);
1491 				break;
1492 			}
1493 		}
1494 	}
1495 
1496 	if ((fmri = topo_mod_hcfmri(mod, pnode, FM_HC_SCHEME_VERSION,
1497 	    nodename, (topo_instance_t)instance, NULL, auth, part, revision,
1498 	    serial)) == NULL) {
1499 		topo_mod_dprintf(mod, "topo_mod_hcfmri() failed: %s",
1500 		    topo_mod_errmsg(mod));
1501 		goto error;
1502 	}
1503 
1504 	if ((tn = topo_node_bind(mod, pnode, nodename,
1505 	    instance, fmri)) == NULL) {
1506 		topo_mod_dprintf(mod, "topo_node_bind() failed: %s",
1507 		    topo_mod_errmsg(mod));
1508 		goto error;
1509 	}
1510 
1511 	/*
1512 	 * For the node label, we look for the following in order:
1513 	 *
1514 	 * <ses-description>
1515 	 * <ses-class-description> <instance>
1516 	 * <default-type-label> <instance>
1517 	 */
1518 	if (nvlist_lookup_string(props, SES_PROP_DESCRIPTION, &desc) != 0 ||
1519 	    desc[0] == '\0') {
1520 		parent = ses_node_parent(np);
1521 		aprops = ses_node_props(parent);
1522 		if (nvlist_lookup_string(aprops, SES_PROP_CLASS_DESCRIPTION,
1523 		    &desc) != 0 || desc[0] == '\0')
1524 			desc = (char *)labelname;
1525 		(void) snprintf(label, sizeof (label), "%s %llu", desc,
1526 		    instance);
1527 		desc = label;
1528 	}
1529 
1530 	if ((clean_label = topo_mod_clean_str(mod, desc)) == NULL)
1531 		goto error;
1532 
1533 	if (topo_prop_set_string(tn, TOPO_PGROUP_PROTOCOL, TOPO_PROP_LABEL,
1534 	    TOPO_PROP_MUTABLE, clean_label, &err) < 0)
1535 		goto error;
1536 
1537 	if (ses_set_standard_props(mod, frutn, tn, NULL, ses_node_id(np),
1538 	    snp->sen_target->set_devpath) != 0)
1539 		goto error;
1540 
1541 	if (strcmp(nodename, BAY) == 0) {
1542 		const char *label_method;
1543 		char *product;
1544 		nvlist_t *args = NULL;
1545 
1546 		if (ses_add_bay_props(mod, tn, snp) != 0)
1547 			goto error;
1548 
1549 		if (topo_method_register(mod, tn, ses_bay_methods) != 0) {
1550 			topo_mod_dprintf(mod,
1551 			    "topo_method_register() failed: %s",
1552 			    topo_mod_errmsg(mod));
1553 			goto error;
1554 		}
1555 
1556 		/*
1557 		 * Ideally we'd perform this sort of override with a platform
1558 		 * specific XML map file, and that would work here if we only
1559 		 * wanted to override the bay node label.  However, we'd also
1560 		 * like the disk node label (if the bay is occupied) to inherit
1561 		 * the overriden bay label.  So we need to ensure the
1562 		 * propmethod is registered before we create the child disk
1563 		 * node.
1564 		 */
1565 		if ((product = topo_mod_product(mod)) == NULL) {
1566 			(void) topo_mod_seterrno(mod, EMOD_NOMEM);
1567 			goto error;
1568 		}
1569 		if ((label_method = lookup_bay_override(product)) != NULL) {
1570 			if (topo_mod_nvalloc(mod, &args, NV_UNIQUE_NAME) != 0 ||
1571 			    topo_prop_method_register(tn, TOPO_PGROUP_PROTOCOL,
1572 			    TOPO_PROP_LABEL, TOPO_TYPE_STRING, label_method,
1573 			    args, &err)) {
1574 				topo_mod_dprintf(mod,
1575 				    "Failed to register method: %s on %s=%"
1576 				    PRIu64, label_method, BAY,
1577 				    topo_node_instance(tn));
1578 				topo_mod_strfree(mod, product);
1579 				nvlist_free(args);
1580 				goto error;
1581 			}
1582 			nvlist_free(args);
1583 		}
1584 		topo_mod_strfree(mod, product);
1585 
1586 		if (ses_create_disk(sdp, tn, props) != 0)
1587 			goto error;
1588 	} else if ((strcmp(nodename, FAN) == 0) ||
1589 	    (strcmp(nodename, PSU) == 0) ||
1590 	    (strcmp(nodename, CONTROLLER) == 0)) {
1591 		/*
1592 		 * Only fan, psu, and controller nodes have a 'present' method.
1593 		 * Bay nodes are always present, and disk nodes are present by
1594 		 * virtue of being enumerated and SAS expander nodes and
1595 		 * SAS connector nodes are also always present once
1596 		 * the parent controller is found.
1597 		 */
1598 		if (topo_method_register(mod, tn, ses_component_methods) != 0) {
1599 			topo_mod_dprintf(mod,
1600 			    "topo_method_register() failed: %s",
1601 			    topo_mod_errmsg(mod));
1602 			goto error;
1603 		}
1604 
1605 	}
1606 
1607 	snp->sen_target->set_refcount++;
1608 	topo_node_setspecific(tn, snp->sen_target);
1609 
1610 	nvlist_free(auth);
1611 	nvlist_free(fmri);
1612 	topo_mod_strfree(mod, clean_label);
1613 	if (node != NULL) *node = tn;
1614 	return (0);
1615 
1616 error:
1617 	nvlist_free(auth);
1618 	nvlist_free(fmri);
1619 	topo_mod_strfree(mod, clean_label);
1620 	return (-1);
1621 }
1622 
1623 /*
1624  * Create SAS expander specific props.
1625  */
1626 /*ARGSUSED*/
1627 static int
1628 ses_set_expander_props(ses_enum_data_t *sdp, ses_enum_node_t *snp,
1629     tnode_t *ptnode, tnode_t *tnode, int *phycount, int64_t *connlist)
1630 {
1631 	ses_node_t *np = snp->sen_node;
1632 	topo_mod_t *mod = sdp->sed_mod;
1633 	nvlist_t *auth = NULL, *fmri = NULL;
1634 	nvlist_t *props, **phylist;
1635 	int err, i;
1636 	uint_t pcount;
1637 	uint64_t sasaddr, connidx;
1638 	char sasaddr_str[17];
1639 	boolean_t found = B_FALSE, ses_found = B_FALSE;
1640 	dev_di_node_t *dnode, *sesdnode;
1641 
1642 	props = ses_node_props(np);
1643 
1644 	/*
1645 	 * the uninstalled expander is not enumerated by checking
1646 	 * the element status code.  No present present' method provided.
1647 	 */
1648 	/*
1649 	 * Get the Expander SAS address.  It should exist.
1650 	 */
1651 	if (nvlist_lookup_uint64(props, SES_EXP_PROP_SAS_ADDR,
1652 	    &sasaddr) != 0) {
1653 		topo_mod_dprintf(mod,
1654 		    "Failed to get prop %s.", SES_EXP_PROP_SAS_ADDR);
1655 		goto error;
1656 	}
1657 
1658 	(void) sprintf(sasaddr_str, "%llx", sasaddr);
1659 
1660 	/* search matching dev_di_node. */
1661 	for (dnode = topo_list_next(&sdp->sed_devs); dnode != NULL;
1662 	    dnode = topo_list_next(dnode)) {
1663 		for (i = 0; i < dnode->ddn_ppath_count; i++) {
1664 			if ((dnode->ddn_target_port[i] != NULL) &&
1665 			    (strstr(dnode->ddn_target_port[i],
1666 			    sasaddr_str) != NULL)) {
1667 				found = B_TRUE;
1668 				break;
1669 			}
1670 		}
1671 		if (found)
1672 			break;
1673 	}
1674 
1675 	if (!found) {
1676 		topo_mod_dprintf(mod,
1677 		    "ses_set_expander_props: Failed to find matching "
1678 		    "devinfo node for Exapnder SAS address %s",
1679 		    SES_EXP_PROP_SAS_ADDR);
1680 		/* continue on to get storage group props. */
1681 	} else {
1682 		/* create/set the devfs-path and devid in the smp group */
1683 		if (topo_pgroup_create(tnode, &smp_pgroup, &err) != 0) {
1684 			topo_mod_dprintf(mod, "ses_set_expander_props: "
1685 			    "failed to create smp property group %s\n",
1686 			    topo_strerror(err));
1687 			goto error;
1688 		} else {
1689 			if (topo_prop_set_string(tnode, TOPO_PGROUP_SMP,
1690 			    TOPO_PROP_SMP_TARGET_PORT, TOPO_PROP_IMMUTABLE,
1691 			    dnode->ddn_target_port[i], &err) != 0) {
1692 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1693 				    "set %S error %s\n", TOPO_PROP_SAS_ADDR,
1694 				    topo_strerror(err));
1695 			}
1696 			if (topo_prop_set_string(tnode, TOPO_PGROUP_SMP,
1697 			    TOPO_PROP_SMP_DEV_PATH, TOPO_PROP_IMMUTABLE,
1698 			    dnode->ddn_dpath, &err) != 0) {
1699 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1700 				    "set dev error %s\n", topo_strerror(err));
1701 			}
1702 			if (topo_prop_set_string(tnode, TOPO_PGROUP_SMP,
1703 			    TOPO_PROP_SMP_DEVID, TOPO_PROP_IMMUTABLE,
1704 			    dnode->ddn_devid, &err) != 0) {
1705 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1706 				    "set devid error %s\n", topo_strerror(err));
1707 			}
1708 			if (dnode->ddn_ppath_count != 0 &&
1709 			    topo_prop_set_string_array(tnode, TOPO_PGROUP_SMP,
1710 			    TOPO_PROP_SMP_PHYS_PATH, TOPO_PROP_IMMUTABLE,
1711 			    (const char **)dnode->ddn_ppath,
1712 			    dnode->ddn_ppath_count, &err) != 0) {
1713 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1714 				    "set phys-path error %s\n",
1715 				    topo_strerror(err));
1716 			}
1717 		}
1718 	}
1719 
1720 	/* update the ses property group with SES target info */
1721 	if ((topo_pgroup_create(tnode, &ses_pgroup, &err) != 0) &&
1722 	    (err != ETOPO_PROP_DEFD)) {
1723 		/* SES prop group doesn't exist but failed to be created. */
1724 		topo_mod_dprintf(mod, "ses_set_expander_props: "
1725 		    "ses pgroup create error %s\n", topo_strerror(err));
1726 		goto error;
1727 	} else {
1728 		/* locate assciated enclosure dev_di_node. */
1729 		for (sesdnode = topo_list_next(&sdp->sed_devs);
1730 		    sesdnode != NULL; sesdnode = topo_list_next(sesdnode)) {
1731 			for (i = 0; i < sesdnode->ddn_ppath_count; i++) {
1732 				/*
1733 				 * check if attached port exists and
1734 				 * its node type is enclosure and
1735 				 * attached port is same as sas address of
1736 				 * the expander and
1737 				 * bridge port for virtual phy indication
1738 				 * exist.
1739 				 */
1740 				if ((sesdnode->ddn_attached_port[i] != NULL) &&
1741 				    (sesdnode->ddn_dtype == DTYPE_ESI) &&
1742 				    (strstr(sesdnode->ddn_attached_port[i],
1743 				    sasaddr_str) != NULL) &&
1744 				    (sesdnode->ddn_bridge_port[i] != NULL)) {
1745 					ses_found = B_TRUE;
1746 					break;
1747 				}
1748 			}
1749 			if (ses_found) break;
1750 		}
1751 
1752 		if (ses_found) {
1753 			if (topo_prop_set_string(tnode, TOPO_PGROUP_SES,
1754 			    TOPO_PROP_SES_TARGET_PORT, TOPO_PROP_IMMUTABLE,
1755 			    sesdnode->ddn_target_port[i], &err) != 0) {
1756 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1757 				    "set ses %S error %s\n", TOPO_PROP_SAS_ADDR,
1758 				    topo_strerror(err));
1759 			}
1760 			if (topo_prop_set_string(tnode, TOPO_PGROUP_SES,
1761 			    TOPO_PROP_SES_DEV_PATH, TOPO_PROP_IMMUTABLE,
1762 			    sesdnode->ddn_dpath, &err) != 0) {
1763 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1764 				    "set ses dev error %s\n",
1765 				    topo_strerror(err));
1766 			}
1767 			if (topo_prop_set_string(tnode, TOPO_PGROUP_SES,
1768 			    TOPO_PROP_SES_DEVID, TOPO_PROP_IMMUTABLE,
1769 			    sesdnode->ddn_devid, &err) != 0) {
1770 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1771 				    "set ses devid error %s\n",
1772 				    topo_strerror(err));
1773 			}
1774 			if (sesdnode->ddn_ppath_count != 0 &&
1775 			    topo_prop_set_string_array(tnode, TOPO_PGROUP_SES,
1776 			    TOPO_PROP_SES_PHYS_PATH, TOPO_PROP_IMMUTABLE,
1777 			    (const char **)sesdnode->ddn_ppath,
1778 			    sesdnode->ddn_ppath_count, &err) != 0) {
1779 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1780 				    "set ses phys-path error %s\n",
1781 				    topo_strerror(err));
1782 			}
1783 
1784 		}
1785 	}
1786 
1787 	/* create the storage group */
1788 	if (topo_pgroup_create(tnode, &storage_pgroup, &err) != 0) {
1789 		topo_mod_dprintf(mod, "ses_set_expander_props: "
1790 		    "create storage error %s\n", topo_strerror(err));
1791 		goto error;
1792 	} else {
1793 		/* set the SAS address prop out of expander element status. */
1794 		if (topo_prop_set_string(tnode, TOPO_PGROUP_STORAGE,
1795 		    TOPO_PROP_SAS_ADDR, TOPO_PROP_IMMUTABLE, sasaddr_str,
1796 		    &err) != 0) {
1797 			topo_mod_dprintf(mod, "ses_set_expander_props: "
1798 			    "set %S error %s\n", TOPO_PROP_SAS_ADDR,
1799 			    topo_strerror(err));
1800 		}
1801 
1802 		/* Get the phy information for the expander */
1803 		if (nvlist_lookup_nvlist_array(props, SES_SAS_PROP_PHYS,
1804 		    &phylist, &pcount) != 0) {
1805 			topo_mod_dprintf(mod,
1806 			    "Failed to get prop %s.", SES_SAS_PROP_PHYS);
1807 		} else {
1808 			/*
1809 			 * For each phy, get the connector element index and
1810 			 * stores into connector element index array.
1811 			 */
1812 			*phycount = pcount;
1813 			for (i = 0; i < pcount; i++) {
1814 				if (nvlist_lookup_uint64(phylist[i],
1815 				    SES_PROP_CE_IDX, &connidx) == 0) {
1816 					if (connidx != 0xff) {
1817 						connlist[i] = connidx;
1818 					} else {
1819 						connlist[i] = -1;
1820 					}
1821 				} else {
1822 					/* Fail to get the index. set to -1. */
1823 					connlist[i] = -1;
1824 				}
1825 			}
1826 
1827 			/* set the phy count prop of the expander. */
1828 			if (topo_prop_set_uint64(tnode, TOPO_PGROUP_STORAGE,
1829 			    TOPO_PROP_PHY_COUNT, TOPO_PROP_IMMUTABLE, pcount,
1830 			    &err) != 0) {
1831 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1832 				    "set %S error %s\n", TOPO_PROP_PHY_COUNT,
1833 				    topo_strerror(err));
1834 			}
1835 
1836 			/*
1837 			 * set the connector element index of
1838 			 * the expander phys.
1839 			 */
1840 		}
1841 
1842 		/* populate other misc storage group properties */
1843 		if (found) {
1844 			if (dnode->ddn_mfg && (topo_prop_set_string(tnode,
1845 			    TOPO_PGROUP_STORAGE, TOPO_STORAGE_MANUFACTURER,
1846 			    TOPO_PROP_IMMUTABLE, dnode->ddn_mfg, &err) != 0)) {
1847 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1848 				    "set mfg error %s\n", topo_strerror(err));
1849 			}
1850 
1851 			if (dnode->ddn_model && (topo_prop_set_string(tnode,
1852 			    TOPO_PGROUP_STORAGE, TOPO_STORAGE_MODEL,
1853 			    TOPO_PROP_IMMUTABLE,
1854 			    dnode->ddn_model, &err) != 0)) {
1855 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1856 				    "set model error %s\n", topo_strerror(err));
1857 			}
1858 
1859 			if (dnode->ddn_serial && (topo_prop_set_string(tnode,
1860 			    TOPO_PGROUP_STORAGE, TOPO_STORAGE_SERIAL_NUM,
1861 			    TOPO_PROP_IMMUTABLE,
1862 			    dnode->ddn_serial, &err) != 0)) {
1863 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1864 				    "set serial error %s\n",
1865 				    topo_strerror(err));
1866 			}
1867 
1868 			if (dnode->ddn_firm && (topo_prop_set_string(tnode,
1869 			    TOPO_PGROUP_STORAGE,
1870 			    TOPO_STORAGE_FIRMWARE_REV, TOPO_PROP_IMMUTABLE,
1871 			    dnode->ddn_firm, &err) != 0)) {
1872 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1873 				    "set firm error %s\n", topo_strerror(err));
1874 			}
1875 		}
1876 	}
1877 
1878 	return (0);
1879 
1880 error:
1881 	nvlist_free(auth);
1882 	nvlist_free(fmri);
1883 	return (-1);
1884 }
1885 
1886 /*
1887  * Create SAS expander specific props.
1888  */
1889 /*ARGSUSED*/
1890 static int
1891 ses_set_connector_props(ses_enum_data_t *sdp, ses_enum_node_t *snp,
1892     tnode_t *tnode, int64_t phy_mask)
1893 {
1894 	ses_node_t *np = snp->sen_node;
1895 	topo_mod_t *mod = sdp->sed_mod;
1896 	nvlist_t *props;
1897 	int err, i;
1898 	uint64_t conntype;
1899 	char phymask_str[17], *conntype_str;
1900 	boolean_t   found;
1901 
1902 	props = ses_node_props(np);
1903 
1904 	/*
1905 	 * convert phy mask to string.
1906 	 */
1907 	(void) snprintf(phymask_str, 17, "%llx", phy_mask);
1908 
1909 	/* create the storage group */
1910 	if (topo_pgroup_create(tnode, &storage_pgroup, &err) != 0) {
1911 		topo_mod_dprintf(mod, "ses_set_expander_props: "
1912 		    "create storage error %s\n", topo_strerror(err));
1913 		return (-1);
1914 	} else {
1915 		/* set the SAS address prop of the expander. */
1916 		if (topo_prop_set_string(tnode, TOPO_PGROUP_STORAGE,
1917 		    TOPO_STORAGE_SAS_PHY_MASK, TOPO_PROP_IMMUTABLE,
1918 		    phymask_str, &err) != 0) {
1919 			topo_mod_dprintf(mod, "ses_set_expander_props: "
1920 			    "set %S error %s\n", TOPO_STORAGE_SAS_PHY_MASK,
1921 			    topo_strerror(err));
1922 		}
1923 
1924 		/* Get the connector type information for the expander */
1925 		if (nvlist_lookup_uint64(props,
1926 		    SES_SC_PROP_CONNECTOR_TYPE, &conntype) != 0) {
1927 			topo_mod_dprintf(mod, "Failed to get prop %s.",
1928 			    TOPO_STORAGE_SAS_PHY_MASK);
1929 		} else {
1930 			found = B_FALSE;
1931 			for (i = 0; ; i++) {
1932 				if (sas_connector_type_list[i].sct_type ==
1933 				    SAS_CONNECTOR_TYPE_CODE_NOT_DEFINED) {
1934 					break;
1935 				}
1936 				if (sas_connector_type_list[i].sct_type ==
1937 				    conntype) {
1938 					conntype_str =
1939 					    sas_connector_type_list[i].sct_name;
1940 					found = B_TRUE;
1941 					break;
1942 				}
1943 			}
1944 
1945 			if (!found) {
1946 				if (conntype <
1947 				    SAS_CONNECTOR_TYPE_CODE_NOT_DEFINED) {
1948 					conntype_str =
1949 					    SAS_CONNECTOR_TYPE_RESERVED;
1950 				} else {
1951 					conntype_str =
1952 					    SAS_CONNECTOR_TYPE_NOT_DEFINED;
1953 				}
1954 			}
1955 
1956 			/* set the phy count prop of the expander. */
1957 			if (topo_prop_set_string(tnode, TOPO_PGROUP_STORAGE,
1958 			    TOPO_STORAGE_SAS_CONNECTOR_TYPE,
1959 			    TOPO_PROP_IMMUTABLE, conntype_str, &err) != 0) {
1960 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1961 				    "set %S error %s\n", TOPO_PROP_PHY_COUNT,
1962 				    topo_strerror(err));
1963 			}
1964 		}
1965 	}
1966 
1967 	return (0);
1968 }
1969 
1970 /*
1971  * Instantiate SAS expander nodes for a given ESC Electronics node(controller)
1972  * nodes.
1973  */
1974 /*ARGSUSED*/
1975 static int
1976 ses_create_esc_sasspecific(ses_enum_data_t *sdp, ses_enum_node_t *snp,
1977     tnode_t *pnode, ses_enum_chassis_t *cp,
1978     boolean_t dorange)
1979 {
1980 	topo_mod_t *mod = sdp->sed_mod;
1981 	tnode_t	*exptn, *contn;
1982 	boolean_t found;
1983 	sas_connector_phy_data_t connectors[64] = {0};
1984 	uint64_t max;
1985 	ses_enum_node_t *ctlsnp, *xsnp, *consnp;
1986 	ses_node_t *np = snp->sen_node;
1987 	nvlist_t *props, *psprops;
1988 	uint64_t index, psindex, conindex, psstatus, i, j, count;
1989 	int64_t cidxlist[256] = {0};
1990 	int phycount;
1991 
1992 	props = ses_node_props(np);
1993 
1994 	if (nvlist_lookup_uint64(props, SES_PROP_ELEMENT_ONLY_INDEX,
1995 	    &index) != 0)
1996 		return (-1);
1997 
1998 	/*
1999 	 * For SES constroller node, check to see if there are
2000 	 * associated SAS expanders.
2001 	 */
2002 	found = B_FALSE;
2003 	max = 0;
2004 	for (ctlsnp = topo_list_next(&cp->sec_nodes); ctlsnp != NULL;
2005 	    ctlsnp = topo_list_next(ctlsnp)) {
2006 		if (ctlsnp->sen_type == SES_ET_SAS_EXPANDER) {
2007 			found = B_TRUE;
2008 			if (ctlsnp->sen_instance > max)
2009 				max = ctlsnp->sen_instance;
2010 		}
2011 	}
2012 
2013 	/*
2014 	 * No SAS expander found notthing to process.
2015 	 */
2016 	if (!found)
2017 		return (0);
2018 
2019 	topo_mod_dprintf(mod, "%s Controller %d: creating "
2020 	    "%llu %s nodes", cp->sec_csn, index, max + 1, SASEXPANDER);
2021 
2022 	/*
2023 	 * The max number represent the number of elements
2024 	 * deducted from the highest SES_PROP_ELEMENT_CLASS_INDEX
2025 	 * of SET_ET_SAS_EXPANDER type element.
2026 	 *
2027 	 * There may be multiple ESC Electronics element(controllers)
2028 	 * within JBOD(typicall two for redundancy) and SAS expander
2029 	 * elements are associated with only one of them.  We are
2030 	 * still creating the range based max number here.
2031 	 * That will cover the case that all expanders are associated
2032 	 * with one SES controller.
2033 	 */
2034 	if (dorange && topo_node_range_create(mod, pnode,
2035 	    SASEXPANDER, 0, max) != 0) {
2036 		topo_mod_dprintf(mod,
2037 		    "topo_node_create_range() failed: %s",
2038 		    topo_mod_errmsg(mod));
2039 		return (-1);
2040 	}
2041 
2042 	/*
2043 	 * Search exapnders with the parent index matching with
2044 	 * ESC Electronics element index.
2045 	 * Note the index used here is a global index across
2046 	 * SES elements.
2047 	 */
2048 	for (xsnp = topo_list_next(&cp->sec_nodes); xsnp != NULL;
2049 	    xsnp = topo_list_next(xsnp)) {
2050 		if (xsnp->sen_type == SES_ET_SAS_EXPANDER) {
2051 			/*
2052 			 * get the parent ESC controller.
2053 			 */
2054 			psprops = ses_node_props(xsnp->sen_node);
2055 			if (nvlist_lookup_uint64(psprops,
2056 			    SES_PROP_STATUS_CODE, &psstatus) == 0) {
2057 				if (psstatus == SES_ESC_NOT_INSTALLED) {
2058 					/*
2059 					 * Not installed.
2060 					 * Don't create a ndoe.
2061 					 */
2062 					continue;
2063 				}
2064 			} else {
2065 				/*
2066 				 * The element should have status code.
2067 				 * If not there is no way to find
2068 				 * out if the expander element exist or
2069 				 * not.
2070 				 */
2071 				continue;
2072 			}
2073 
2074 			/* Get the physical parent index to compare. */
2075 			if (nvlist_lookup_uint64(psprops,
2076 			    LIBSES_PROP_PHYS_PARENT, &psindex) == 0) {
2077 				if (index == psindex) {
2078 		/* indentation moved forward */
2079 		/*
2080 		 * Handle basic node information of SAS expander
2081 		 * element - binding to parent node and
2082 		 * allocating FMRI...
2083 		 */
2084 		if (ses_create_generic(sdp, xsnp, pnode, pnode, SASEXPANDER,
2085 		    "SAS-EXPANDER", &exptn) != 0)
2086 			continue;
2087 		/*
2088 		 * Now handle SAS expander unique portion of node creation.
2089 		 * The max nubmer of the phy count is 256 since SES-2
2090 		 * defines as 1 byte field.  The cidxlist has the same
2091 		 * number of elements.
2092 		 *
2093 		 * We use size 64 array to store the connectors.
2094 		 * Typically a connectors associated with 4 phys so that
2095 		 * matches with the max number of connecters associated
2096 		 * with an expander.
2097 		 * The phy count goes up to 38 for Sun supported
2098 		 * JBOD.
2099 		 */
2100 		(void) memset(cidxlist, 0, sizeof (int64_t) * 64);
2101 		if (ses_set_expander_props(sdp, xsnp, pnode, exptn, &phycount,
2102 		    cidxlist) != 0) {
2103 			/*
2104 			 * error on getting specific prop failed.
2105 			 * continue on.  Note that the node is
2106 			 * left bound.
2107 			 */
2108 			continue;
2109 		}
2110 
2111 		/*
2112 		 * count represetns the number of connectors discovered so far.
2113 		 */
2114 		count = 0;
2115 		(void) memset(connectors, 0,
2116 		    sizeof (sas_connector_phy_data_t) * 64);
2117 		for (i = 0; i < phycount; i++) {
2118 			if (cidxlist[i] != -1) {
2119 				/* connector index is valid. */
2120 				for (j = 0; j < count; j++) {
2121 					if (connectors[j].scpd_index ==
2122 					    cidxlist[i]) {
2123 						/*
2124 						 * Just update phy mask.
2125 						 * The postion for connector
2126 						 * index lists(cidxlist index)
2127 						 * is set.
2128 						 */
2129 						connectors[j].scpd_pm =
2130 						    connectors[j].scpd_pm |
2131 						    (1ULL << i);
2132 						break;
2133 					}
2134 				}
2135 				/*
2136 				 * If j and count matche a  new connector
2137 				 * index is found.
2138 				 */
2139 				if (j == count) {
2140 					/* add a new index and phy mask. */
2141 					connectors[count].scpd_index =
2142 					    cidxlist[i];
2143 					connectors[count].scpd_pm =
2144 					    connectors[count].scpd_pm |
2145 					    (1ULL << i);
2146 					count++;
2147 				}
2148 			}
2149 		}
2150 
2151 		/*
2152 		 * create range for the connector nodes.
2153 		 * The class index of the ses connector element
2154 		 * is set as the instance nubmer for the node.
2155 		 * Even though one expander may not have all connectors
2156 		 * are associated with we are creating the range with
2157 		 * max possible instance number.
2158 		 */
2159 		found = B_FALSE;
2160 		max = 0;
2161 		for (consnp = topo_list_next(&cp->sec_nodes);
2162 		    consnp != NULL; consnp = topo_list_next(consnp)) {
2163 			if (consnp->sen_type == SES_ET_SAS_CONNECTOR) {
2164 				psprops = ses_node_props(consnp->sen_node);
2165 				found = B_TRUE;
2166 				if (consnp->sen_instance > max)
2167 					max = consnp->sen_instance;
2168 			}
2169 		}
2170 
2171 		/*
2172 		 * No SAS connector found nothing to process.
2173 		 */
2174 		if (!found)
2175 			return (0);
2176 
2177 		if (dorange && topo_node_range_create(mod, exptn,
2178 		    RECEPTACLE, 0, max) != 0) {
2179 			topo_mod_dprintf(mod,
2180 			    "topo_node_create_range() failed: %s",
2181 			    topo_mod_errmsg(mod));
2182 			return (-1);
2183 		}
2184 
2185 		/* search matching connector element using the index. */
2186 		for (i = 0; i < count; i++) {
2187 			found = B_FALSE;
2188 			for (consnp = topo_list_next(&cp->sec_nodes);
2189 			    consnp != NULL; consnp = topo_list_next(consnp)) {
2190 				if (consnp->sen_type == SES_ET_SAS_CONNECTOR) {
2191 					psprops = ses_node_props(
2192 					    consnp->sen_node);
2193 					/*
2194 					 * Get the physical parent index to
2195 					 * compare.
2196 					 * The connector elements are children
2197 					 * of ESC Electronics element even
2198 					 * though we enumerate them under
2199 					 * an expander in libtopo.
2200 					 */
2201 					if (nvlist_lookup_uint64(psprops,
2202 					    SES_PROP_ELEMENT_ONLY_INDEX,
2203 					    &conindex) == 0) {
2204 						if (conindex ==
2205 						    connectors[i].scpd_index) {
2206 							found = B_TRUE;
2207 							break;
2208 						}
2209 					}
2210 				}
2211 			}
2212 
2213 			/* now create a libtopo node. */
2214 			if (found) {
2215 				/* Create generic props. */
2216 				if (ses_create_generic(sdp, consnp, exptn,
2217 				    topo_node_parent(exptn),
2218 				    RECEPTACLE, "RECEPTACLE", &contn) !=
2219 				    0) {
2220 					continue;
2221 				}
2222 				/* Create connector specific props. */
2223 				if (ses_set_connector_props(sdp, consnp,
2224 				    contn, connectors[i].scpd_pm) != 0) {
2225 					continue;
2226 				}
2227 				if (topo_method_register(mod, contn,
2228 				    ses_recep_methods) != 0) {
2229 					topo_mod_dprintf(mod,
2230 					    "topo_method_register() failed: "
2231 					    "%s",
2232 					    topo_mod_errmsg(mod));
2233 					continue;
2234 				}
2235 
2236 			}
2237 		}
2238 		/* end indentation change */
2239 				}
2240 			}
2241 		}
2242 	}
2243 
2244 	return (0);
2245 }
2246 
2247 /*
2248  * Instantiate any protocol specific portion of a node.
2249  */
2250 /*ARGSUSED*/
2251 static int
2252 ses_create_protocol_specific(ses_enum_data_t *sdp, ses_enum_node_t *snp,
2253     tnode_t *pnode, uint64_t type, ses_enum_chassis_t *cp,
2254     boolean_t dorange)
2255 {
2256 
2257 	if (type == SES_ET_ESC_ELECTRONICS) {
2258 		/* create SAS specific children(expanders and connectors. */
2259 		return (ses_create_esc_sasspecific(sdp, snp, pnode, cp,
2260 		    dorange));
2261 	}
2262 
2263 	return (0);
2264 }
2265 
2266 /*
2267  * Instantiate any children of a given type.
2268  */
2269 static int
2270 ses_create_children(ses_enum_data_t *sdp, tnode_t *pnode, uint64_t type,
2271     const char *nodename, const char *defaultlabel, ses_enum_chassis_t *cp,
2272     boolean_t dorange)
2273 {
2274 	topo_mod_t *mod = sdp->sed_mod;
2275 	boolean_t found;
2276 	uint64_t max;
2277 	ses_enum_node_t *snp;
2278 	tnode_t	*tn;
2279 
2280 	/*
2281 	 * First go through and count how many matching nodes we have.
2282 	 */
2283 	max = 0;
2284 	found = B_FALSE;
2285 	for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
2286 	    snp = topo_list_next(snp)) {
2287 		if (snp->sen_type == type) {
2288 			found = B_TRUE;
2289 			if (snp->sen_instance > max)
2290 				max = snp->sen_instance;
2291 		}
2292 	}
2293 
2294 	/*
2295 	 * No enclosure should export both DEVICE and ARRAY_DEVICE elements.
2296 	 * Since we map both of these to 'disk', if an enclosure does this, we
2297 	 * just ignore the array elements.
2298 	 */
2299 	if (!found ||
2300 	    (type == SES_ET_ARRAY_DEVICE && cp->sec_hasdev))
2301 		return (0);
2302 
2303 	topo_mod_dprintf(mod, "%s: creating %llu %s nodes",
2304 	    cp->sec_csn, max + 1, nodename);
2305 
2306 	if (dorange && topo_node_range_create(mod, pnode,
2307 	    nodename, 0, max) != 0) {
2308 		topo_mod_dprintf(mod,
2309 		    "topo_node_create_range() failed: %s",
2310 		    topo_mod_errmsg(mod));
2311 		return (-1);
2312 	}
2313 
2314 	for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
2315 	    snp = topo_list_next(snp)) {
2316 		if (snp->sen_type == type) {
2317 			/*
2318 			 * With flat layout of ses nodes there is no
2319 			 * way to find out the direct FRU for a node.
2320 			 * Passing NULL for fru topo node.  Note that
2321 			 * ses_create_children_from_phys_tree() provides
2322 			 * the actual direct FRU for a node.
2323 			 */
2324 			if (ses_create_generic(sdp, snp, pnode, NULL,
2325 			    nodename, defaultlabel, &tn) != 0)
2326 				return (-1);
2327 			/*
2328 			 * For some SES element there may be protocol specific
2329 			 * information to process.   Here we are processing
2330 			 * the association between enclosure controller and
2331 			 * SAS expanders.
2332 			 */
2333 			if (type == SES_ET_ESC_ELECTRONICS) {
2334 				/* create SAS expander node */
2335 				if (ses_create_protocol_specific(sdp, snp,
2336 				    tn, type, cp, dorange) != 0) {
2337 					return (-1);
2338 				}
2339 			}
2340 
2341 		}
2342 	}
2343 
2344 	return (0);
2345 }
2346 
2347 /*
2348  * Instantiate a new subchassis instance in the topology.
2349  */
2350 static int
2351 ses_create_subchassis(ses_enum_data_t *sdp, tnode_t *pnode,
2352     ses_enum_chassis_t *scp)
2353 {
2354 	topo_mod_t *mod = sdp->sed_mod;
2355 	tnode_t *tn;
2356 	nvlist_t *props;
2357 	nvlist_t *auth = NULL, *fmri = NULL;
2358 	uint64_t instance = scp->sec_instance;
2359 	char *desc;
2360 	char *clean_label = NULL, label[128];
2361 	char **paths;
2362 	int i, err;
2363 	ses_enum_target_t *stp;
2364 	int ret = -1;
2365 
2366 	/*
2367 	 * Copy authority information from parent enclosure node
2368 	 */
2369 	if ((auth = topo_mod_auth(mod, pnode)) == NULL)
2370 		goto error;
2371 
2372 	/*
2373 	 * Record the subchassis serial number in the FMRI.
2374 	 * For now, we assume that logical id is the subchassis serial number.
2375 	 * If this assumption changes in future, then the following
2376 	 * piece of code will need to be updated via an RFE.
2377 	 */
2378 	if ((fmri = topo_mod_hcfmri(mod, pnode, FM_HC_SCHEME_VERSION,
2379 	    SUBCHASSIS, (topo_instance_t)instance, NULL, auth, NULL, NULL,
2380 	    NULL)) == NULL) {
2381 		topo_mod_dprintf(mod, "topo_mod_hcfmri() failed: %s",
2382 		    topo_mod_errmsg(mod));
2383 		goto error;
2384 	}
2385 
2386 	if ((tn = topo_node_bind(mod, pnode, SUBCHASSIS,
2387 	    instance, fmri)) == NULL) {
2388 		topo_mod_dprintf(mod, "topo_node_bind() failed: %s",
2389 		    topo_mod_errmsg(mod));
2390 		goto error;
2391 	}
2392 
2393 	props = ses_node_props(scp->sec_enclosure);
2394 
2395 	/*
2396 	 * Look for the subchassis label in the following order:
2397 	 *	<ses-description>
2398 	 *	<ses-class-description> <instance>
2399 	 *	<default-type-label> <instance>
2400 	 *
2401 	 * For subchassis, the default label is "SUBCHASSIS"
2402 	 */
2403 	if (nvlist_lookup_string(props, SES_PROP_DESCRIPTION, &desc) != 0 ||
2404 	    desc[0] == '\0') {
2405 		if (nvlist_lookup_string(props, SES_PROP_CLASS_DESCRIPTION,
2406 		    &desc) == 0 && desc[0] != '\0')
2407 			(void) snprintf(label, sizeof (label), "%s %llu", desc,
2408 			    instance);
2409 		else
2410 			(void) snprintf(label, sizeof (label),
2411 			    "SUBCHASSIS %llu", instance);
2412 		desc = label;
2413 	}
2414 
2415 	if ((clean_label = topo_mod_clean_str(mod, desc)) == NULL)
2416 		goto error;
2417 
2418 	if (topo_prop_set_string(tn, TOPO_PGROUP_PROTOCOL, TOPO_PROP_LABEL,
2419 	    TOPO_PROP_MUTABLE, clean_label, &err) < 0)
2420 		goto error;
2421 
2422 	if (ses_set_standard_props(mod, NULL, tn, NULL,
2423 	    ses_node_id(scp->sec_enclosure), scp->sec_target->set_devpath) != 0)
2424 		goto error;
2425 
2426 	/*
2427 	 * Set the 'chassis-type' property for this subchassis.  This is either
2428 	 * 'ses-class-description' or 'subchassis'.
2429 	 */
2430 	if (nvlist_lookup_string(props, SES_PROP_CLASS_DESCRIPTION, &desc) != 0)
2431 		desc = "subchassis";
2432 
2433 	if (topo_prop_set_string(tn, TOPO_PGROUP_SES,
2434 	    TOPO_PROP_CHASSIS_TYPE, TOPO_PROP_IMMUTABLE, desc, &err) != 0) {
2435 		topo_mod_dprintf(mod, "failed to create property %s: %s\n",
2436 		    TOPO_PROP_CHASSIS_TYPE, topo_strerror(err));
2437 		goto error;
2438 	}
2439 
2440 	/*
2441 	 * For enclosures, we want to include all possible targets (for upgrade
2442 	 * purposes).
2443 	 */
2444 	for (i = 0, stp = topo_list_next(&scp->sec_targets); stp != NULL;
2445 	    stp = topo_list_next(stp), i++)
2446 		;
2447 
2448 	verify(i != 0);
2449 	paths = alloca(i * sizeof (char *));
2450 
2451 	for (i = 0, stp = topo_list_next(&scp->sec_targets); stp != NULL;
2452 	    stp = topo_list_next(stp), i++)
2453 		paths[i] = stp->set_devpath;
2454 
2455 	if (topo_prop_set_string_array(tn, TOPO_PGROUP_SES,
2456 	    TOPO_PROP_PATHS, TOPO_PROP_IMMUTABLE, (const char **)paths,
2457 	    i, &err) != 0) {
2458 		topo_mod_dprintf(mod, "failed to create property %s: %s\n",
2459 		    TOPO_PROP_PATHS, topo_strerror(err));
2460 		goto error;
2461 	}
2462 
2463 	if (topo_method_register(mod, tn, ses_enclosure_methods) != 0) {
2464 		topo_mod_dprintf(mod, "topo_method_register() failed: %s",
2465 		    topo_mod_errmsg(mod));
2466 		goto error;
2467 	}
2468 
2469 	/*
2470 	 * Create the nodes for controllers and bays.
2471 	 */
2472 	if (ses_create_children(sdp, tn, SES_ET_ESC_ELECTRONICS,
2473 	    CONTROLLER, "CONTROLLER", scp, B_TRUE) != 0 ||
2474 	    ses_create_children(sdp, tn, SES_ET_DEVICE,
2475 	    BAY, "BAY", scp, B_TRUE) != 0 ||
2476 	    ses_create_children(sdp, tn, SES_ET_ARRAY_DEVICE,
2477 	    BAY, "BAY", scp, B_TRUE) != 0)
2478 		goto error;
2479 
2480 	ret = 0;
2481 
2482 error:
2483 	nvlist_free(auth);
2484 	nvlist_free(fmri);
2485 	topo_mod_strfree(mod, clean_label);
2486 	return (ret);
2487 }
2488 
2489 /*
2490  * Function we use to insert a node.
2491  */
2492 static int
2493 ses_phys_tree_insert(topo_mod_t *mod, ses_phys_tree_t **sproot,
2494     ses_phys_tree_t *child)
2495 {
2496 	uint64_t ppindex, eindex, pindex;
2497 	ses_phys_tree_t *node_ptr;
2498 	int ret = 0;
2499 
2500 	assert(sproot != NULL);
2501 	assert(child != NULL);
2502 
2503 	if (*sproot == NULL) {
2504 		*sproot = child;
2505 		return (0);
2506 	}
2507 
2508 	pindex = child->spt_pindex;
2509 	ppindex = (*sproot)->spt_pindex;
2510 	eindex = (*sproot)->spt_eonlyindex;
2511 
2512 	/*
2513 	 * If the element only index of the root is same as the physical
2514 	 * parent index of a node to be added, add the node as a child of
2515 	 * the current root.
2516 	 */
2517 	if (eindex == pindex) {
2518 		(void) ses_phys_tree_insert(mod, &(*sproot)->spt_child, child);
2519 		child->spt_parent = *sproot;
2520 	} else if (ppindex == pindex) {
2521 		/*
2522 		 * if the physical parent of the current root and the child
2523 		 * is same, then this should be a sibling node.
2524 		 * Siblings can be different element types and arrange
2525 		 * them by group.
2526 		 */
2527 		if ((*sproot)->spt_senumnode->sen_type ==
2528 		    child->spt_senumnode->sen_type) {
2529 			child->spt_sibling = *sproot;
2530 			*sproot = child;
2531 		} else {
2532 			/* add a node in front of matching element type. */
2533 			node_ptr = *sproot;
2534 			while (node_ptr->spt_sibling != NULL) {
2535 				if (node_ptr->spt_sibling->
2536 				    spt_senumnode->sen_type ==
2537 				    child->spt_senumnode->sen_type) {
2538 					child->spt_sibling =
2539 					    node_ptr->spt_sibling;
2540 					node_ptr->spt_sibling = child;
2541 					break;
2542 				}
2543 				node_ptr = node_ptr->spt_sibling;
2544 			}
2545 			/* no matching.  Add the child at the end. */
2546 			if (node_ptr->spt_sibling == NULL) {
2547 				node_ptr->spt_sibling = child;
2548 			}
2549 		}
2550 		child->spt_parent = (*sproot)->spt_parent;
2551 	} else {
2552 		/*
2553 		 * The root and the node is not directly related.
2554 		 * Try to insert to the child sub-tree first and then try to
2555 		 * insert to the sibling sub-trees.  If fails for both
2556 		 * the caller will retry insertion later.
2557 		 */
2558 		if ((*sproot)->spt_child) {
2559 			ret = ses_phys_tree_insert(mod, &(*sproot)->spt_child,
2560 			    child);
2561 		}
2562 		if ((*sproot)->spt_child == NULL || ret != 0) {
2563 			if ((*sproot)->spt_sibling) {
2564 				ret = ses_phys_tree_insert(mod,
2565 				    &(*sproot)->spt_sibling, child);
2566 			} else {
2567 				ret = 1;
2568 			}
2569 		}
2570 		return (ret);
2571 	}
2572 	return (0);
2573 }
2574 
2575 /*
2576  * Construct tree view of ses elements through parent phyiscal element index.
2577  * The root of tree is already constructed using the enclosure element.
2578  */
2579 static int
2580 ses_construct_phys_tree(ses_enum_data_t *sdp, ses_enum_chassis_t *cp,
2581     ses_phys_tree_t *sproot)
2582 {
2583 	ses_enum_node_t *snp;
2584 	ses_phys_tree_t	*child;
2585 	ses_phys_tree_t	*u_watch = NULL;
2586 	ses_phys_tree_t	*u_head = NULL;
2587 	ses_phys_tree_t	*u_tail = NULL;
2588 	int u_inserted = 0, u_left = 0;
2589 	nvlist_t *props;
2590 	topo_mod_t *mod = sdp->sed_mod;
2591 
2592 	for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
2593 	    snp = topo_list_next(snp)) {
2594 		if ((child = topo_mod_zalloc(mod,
2595 		    sizeof (ses_phys_tree_t))) == NULL) {
2596 			topo_mod_dprintf(mod,
2597 			    "failed to allocate root.");
2598 			return (-1);
2599 		}
2600 		child->spt_snode = snp->sen_node;
2601 		props = ses_node_props(snp->sen_node);
2602 		if (nvlist_lookup_uint64(props,
2603 		    LIBSES_PROP_PHYS_PARENT, &child->spt_pindex) != 0) {
2604 			/*
2605 			 * the prop should exist. continue to see if
2606 			 * we can build a partial tree with other elements.
2607 			 */
2608 			topo_mod_dprintf(mod,
2609 			    "ses_construct_phys_tree(): Failed to find prop %s "
2610 			    "on ses element type %d and instance %d "
2611 			    "(CSN %s).", LIBSES_PROP_PHYS_PARENT,
2612 			    snp->sen_type, snp->sen_instance, cp->sec_csn);
2613 			topo_mod_free(mod, child, sizeof (ses_phys_tree_t));
2614 			continue;
2615 		} else {
2616 			if (nvlist_lookup_boolean_value(props,
2617 			    LIBSES_PROP_FRU, &child->spt_isfru) != 0) {
2618 				topo_mod_dprintf(mod,
2619 				    "ses_construct_phys_tree(): Failed to "
2620 				    "find prop %s on ses element type %d "
2621 				    "and instance %d (CSN %s).",
2622 				    LIBSES_PROP_FRU,
2623 				    snp->sen_type, snp->sen_instance,
2624 				    cp->sec_csn);
2625 				/*
2626 				 * Ignore if the prop doesn't exist.
2627 				 * Note that the enclosure itself should be
2628 				 * a FRU so if no FRU found the enclosure FRU
2629 				 * can be a direct FRU.
2630 				 */
2631 			}
2632 			verify(nvlist_lookup_uint64(props,
2633 			    SES_PROP_ELEMENT_ONLY_INDEX,
2634 			    &child->spt_eonlyindex) == 0);
2635 			verify(nvlist_lookup_uint64(props,
2636 			    SES_PROP_ELEMENT_CLASS_INDEX,
2637 			    &child->spt_cindex) == 0);
2638 		}
2639 		child->spt_senumnode = snp;
2640 		if (ses_phys_tree_insert(mod, &sproot, child) != 0) {
2641 			/* collect unresolved element to process later. */
2642 			if (u_head == NULL) {
2643 				u_head = child;
2644 				u_tail = child;
2645 			} else {
2646 				child->spt_sibling = u_head;
2647 				u_head = child;
2648 			}
2649 		}
2650 	}
2651 
2652 	/*
2653 	 * The parent of a child node may not be inserted yet.
2654 	 * Trying to insert the child until no child is left or
2655 	 * no child is not added further.  For the latter
2656 	 * the hierarchical relationship between elements
2657 	 * should be checked through SUNW,FRUID page.
2658 	 * u_watch is a watch dog to check the prgress of unresolved
2659 	 * node.
2660 	 */
2661 	u_watch = u_tail;
2662 	while (u_head) {
2663 		child = u_head;
2664 		u_head = u_head->spt_sibling;
2665 		if (u_head == NULL)
2666 			u_tail = NULL;
2667 		child->spt_sibling = NULL;
2668 		if (ses_phys_tree_insert(mod, &sproot, child) != 0) {
2669 			u_tail->spt_sibling = child;
2670 			u_tail = child;
2671 			if (child == u_watch) {
2672 				/*
2673 				 * We just scanned one round for the
2674 				 * unresolved list. Check to see whether we
2675 				 * have nodes inserted, if none, we should
2676 				 * break in case of an indefinite loop.
2677 				 */
2678 				if (u_inserted == 0) {
2679 					/*
2680 					 * Indicate there is unhandled node.
2681 					 * Chain free the whole unsolved
2682 					 * list here.
2683 					 */
2684 					u_left++;
2685 					break;
2686 				} else {
2687 					u_inserted = 0;
2688 					u_watch = u_tail;
2689 				}
2690 			}
2691 		} else {
2692 			/*
2693 			 * We just inserted one rpnode, increment the
2694 			 * unsolved_inserted counter. We will utilize this
2695 			 * counter to detect an indefinite insertion loop.
2696 			 */
2697 			u_inserted++;
2698 			if (child == u_watch) {
2699 				/*
2700 				 * watch dog node itself is inserted.
2701 				 * Set it to the tail and refresh the watching.
2702 				 */
2703 				u_watch = u_tail;
2704 				u_inserted = 0;
2705 				u_left = 0;
2706 			}
2707 		}
2708 	}
2709 
2710 	/* check if there is left out unresolved nodes. */
2711 	if (u_left) {
2712 		topo_mod_dprintf(mod, "ses_construct_phys_tree(): "
2713 		    "Failed to construct physical view of the following "
2714 		    "ses elements of Chassis CSN %s.", cp->sec_csn);
2715 		while (u_head) {
2716 			u_tail = u_head->spt_sibling;
2717 			topo_mod_dprintf(mod,
2718 			    "\telement type (%d) and instance (%d)",
2719 			    u_head->spt_senumnode->sen_type,
2720 			    u_head->spt_senumnode->sen_instance);
2721 			topo_mod_free(mod, u_head, sizeof (ses_phys_tree_t));
2722 			u_head = u_tail;
2723 		}
2724 		return (-1);
2725 	}
2726 
2727 	return (0);
2728 }
2729 
2730 /*
2731  * Free the whole phys tree.
2732  */
2733 static void ses_phys_tree_free(topo_mod_t *mod, ses_phys_tree_t *sproot)
2734 {
2735 	if (sproot == NULL)
2736 		return;
2737 
2738 	/* Free child tree. */
2739 	if (sproot->spt_child) {
2740 		ses_phys_tree_free(mod, sproot->spt_child);
2741 	}
2742 
2743 	/* Free sibling trees. */
2744 	if (sproot->spt_sibling) {
2745 		ses_phys_tree_free(mod, sproot->spt_sibling);
2746 	}
2747 
2748 	/* Free root node itself. */
2749 	topo_mod_free(mod, sproot, sizeof (ses_phys_tree_t));
2750 }
2751 
2752 /*
2753  * Parses phys_enum_type table to get the index of the given type.
2754  */
2755 static boolean_t
2756 is_type_enumerated(ses_phys_tree_t *node, int *index)
2757 {
2758 	int i;
2759 
2760 	for (i = 0; i < N_PHYS_ENUM_TYPES; i++) {
2761 		if (node->spt_senumnode->sen_type ==
2762 		    phys_enum_type_list[i].pet_type) {
2763 			*index = i;
2764 			return (B_TRUE);
2765 		}
2766 	}
2767 	return (B_FALSE);
2768 }
2769 
2770 /*
2771  * Recusrive routine for top-down enumeration of the tree.
2772  */
2773 static int
2774 ses_enumerate_node(ses_enum_data_t *sdp, tnode_t *pnode, ses_enum_chassis_t *cp,
2775     ses_phys_tree_t *parent, int mrange[])
2776 {
2777 	topo_mod_t *mod = sdp->sed_mod;
2778 	ses_phys_tree_t *child = NULL;
2779 	int i, ret = 0, ret_ch;
2780 	uint64_t prevtype = SES_ET_UNSPECIFIED;
2781 	ses_phys_tree_t *dirfru = NULL;
2782 	tnode_t *tn = NULL, *frutn = NULL;
2783 
2784 	if (parent == NULL) {
2785 		return (0);
2786 	}
2787 
2788 	for (child = parent->spt_child; child != NULL;
2789 	    child = child->spt_sibling) {
2790 		if (is_type_enumerated(child, &i)) {
2791 			if (prevtype != phys_enum_type_list[i].pet_type) {
2792 				/* check if range needs to be created. */
2793 				if (phys_enum_type_list[i].pet_dorange &&
2794 				    topo_node_range_create(mod, pnode,
2795 				    phys_enum_type_list[i].pet_nodename, 0,
2796 				    mrange[i]) != 0) {
2797 					topo_mod_dprintf(mod,
2798 					    "topo_node_create_range() failed: "
2799 					    "%s", topo_mod_errmsg(mod));
2800 					return (-1);
2801 				}
2802 				prevtype = phys_enum_type_list[i].pet_type;
2803 			}
2804 
2805 			if (!(child->spt_isfru)) {
2806 				for (dirfru = parent; dirfru != NULL;
2807 				    dirfru = dirfru->spt_parent) {
2808 					if (dirfru->spt_isfru) {
2809 						break;
2810 					}
2811 				}
2812 				/* found direct FRU node. */
2813 				if (dirfru) {
2814 					frutn = dirfru->spt_tnode;
2815 				} else {
2816 					frutn = NULL;
2817 				}
2818 			} else {
2819 				frutn = NULL;
2820 			}
2821 
2822 			if (ses_create_generic(sdp, child->spt_senumnode,
2823 			    pnode, frutn, phys_enum_type_list[i].pet_nodename,
2824 			    phys_enum_type_list[i].pet_defaultlabel, &tn) != 0)
2825 				return (-1);
2826 
2827 			child->spt_tnode = tn;
2828 			/*
2829 			 * For some SES element there may be protocol specific
2830 			 * information to process.   Here we are processing
2831 			 * the association between enclosure controller and
2832 			 * SAS expanders.
2833 			 */
2834 			if (phys_enum_type_list[i].pet_type ==
2835 			    SES_ET_ESC_ELECTRONICS) {
2836 				/* create SAS expander node */
2837 				if (ses_create_protocol_specific(sdp,
2838 				    child->spt_senumnode, tn,
2839 				    phys_enum_type_list[i].pet_type,
2840 				    cp, phys_enum_type_list[i].pet_dorange) !=
2841 				    0) {
2842 					return (-1);
2843 				}
2844 			}
2845 		} else {
2846 			continue;
2847 		}
2848 		ret_ch = ses_enumerate_node(sdp, tn, cp, child, mrange);
2849 		if (ret_ch)
2850 			ret = ret_ch; /* there was an error and set the ret. */
2851 	}
2852 
2853 	return (ret);
2854 }
2855 
2856 /*
2857  * Instantiate types of nodes that are specified in the hierarchy
2858  * element type list.
2859  */
2860 static int
2861 ses_create_children_from_phys_tree(ses_enum_data_t *sdp, tnode_t *pnode,
2862     ses_enum_chassis_t *cp, ses_phys_tree_t *phys_tree)
2863 {
2864 	topo_mod_t *mod = sdp->sed_mod;
2865 	int mrange[N_PHYS_ENUM_TYPES] = { 0 };
2866 	ses_enum_node_t *snp;
2867 	int i, ret;
2868 
2869 	/*
2870 	 * First get max range for each type of element to be enumerated.
2871 	 */
2872 	for (i = 0; i < N_PHYS_ENUM_TYPES; i++) {
2873 		if (phys_enum_type_list[i].pet_dorange) {
2874 			for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
2875 			    snp = topo_list_next(snp)) {
2876 				if (snp->sen_type ==
2877 				    phys_enum_type_list[i].pet_type) {
2878 					if (snp->sen_instance > mrange[i])
2879 						mrange[i] =
2880 						    snp->sen_instance;
2881 				}
2882 			}
2883 		}
2884 	}
2885 
2886 	topo_mod_dprintf(mod, "%s: creating nodes from FRU hierarchy tree.",
2887 	    cp->sec_csn);
2888 
2889 	if ((ret = ses_enumerate_node(sdp, pnode, cp, phys_tree, mrange)) !=
2890 	    0) {
2891 		topo_mod_dprintf(mod,
2892 		    "ses_create_children_from_phys_tree() failed: ");
2893 		return (ret);
2894 	}
2895 
2896 	return (0);
2897 }
2898 
2899 /*
2900  * Instantiate a new chassis instance in the topology.
2901  */
2902 static int
2903 ses_create_chassis(ses_enum_data_t *sdp, tnode_t *pnode, ses_enum_chassis_t *cp)
2904 {
2905 	topo_mod_t *mod = sdp->sed_mod;
2906 	nvlist_t *props;
2907 	char *raw_manufacturer, *raw_model, *raw_revision;
2908 	char *manufacturer = NULL, *model = NULL, *product = NULL;
2909 	char *revision = NULL;
2910 	char *serial;
2911 	char **paths;
2912 	size_t prodlen;
2913 	tnode_t *tn;
2914 	nvlist_t *fmri = NULL, *auth = NULL;
2915 	int ret = -1;
2916 	ses_enum_node_t *snp;
2917 	ses_enum_target_t *stp;
2918 	ses_enum_chassis_t *scp;
2919 	int i, err;
2920 	uint64_t sc_count = 0, pindex;
2921 	ses_phys_tree_t	*sproot = NULL;
2922 	hrtime_t start;
2923 	hrtime_t end;
2924 	double duration;
2925 
2926 	/*
2927 	 * Ignore any internal enclosures.
2928 	 */
2929 	if (cp->sec_internal)
2930 		return (0);
2931 
2932 	/*
2933 	 * Check to see if there are any devices presennt in the chassis.  If
2934 	 * not, ignore the chassis alltogether.  This is most useful for
2935 	 * ignoring internal HBAs that present a SES target but don't actually
2936 	 * manage any of the devices.
2937 	 */
2938 	for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
2939 	    snp = topo_list_next(snp)) {
2940 		if (snp->sen_type == SES_ET_DEVICE ||
2941 		    snp->sen_type == SES_ET_ARRAY_DEVICE)
2942 			break;
2943 	}
2944 
2945 	if (snp == NULL)
2946 		return (0);
2947 
2948 	props = ses_node_props(cp->sec_enclosure);
2949 
2950 	/*
2951 	 * We use the following property mappings:
2952 	 *
2953 	 * manufacturer		vendor-id
2954 	 * model		product-id
2955 	 * serial-number	libses-chassis-serial
2956 	 */
2957 	verify(nvlist_lookup_string(props, SES_EN_PROP_VID,
2958 	    &raw_manufacturer) == 0);
2959 	verify(nvlist_lookup_string(props, SES_EN_PROP_PID, &raw_model) == 0);
2960 	verify(nvlist_lookup_string(props, SES_EN_PROP_REV,
2961 	    &raw_revision) == 0);
2962 	verify(nvlist_lookup_string(props, LIBSES_EN_PROP_CSN, &serial) == 0);
2963 
2964 	/*
2965 	 * To construct the authority information, we 'clean' each string by
2966 	 * removing any offensive characters and trimmming whitespace.  For the
2967 	 * 'product-id', we use a concatenation of 'manufacturer-model'.  We
2968 	 * also take the numerical serial number and convert it to a string.
2969 	 */
2970 	if ((manufacturer = topo_mod_clean_str(mod, raw_manufacturer)) ==
2971 	    NULL || (model = topo_mod_clean_str(mod, raw_model)) == NULL ||
2972 	    (revision = topo_mod_clean_str(mod, raw_revision)) == NULL) {
2973 		goto error;
2974 	}
2975 
2976 	prodlen = strlen(manufacturer) + strlen(model) + 2;
2977 	if ((product = topo_mod_alloc(mod, prodlen)) == NULL)
2978 		goto error;
2979 
2980 	(void) snprintf(product, prodlen, "%s-%s", manufacturer, model);
2981 
2982 	/*
2983 	 * Construct the topo node and bind it to our parent.
2984 	 */
2985 	if (topo_mod_nvalloc(mod, &auth, NV_UNIQUE_NAME) != 0)
2986 		goto error;
2987 
2988 	if (nvlist_add_string(auth, FM_FMRI_AUTH_PRODUCT, product) != 0 ||
2989 	    nvlist_add_string(auth, FM_FMRI_AUTH_CHASSIS, serial) != 0) {
2990 		(void) topo_mod_seterrno(mod, EMOD_NVL_INVAL);
2991 		goto error;
2992 	}
2993 
2994 	/*
2995 	 * We pass NULL for the parent FMRI because there is no resource
2996 	 * associated with it.  For the toplevel enclosure, we leave the
2997 	 * serial/part/revision portions empty, which are reserved for
2998 	 * individual components within the chassis.
2999 	 */
3000 	if ((fmri = topo_mod_hcfmri(mod, NULL, FM_HC_SCHEME_VERSION,
3001 	    SES_ENCLOSURE, cp->sec_instance, NULL, auth,
3002 	    model, revision, serial)) == NULL) {
3003 		topo_mod_dprintf(mod, "topo_mod_hcfmri() failed: %s",
3004 		    topo_mod_errmsg(mod));
3005 		goto error;
3006 	}
3007 
3008 	if ((tn = topo_node_bind(mod, pnode, SES_ENCLOSURE,
3009 	    cp->sec_instance, fmri)) == NULL) {
3010 		topo_mod_dprintf(mod, "topo_node_bind() failed: %s",
3011 		    topo_mod_errmsg(mod));
3012 		goto error;
3013 	}
3014 
3015 	if (topo_method_register(mod, tn, ses_enclosure_methods) != 0) {
3016 		topo_mod_dprintf(mod,
3017 		    "topo_method_register() failed: %s",
3018 		    topo_mod_errmsg(mod));
3019 		goto error;
3020 	}
3021 
3022 	if (ses_set_standard_props(mod, NULL, tn, auth,
3023 	    ses_node_id(cp->sec_enclosure), cp->sec_target->set_devpath) != 0)
3024 		goto error;
3025 
3026 	/*
3027 	 * For enclosures, we want to include all possible targets (for upgrade
3028 	 * purposes).
3029 	 */
3030 	for (i = 0, stp = topo_list_next(&cp->sec_targets); stp != NULL;
3031 	    stp = topo_list_next(stp), i++)
3032 		;
3033 
3034 	verify(i != 0);
3035 	paths = alloca(i * sizeof (char *));
3036 
3037 	for (i = 0, stp = topo_list_next(&cp->sec_targets); stp != NULL;
3038 	    stp = topo_list_next(stp), i++)
3039 		paths[i] = stp->set_devpath;
3040 
3041 
3042 	if (topo_prop_set_string_array(tn, TOPO_PGROUP_SES,
3043 	    TOPO_PROP_PATHS, TOPO_PROP_IMMUTABLE, (const char **)paths,
3044 	    i, &err) != 0) {
3045 		topo_mod_dprintf(mod,
3046 		    "failed to create property %s: %s\n",
3047 		    TOPO_PROP_PATHS, topo_strerror(err));
3048 		goto error;
3049 	}
3050 
3051 	if (nvlist_lookup_uint64(props,
3052 	    LIBSES_PROP_PHYS_PARENT, &pindex) == 0) {
3053 		start = gethrtime(); /* to mearusre performance */
3054 		/*
3055 		 * The enclosure is supported through SUNW,FRUID.
3056 		 * Need to enumerate the nodes through hierarchical order.
3057 		 */
3058 		if ((sproot = topo_mod_zalloc(mod,
3059 		    sizeof (ses_phys_tree_t))) == NULL) {
3060 			topo_mod_dprintf(mod,
3061 			    "failed to allocate root: %s\n",
3062 			    topo_strerror(err));
3063 			goto error;
3064 		}
3065 		sproot->spt_pindex = pindex;
3066 		if (nvlist_lookup_boolean_value(props,
3067 		    LIBSES_PROP_FRU, &sproot->spt_isfru) != 0) {
3068 			topo_mod_dprintf(mod,
3069 			    "ses_create_chassis(): Failed to find prop %s "
3070 			    "on enclosure element (CSN %s).",
3071 			    LIBSES_PROP_FRU, cp->sec_csn);
3072 			/* an enclosure should be a FRU. continue to process. */
3073 			sproot->spt_isfru = B_TRUE;
3074 		}
3075 		if (nvlist_lookup_uint64(props,
3076 		    SES_PROP_ELEMENT_ONLY_INDEX,
3077 		    &sproot->spt_eonlyindex) != 0) {
3078 			topo_mod_dprintf(mod,
3079 			    "ses_create_chassis(): Failed to find prop %s "
3080 			    "on enclosure element (CSN %s).",
3081 			    LIBSES_PROP_PHYS_PARENT, cp->sec_csn);
3082 			topo_mod_free(mod, sproot, sizeof (ses_phys_tree_t));
3083 			goto error;
3084 		}
3085 		if (sproot->spt_pindex != sproot->spt_eonlyindex) {
3086 			topo_mod_dprintf(mod, "ses_create_chassis(): "
3087 			    "Enclosure element(CSN %s) should have "
3088 			    "itself as the parent to be the root node "
3089 			    "of FRU hierarchical tree.)", cp->sec_csn);
3090 			topo_mod_free(mod, sproot, sizeof (ses_phys_tree_t));
3091 			goto error;
3092 		} else {
3093 			sproot->spt_snode = cp->sec_enclosure;
3094 			sproot->spt_tnode = tn;
3095 			/* construct a tree. */
3096 			if (ses_construct_phys_tree(sdp, cp, sproot) != 0) {
3097 				topo_mod_dprintf(mod, "ses_create_chassis(): "
3098 				    "Failed to construct FRU hierarchical "
3099 				    "tree on enclosure (CSN %s.)",
3100 				    cp->sec_csn);
3101 			}
3102 
3103 			/* enumerate elements from the tree. */
3104 			if (ses_create_children_from_phys_tree(sdp, tn, cp,
3105 			    sproot) != 0) {
3106 				topo_mod_dprintf(mod, "ses_create_chassis(): "
3107 				    "Failed to create children topo nodes out "
3108 				    "of FRU hierarchical tree on enclosure "
3109 				    "(CSN %s).", cp->sec_csn);
3110 			}
3111 			/* destroy the phys tree. */
3112 			ses_phys_tree_free(mod, sproot);
3113 		}
3114 
3115 		end = gethrtime();
3116 		duration = end - start;
3117 		duration /= HR_SECOND;
3118 		topo_mod_dprintf(mod,
3119 		    "FRU boundary tree based enumeration: %.6f seconds",
3120 		    duration);
3121 	} else {
3122 		/*
3123 		 * Create the nodes for power supplies, fans, controllers and
3124 		 * devices.  Note that SAS exopander nodes and connector nodes
3125 		 * are handled through protocol specific processing of
3126 		 * controllers.
3127 		 */
3128 		if (ses_create_children(sdp, tn, SES_ET_POWER_SUPPLY,
3129 		    PSU, "PSU", cp, B_TRUE) != 0 ||
3130 		    ses_create_children(sdp, tn, SES_ET_COOLING,
3131 		    FAN, "FAN", cp, B_TRUE) != 0 ||
3132 		    ses_create_children(sdp, tn, SES_ET_ESC_ELECTRONICS,
3133 		    CONTROLLER, "CONTROLLER", cp, B_TRUE) != 0 ||
3134 		    ses_create_children(sdp, tn, SES_ET_DEVICE,
3135 		    BAY, "BAY", cp, B_TRUE) != 0 ||
3136 		    ses_create_children(sdp, tn, SES_ET_ARRAY_DEVICE,
3137 		    BAY, "BAY", cp, B_TRUE) != 0)
3138 			goto error;
3139 	}
3140 
3141 	if (cp->sec_maxinstance >= 0 &&
3142 	    (topo_node_range_create(mod, tn, SUBCHASSIS, 0,
3143 	    cp->sec_maxinstance) != 0)) {
3144 		topo_mod_dprintf(mod, "topo_node_create_range() failed: %s",
3145 		    topo_mod_errmsg(mod));
3146 		goto error;
3147 	}
3148 
3149 	for (scp = topo_list_next(&cp->sec_subchassis); scp != NULL;
3150 	    scp = topo_list_next(scp)) {
3151 
3152 		if (ses_create_subchassis(sdp, tn, scp) != 0)
3153 			goto error;
3154 
3155 		topo_mod_dprintf(mod, "created Subchassis node with "
3156 		    "instance %u\nand target (%s) under Chassis with CSN %s",
3157 		    scp->sec_instance, scp->sec_target->set_devpath,
3158 		    cp->sec_csn);
3159 
3160 		sc_count++;
3161 	}
3162 
3163 	topo_mod_dprintf(mod, "%s: created %llu %s nodes",
3164 	    cp->sec_csn, sc_count, SUBCHASSIS);
3165 
3166 	cp->sec_target->set_refcount++;
3167 	topo_node_setspecific(tn, cp->sec_target);
3168 
3169 	ret = 0;
3170 error:
3171 	topo_mod_strfree(mod, manufacturer);
3172 	topo_mod_strfree(mod, model);
3173 	topo_mod_strfree(mod, revision);
3174 	topo_mod_strfree(mod, product);
3175 
3176 	nvlist_free(fmri);
3177 	nvlist_free(auth);
3178 	return (ret);
3179 }
3180 
3181 /*
3182  * Create a bay node explicitly enumerated via XML.
3183  */
3184 static int
3185 ses_create_bays(ses_enum_data_t *sdp, tnode_t *pnode)
3186 {
3187 	topo_mod_t *mod = sdp->sed_mod;
3188 	ses_enum_chassis_t *cp;
3189 
3190 	/*
3191 	 * Iterate over chassis looking for an internal enclosure.  This
3192 	 * property is set via a vendor-specific plugin, and there should only
3193 	 * ever be a single internal chassis in a system.
3194 	 */
3195 	for (cp = topo_list_next(&sdp->sed_chassis); cp != NULL;
3196 	    cp = topo_list_next(cp)) {
3197 		if (cp->sec_internal)
3198 			break;
3199 	}
3200 
3201 	if (cp == NULL) {
3202 		topo_mod_dprintf(mod, "failed to find internal chassis\n");
3203 		return (-1);
3204 	}
3205 
3206 	if (ses_create_children(sdp, pnode, SES_ET_DEVICE,
3207 	    BAY, "BAY", cp, B_FALSE) != 0 ||
3208 	    ses_create_children(sdp, pnode, SES_ET_ARRAY_DEVICE,
3209 	    BAY, "BAY", cp, B_FALSE) != 0)
3210 		return (-1);
3211 
3212 	return (0);
3213 }
3214 
3215 /*
3216  * Initialize chassis or subchassis.
3217  */
3218 static int
3219 ses_init_chassis(topo_mod_t *mod, ses_enum_data_t *sdp, ses_enum_chassis_t *pcp,
3220     ses_enum_chassis_t *cp, ses_node_t *np, nvlist_t *props,
3221     uint64_t subchassis, ses_chassis_type_e flags)
3222 {
3223 	boolean_t internal, ident;
3224 
3225 	assert((flags & (SES_NEW_CHASSIS | SES_NEW_SUBCHASSIS |
3226 	    SES_DUP_CHASSIS | SES_DUP_SUBCHASSIS)) != 0);
3227 
3228 	assert(cp != NULL);
3229 	assert(np != NULL);
3230 	assert(props != NULL);
3231 
3232 	if (flags & (SES_NEW_SUBCHASSIS | SES_DUP_SUBCHASSIS))
3233 		assert(pcp != NULL);
3234 
3235 	topo_mod_dprintf(mod, "ses_init_chassis: %s: index %llu, flags (%d)",
3236 	    sdp->sed_name, subchassis, flags);
3237 
3238 	if (flags & (SES_NEW_CHASSIS | SES_NEW_SUBCHASSIS)) {
3239 
3240 		topo_mod_dprintf(mod, "new chassis/subchassis");
3241 		if (nvlist_lookup_boolean_value(props,
3242 		    LIBSES_EN_PROP_INTERNAL, &internal) == 0)
3243 			cp->sec_internal = internal;
3244 
3245 		cp->sec_enclosure = np;
3246 		cp->sec_target = sdp->sed_target;
3247 
3248 		if (flags & SES_NEW_CHASSIS) {
3249 			if (!cp->sec_internal)
3250 				cp->sec_instance = sdp->sed_instance++;
3251 			topo_list_append(&sdp->sed_chassis, cp);
3252 		} else {
3253 			if (subchassis != NO_SUBCHASSIS)
3254 				cp->sec_instance = subchassis;
3255 			else
3256 				cp->sec_instance = pcp->sec_scinstance++;
3257 
3258 			if (cp->sec_instance > pcp->sec_maxinstance)
3259 				pcp->sec_maxinstance = cp->sec_instance;
3260 
3261 			topo_list_append(&pcp->sec_subchassis, cp);
3262 		}
3263 
3264 	} else {
3265 		topo_mod_dprintf(mod, "dup chassis/subchassis");
3266 		if (nvlist_lookup_boolean_value(props,
3267 		    SES_PROP_IDENT, &ident) == 0) {
3268 			topo_mod_dprintf(mod,  "overriding enclosure node");
3269 
3270 			cp->sec_enclosure = np;
3271 			cp->sec_target = sdp->sed_target;
3272 		}
3273 	}
3274 
3275 	topo_list_append(&cp->sec_targets, sdp->sed_target);
3276 	sdp->sed_current = cp;
3277 
3278 	return (0);
3279 }
3280 
3281 /*
3282  * Gather nodes from the current SES target into our chassis list, merging the
3283  * results if necessary.
3284  */
3285 static ses_walk_action_t
3286 ses_enum_gather(ses_node_t *np, void *data)
3287 {
3288 	nvlist_t *props = ses_node_props(np);
3289 	ses_enum_data_t *sdp = data;
3290 	topo_mod_t *mod = sdp->sed_mod;
3291 	ses_enum_chassis_t *cp, *scp;
3292 	ses_enum_node_t *snp;
3293 	ses_alt_node_t *sap;
3294 	char *csn;
3295 	uint64_t instance, type;
3296 	uint64_t prevstatus, status;
3297 	boolean_t report;
3298 	uint64_t subchassis = NO_SUBCHASSIS;
3299 
3300 	if (ses_node_type(np) == SES_NODE_ENCLOSURE) {
3301 		/*
3302 		 * If we have already identified the chassis for this target,
3303 		 * then this is a secondary enclosure and we should ignore it,
3304 		 * along with the rest of the tree (since this is depth-first).
3305 		 */
3306 		if (sdp->sed_current != NULL)
3307 			return (SES_WALK_ACTION_TERMINATE);
3308 
3309 		/*
3310 		 * Go through the list of chassis we have seen so far and see
3311 		 * if this serial number matches one of the known values.
3312 		 * If so, check whether this enclosure is a subchassis.
3313 		 */
3314 		if (nvlist_lookup_string(props, LIBSES_EN_PROP_CSN,
3315 		    &csn) != 0)
3316 			return (SES_WALK_ACTION_TERMINATE);
3317 
3318 		(void) nvlist_lookup_uint64(props, LIBSES_EN_PROP_SUBCHASSIS_ID,
3319 		    &subchassis);
3320 
3321 		topo_mod_dprintf(mod, "ses_enum_gather: Enclosure Node (%s) "
3322 		    "CSN (%s), subchassis (%llu)", sdp->sed_name, csn,
3323 		    subchassis);
3324 
3325 		/*
3326 		 * We need to determine whether this enclosure node
3327 		 * represents a chassis or a subchassis. Since we may
3328 		 * receive the enclosure nodes in a non-deterministic
3329 		 * manner, we need to account for all possible combinations:
3330 		 *	1. Chassis for the current CSN has not yet been
3331 		 *	   allocated
3332 		 *		1.1 This is a new chassis:
3333 		 *			allocate and instantiate the chassis
3334 		 *		1.2 This is a new subchassis:
3335 		 *			allocate a placeholder chassis
3336 		 *			allocate and instantiate the subchassis
3337 		 *			link the subchassis to the chassis
3338 		 *	2. Chassis for the current CSN has been allocated
3339 		 *		2.1 This is a duplicate chassis enclosure
3340 		 *			check whether to override old chassis
3341 		 *			append to chassis' target list
3342 		 *		2.2 Only placeholder chassis exists
3343 		 *			fill in the chassis fields
3344 		 *		2.3 This is a new subchassis
3345 		 *			allocate and instantiate the subchassis
3346 		 *			link the subchassis to the chassis
3347 		 *		2.4 This is a duplicate subchassis enclosure
3348 		 *			 check whether to override old chassis
3349 		 *			 append to chassis' target list
3350 		 */
3351 
3352 		for (cp = topo_list_next(&sdp->sed_chassis); cp != NULL;
3353 		    cp = topo_list_next(cp))
3354 			if (strcmp(cp->sec_csn, csn) == 0)
3355 				break;
3356 
3357 		if (cp == NULL) {
3358 			/* 1. Haven't seen a chassis with this CSN before */
3359 
3360 			if ((cp = topo_mod_zalloc(mod,
3361 			    sizeof (ses_enum_chassis_t))) == NULL)
3362 				goto error;
3363 
3364 			cp->sec_scinstance = SES_STARTING_SUBCHASSIS;
3365 			cp->sec_maxinstance = -1;
3366 			cp->sec_csn = csn;
3367 
3368 			if (subchassis == NO_SUBCHASSIS) {
3369 				/* 1.1 This is a new chassis */
3370 
3371 				topo_mod_dprintf(mod, "%s: Initialize new "
3372 				    "chassis with CSN %s", sdp->sed_name, csn);
3373 
3374 				if (ses_init_chassis(mod, sdp, NULL, cp,
3375 				    np, props, NO_SUBCHASSIS,
3376 				    SES_NEW_CHASSIS) < 0)
3377 					goto error;
3378 			} else {
3379 				/* 1.2 This is a new subchassis */
3380 
3381 				topo_mod_dprintf(mod, "%s: Initialize new "
3382 				    "subchassis with CSN %s and index %llu",
3383 				    sdp->sed_name, csn, subchassis);
3384 
3385 				if ((scp = topo_mod_zalloc(mod,
3386 				    sizeof (ses_enum_chassis_t))) == NULL)
3387 					goto error;
3388 
3389 				scp->sec_csn = csn;
3390 
3391 				if (ses_init_chassis(mod, sdp, cp, scp, np,
3392 				    props, subchassis, SES_NEW_SUBCHASSIS) < 0)
3393 					goto error;
3394 			}
3395 		} else {
3396 			/*
3397 			 * We have a chassis or subchassis with this CSN.  If
3398 			 * it's a chassis, we must check to see whether it is
3399 			 * a placeholder previously created because we found a
3400 			 * subchassis with this CSN.  We will know that because
3401 			 * the sec_target value will not be set; it is set only
3402 			 * in ses_init_chassis().  In that case, initialise it
3403 			 * as a new chassis; otherwise, it's a duplicate and we
3404 			 * need to append only.
3405 			 */
3406 			if (subchassis == NO_SUBCHASSIS) {
3407 				if (cp->sec_target != NULL) {
3408 					/* 2.1 This is a duplicate chassis */
3409 
3410 					topo_mod_dprintf(mod, "%s: Append "
3411 					    "duplicate chassis with CSN (%s)",
3412 					    sdp->sed_name, csn);
3413 
3414 					if (ses_init_chassis(mod, sdp, NULL, cp,
3415 					    np, props, NO_SUBCHASSIS,
3416 					    SES_DUP_CHASSIS) < 0)
3417 						goto error;
3418 				} else {
3419 					/* Placeholder chassis - init it up */
3420 					topo_mod_dprintf(mod, "%s: Initialize"
3421 					    "placeholder chassis with CSN %s",
3422 					    sdp->sed_name, csn);
3423 
3424 					if (ses_init_chassis(mod, sdp, NULL,
3425 					    cp, np, props, NO_SUBCHASSIS,
3426 					    SES_NEW_CHASSIS) < 0)
3427 						goto error;
3428 
3429 				}
3430 			} else {
3431 				/* This is a subchassis */
3432 
3433 				for (scp = topo_list_next(&cp->sec_subchassis);
3434 				    scp != NULL; scp = topo_list_next(scp))
3435 					if (scp->sec_instance == subchassis)
3436 						break;
3437 
3438 				if (scp == NULL) {
3439 					/* 2.3 This is a new subchassis */
3440 
3441 					topo_mod_dprintf(mod, "%s: Initialize "
3442 					    "new subchassis with CSN (%s) "
3443 					    "and LID (%s)",
3444 					    sdp->sed_name, csn);
3445 
3446 					if ((scp = topo_mod_zalloc(mod,
3447 					    sizeof (ses_enum_chassis_t)))
3448 					    == NULL)
3449 						goto error;
3450 
3451 					scp->sec_csn = csn;
3452 
3453 					if (ses_init_chassis(mod, sdp, cp, scp,
3454 					    np, props, subchassis,
3455 					    SES_NEW_SUBCHASSIS) < 0)
3456 						goto error;
3457 				} else {
3458 					/* 2.4 This is a duplicate subchassis */
3459 
3460 					topo_mod_dprintf(mod, "%s: Append "
3461 					    "duplicate subchassis with "
3462 					    "CSN (%s)", sdp->sed_name, csn);
3463 
3464 					if (ses_init_chassis(mod, sdp, cp, scp,
3465 					    np, props, subchassis,
3466 					    SES_DUP_SUBCHASSIS) < 0)
3467 						goto error;
3468 				}
3469 			}
3470 		}
3471 	} else if (ses_node_type(np) == SES_NODE_ELEMENT) {
3472 		/*
3473 		 * If we haven't yet seen an enclosure node and identified the
3474 		 * current chassis, something is very wrong; bail out.
3475 		 */
3476 		if (sdp->sed_current == NULL)
3477 			return (SES_WALK_ACTION_TERMINATE);
3478 
3479 		/*
3480 		 * If this isn't one of the element types we care about, then
3481 		 * ignore it.
3482 		 */
3483 		verify(nvlist_lookup_uint64(props, SES_PROP_ELEMENT_TYPE,
3484 		    &type) == 0);
3485 		if (type != SES_ET_DEVICE &&
3486 		    type != SES_ET_ARRAY_DEVICE &&
3487 		    type != SES_ET_SUNW_FANBOARD &&
3488 		    type != SES_ET_SUNW_FANMODULE &&
3489 		    type != SES_ET_COOLING &&
3490 		    type != SES_ET_SUNW_POWERBOARD &&
3491 		    type != SES_ET_SUNW_POWERMODULE &&
3492 		    type != SES_ET_POWER_SUPPLY &&
3493 		    type != SES_ET_ESC_ELECTRONICS &&
3494 		    type != SES_ET_SAS_EXPANDER &&
3495 		    type != SES_ET_SAS_CONNECTOR)
3496 			return (SES_WALK_ACTION_CONTINUE);
3497 
3498 		/*
3499 		 * Get the current instance number and see if we already know
3500 		 * about this element.  If so, it means we have multiple paths
3501 		 * to the same elements, and we should ignore the current path.
3502 		 */
3503 		verify(nvlist_lookup_uint64(props, SES_PROP_ELEMENT_CLASS_INDEX,
3504 		    &instance) == 0);
3505 		if (type == SES_ET_DEVICE || type == SES_ET_ARRAY_DEVICE)
3506 			(void) nvlist_lookup_uint64(props, SES_PROP_BAY_NUMBER,
3507 			    &instance);
3508 
3509 		cp = sdp->sed_current;
3510 
3511 		for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
3512 		    snp = topo_list_next(snp)) {
3513 			if (snp->sen_type == type &&
3514 			    snp->sen_instance == instance)
3515 				break;
3516 		}
3517 
3518 		/*
3519 		 * We prefer the new element under the following circumstances:
3520 		 *
3521 		 * - The currently known element's status is unknown or not
3522 		 *   available, but the new element has a known status.  This
3523 		 *   occurs if a given element is only available through a
3524 		 *   particular target.
3525 		 *
3526 		 * - This is an ESC_ELECTRONICS element, and the 'reported-via'
3527 		 *   property is set.  This allows us to get reliable firmware
3528 		 *   revision information from the enclosure node.
3529 		 */
3530 		if (snp != NULL) {
3531 			if (nvlist_lookup_uint64(
3532 			    ses_node_props(snp->sen_node),
3533 			    SES_PROP_STATUS_CODE, &prevstatus) != 0)
3534 				prevstatus = SES_ESC_UNSUPPORTED;
3535 			if (nvlist_lookup_uint64(
3536 			    props, SES_PROP_STATUS_CODE, &status) != 0)
3537 				status = SES_ESC_UNSUPPORTED;
3538 			if (nvlist_lookup_boolean_value(
3539 			    props, SES_PROP_REPORT, &report) != 0)
3540 				report = B_FALSE;
3541 
3542 			if ((SES_STATUS_UNAVAIL(prevstatus) &&
3543 			    !SES_STATUS_UNAVAIL(status)) ||
3544 			    (type == SES_ET_ESC_ELECTRONICS &&
3545 			    report)) {
3546 				snp->sen_node = np;
3547 				snp->sen_target = sdp->sed_target;
3548 			}
3549 
3550 			if ((sap = topo_mod_zalloc(mod,
3551 			    sizeof (ses_alt_node_t))) == NULL)
3552 				goto error;
3553 
3554 			sap->san_node = np;
3555 			topo_list_append(&snp->sen_alt_nodes, sap);
3556 
3557 			return (SES_WALK_ACTION_CONTINUE);
3558 		}
3559 
3560 		if ((snp = topo_mod_zalloc(mod,
3561 		    sizeof (ses_enum_node_t))) == NULL)
3562 			goto error;
3563 
3564 		if ((sap = topo_mod_zalloc(mod,
3565 		    sizeof (ses_alt_node_t))) == NULL) {
3566 			topo_mod_free(mod, snp, sizeof (ses_enum_node_t));
3567 			goto error;
3568 		}
3569 
3570 		topo_mod_dprintf(mod, "%s: adding node (%llu, %llu)",
3571 		    sdp->sed_name, type, instance);
3572 		snp->sen_node = np;
3573 		snp->sen_type = type;
3574 		snp->sen_instance = instance;
3575 		snp->sen_target = sdp->sed_target;
3576 		sap->san_node = np;
3577 		topo_list_append(&snp->sen_alt_nodes, sap);
3578 		topo_list_append(&cp->sec_nodes, snp);
3579 
3580 		if (type == SES_ET_DEVICE)
3581 			cp->sec_hasdev = B_TRUE;
3582 	}
3583 
3584 	return (SES_WALK_ACTION_CONTINUE);
3585 
3586 error:
3587 	sdp->sed_errno = -1;
3588 	return (SES_WALK_ACTION_TERMINATE);
3589 }
3590 
3591 static int
3592 ses_process_dir(const char *dirpath, ses_enum_data_t *sdp)
3593 {
3594 	topo_mod_t *mod = sdp->sed_mod;
3595 	DIR *dir;
3596 	struct dirent *dp;
3597 	char path[PATH_MAX];
3598 	ses_enum_target_t *stp;
3599 	int err = -1;
3600 
3601 	/*
3602 	 * Open the SES target directory and iterate over any available
3603 	 * targets.
3604 	 */
3605 	if ((dir = opendir(dirpath)) == NULL) {
3606 		/*
3607 		 * If the SES target directory does not exist, then return as if
3608 		 * there are no active targets.
3609 		 */
3610 		topo_mod_dprintf(mod, "failed to open ses "
3611 		    "directory '%s'", dirpath);
3612 		return (0);
3613 	}
3614 
3615 	while ((dp = readdir(dir)) != NULL) {
3616 		if (strcmp(dp->d_name, ".") == 0 ||
3617 		    strcmp(dp->d_name, "..") == 0)
3618 			continue;
3619 
3620 		/*
3621 		 * Create a new target instance and take a snapshot.
3622 		 */
3623 		if ((stp = topo_mod_zalloc(mod,
3624 		    sizeof (ses_enum_target_t))) == NULL)
3625 			goto error;
3626 
3627 		(void) pthread_mutex_init(&stp->set_lock, NULL);
3628 
3629 		(void) snprintf(path, sizeof (path), "%s/%s", dirpath,
3630 		    dp->d_name);
3631 
3632 		/*
3633 		 * We keep track of the SES device path and export it on a
3634 		 * per-node basis to allow higher level software to get to the
3635 		 * corresponding SES state.
3636 		 */
3637 		if ((stp->set_devpath = topo_mod_strdup(mod, path)) == NULL) {
3638 			topo_mod_free(mod, stp, sizeof (ses_enum_target_t));
3639 			goto error;
3640 		}
3641 
3642 		if ((stp->set_target =
3643 		    ses_open(LIBSES_VERSION, path)) == NULL) {
3644 			topo_mod_dprintf(mod, "failed to open ses target "
3645 			    "'%s': %s", dp->d_name, ses_errmsg());
3646 			ses_sof_alloc(mod, stp->set_devpath);
3647 			topo_mod_free(mod, stp, sizeof (ses_enum_target_t));
3648 			continue;
3649 		}
3650 		topo_mod_dprintf(mod, "open contract");
3651 		ses_ssl_alloc(mod, stp);
3652 		ses_create_contract(mod, stp);
3653 
3654 		stp->set_refcount = 1;
3655 		sdp->sed_target = stp;
3656 		stp->set_snap = ses_snap_hold(stp->set_target);
3657 		stp->set_snaptime = gethrtime();
3658 
3659 		/*
3660 		 * Enumerate over all SES elements and merge them into the
3661 		 * correct ses_enum_chassis_t.
3662 		 */
3663 		sdp->sed_current = NULL;
3664 		sdp->sed_errno = 0;
3665 		sdp->sed_name = dp->d_name;
3666 		(void) ses_walk(stp->set_snap, ses_enum_gather, sdp);
3667 
3668 		if (sdp->sed_errno != 0)
3669 			goto error;
3670 	}
3671 
3672 	err = 0;
3673 error:
3674 	(void) closedir(dir);
3675 	return (err);
3676 }
3677 
3678 /*
3679  * Different generations of SMCI's 4U36 storage servers used different models
3680  * of front and rear SAS expanders.
3681  */
3682 #define	SMCI4U36_FRONT_EXPANDER_PID1	"LSI-SAS2X36"
3683 #define	SMCI4U36_FRONT_EXPANDER_PID2	"LSI-SAS3x40"
3684 #define	SMCI4U36_FRONT_EXPANDER_PID3	"SMC-SC846P"
3685 
3686 #define	SMCI4U36_REAR_EXPANDER_PID1	"LSI-CORP-SAS2X28"
3687 #define	SMCI4U36_REAR_EXPANDER_PID2	"LSI-SAS3x28"
3688 
3689 static int
3690 smci_4u36_bay_label(topo_mod_t *mod, tnode_t *node, topo_version_t version,
3691     nvlist_t *in, nvlist_t **out)
3692 {
3693 	int err, ret = -1;
3694 	nvlist_t *pargs, *auth, *nvl = NULL, *fmri;
3695 	char *label = NULL, *product_id;
3696 
3697 	/*
3698 	 * Now look for a private argument list to determine if the invoker is
3699 	 * trying to do a set operation and if so, return an error as this
3700 	 * method only supports get operations.
3701 	 */
3702 	if ((nvlist_lookup_nvlist(in, TOPO_PROP_PARGS, &pargs) == 0) &&
3703 	    nvlist_exists(pargs, TOPO_PROP_VAL_VAL)) {
3704 		topo_mod_dprintf(mod, "%s: set operation not suppported",
3705 		    __func__);
3706 		return (topo_mod_seterrno(mod, EMOD_NVL_INVAL));
3707 	}
3708 
3709 	if (topo_node_resource(node, &fmri, &err) != 0) {
3710 		(void) topo_mod_seterrno(mod, err);
3711 		goto err;
3712 	}
3713 
3714 	if (nvlist_lookup_nvlist(fmri, FM_FMRI_AUTHORITY, &auth) != 0 ||
3715 	    nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT, &product_id) !=
3716 	    0) {
3717 		topo_mod_dprintf(mod, "%s: malformed FMRI", __func__);
3718 		(void) topo_mod_seterrno(mod, EMOD_UNKNOWN);
3719 		nvlist_free(fmri);
3720 		goto err;
3721 	}
3722 	nvlist_free(fmri);
3723 
3724 	if (strcmp(product_id, SMCI4U36_FRONT_EXPANDER_PID1) == 0 ||
3725 	    strcmp(product_id, SMCI4U36_FRONT_EXPANDER_PID2) == 0 ||
3726 	    strcmp(product_id, SMCI4U36_FRONT_EXPANDER_PID3) == 0) {
3727 		err = asprintf(&label, "Front Slot %" PRIu64,
3728 		    topo_node_instance(node));
3729 	} else if (strcmp(product_id, SMCI4U36_REAR_EXPANDER_PID1) == 0 ||
3730 	    strcmp(product_id, SMCI4U36_REAR_EXPANDER_PID2) == 0) {
3731 		err = asprintf(&label, "Rear Slot %" PRIu64,
3732 		    topo_node_instance(node));
3733 	} else {
3734 		topo_mod_dprintf(mod, "%s: unexpected expander product id: %s",
3735 		    __func__, product_id);
3736 		(void) topo_mod_seterrno(mod, EMOD_UNKNOWN);
3737 		goto err;
3738 	}
3739 
3740 	if (err < 0) {
3741 		(void) topo_mod_seterrno(mod, EMOD_NOMEM);
3742 		goto err;
3743 	}
3744 
3745 	if (topo_mod_nvalloc(mod, &nvl, NV_UNIQUE_NAME) != 0 ||
3746 	    nvlist_add_string(nvl, TOPO_PROP_VAL_NAME, TOPO_PROP_LABEL) != 0 ||
3747 	    nvlist_add_uint32(nvl, TOPO_PROP_VAL_TYPE, TOPO_TYPE_STRING)
3748 	    != 0 ||
3749 	    nvlist_add_string(nvl, TOPO_PROP_VAL_VAL, label)
3750 	    != 0) {
3751 		topo_mod_dprintf(mod, "Failed to allocate 'out' nvlist");
3752 		nvlist_free(nvl);
3753 		(void) topo_mod_seterrno(mod, EMOD_NOMEM);
3754 		goto err;
3755 	}
3756 	*out = nvl;
3757 	ret = 0;
3758 err:
3759 	free(label);
3760 	return (ret);
3761 
3762 }
3763 
3764 static void
3765 ses_release(topo_mod_t *mod, tnode_t *tn)
3766 {
3767 	ses_enum_target_t *stp;
3768 
3769 	if ((stp = topo_node_getspecific(tn)) != NULL) {
3770 		topo_node_setspecific(tn, NULL);
3771 		ses_target_free(mod, stp);
3772 	}
3773 }
3774 
3775 /*ARGSUSED*/
3776 static int
3777 ses_enum(topo_mod_t *mod, tnode_t *rnode, const char *name,
3778     topo_instance_t min, topo_instance_t max, void *arg, void *notused)
3779 {
3780 	ses_enum_chassis_t *cp;
3781 	ses_enum_data_t *data;
3782 
3783 	/*
3784 	 * Check to make sure we're being invoked sensibly, and that we're not
3785 	 * being invoked as part of a post-processing step.
3786 	 */
3787 	if (strcmp(name, SES_ENCLOSURE) != 0 && strcmp(name, BAY) != 0)
3788 		return (0);
3789 
3790 	/*
3791 	 * If this is the first time we've called our enumeration method, then
3792 	 * gather information about any available enclosures.
3793 	 */
3794 	if ((data = topo_mod_getspecific(mod)) == NULL) {
3795 		ses_sof_freeall(mod);
3796 		if ((data = topo_mod_zalloc(mod, sizeof (ses_enum_data_t))) ==
3797 		    NULL)
3798 			return (-1);
3799 
3800 		data->sed_mod = mod;
3801 		topo_mod_setspecific(mod, data);
3802 
3803 		if (dev_list_gather(mod, &data->sed_devs) != 0)
3804 			goto error;
3805 
3806 		/*
3807 		 * We search both the ses(7D) and sgen(7D) locations, so we are
3808 		 * independent of any particular driver class bindings.
3809 		 */
3810 		if (ses_process_dir("/dev/es", data) != 0 ||
3811 		    ses_process_dir("/dev/scsi/ses", data) != 0)
3812 			goto error;
3813 	}
3814 
3815 	if (strcmp(name, SES_ENCLOSURE) == 0) {
3816 		/*
3817 		 * This is a request to enumerate external enclosures.  Go
3818 		 * through all the targets and create chassis nodes where
3819 		 * necessary.
3820 		 */
3821 		for (cp = topo_list_next(&data->sed_chassis); cp != NULL;
3822 		    cp = topo_list_next(cp)) {
3823 			if (ses_create_chassis(data, rnode, cp) != 0)
3824 				goto error;
3825 		}
3826 	} else {
3827 		/*
3828 		 * This is a request to enumerate a specific bay underneath the
3829 		 * root chassis (for internal disks).
3830 		 */
3831 		if (ses_create_bays(data, rnode) != 0)
3832 			goto error;
3833 	}
3834 
3835 	/*
3836 	 * This is a bit of a kludge.  In order to allow internal disks to be
3837 	 * enumerated and share snapshot-specific information with the external
3838 	 * enclosure enumeration, we rely on the fact that we will be invoked
3839 	 * for the 'ses-enclosure' node last.
3840 	 */
3841 	if (strcmp(name, SES_ENCLOSURE) == 0) {
3842 		for (cp = topo_list_next(&data->sed_chassis); cp != NULL;
3843 		    cp = topo_list_next(cp))
3844 			ses_data_free(data, cp);
3845 		ses_data_free(data, NULL);
3846 		topo_mod_setspecific(mod, NULL);
3847 	}
3848 	return (0);
3849 
3850 error:
3851 	for (cp = topo_list_next(&data->sed_chassis); cp != NULL;
3852 	    cp = topo_list_next(cp))
3853 		ses_data_free(data, cp);
3854 	ses_data_free(data, NULL);
3855 	topo_mod_setspecific(mod, NULL);
3856 	return (-1);
3857 }
3858 
3859 static const topo_modops_t ses_ops =
3860 	{ ses_enum, ses_release };
3861 
3862 static topo_modinfo_t ses_info =
3863 	{ SES_ENCLOSURE, FM_FMRI_SCHEME_HC, SES_VERSION, &ses_ops };
3864 
3865 /*ARGSUSED*/
3866 int
3867 _topo_init(topo_mod_t *mod, topo_version_t version)
3868 {
3869 	int rval;
3870 
3871 	if (getenv("TOPOSESDEBUG") != NULL)
3872 		topo_mod_setdebug(mod);
3873 
3874 	topo_mod_dprintf(mod, "initializing %s enumerator\n",
3875 	    SES_ENCLOSURE);
3876 
3877 	if ((rval = topo_mod_register(mod, &ses_info, TOPO_VERSION)) == 0)
3878 		ses_thread_init(mod);
3879 
3880 	return (rval);
3881 }
3882 
3883 void
3884 _topo_fini(topo_mod_t *mod)
3885 {
3886 	ses_thread_fini(mod);
3887 	ses_sof_freeall(mod);
3888 	topo_mod_unregister(mod);
3889 }
3890