1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright 2012 Milan Jurik. All rights reserved.
25  * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
26  * Copyright 2020 Joyent, Inc.
27  */
28 
29 #include <alloca.h>
30 #include <dirent.h>
31 #include <devid.h>
32 #include <fm/libdiskstatus.h>
33 #include <inttypes.h>
34 #include <pthread.h>
35 #include <strings.h>
36 #include <string.h>
37 #include <unistd.h>
38 #include <sys/dkio.h>
39 #include <sys/fm/protocol.h>
40 #include <sys/libdevid.h>
41 #include <sys/scsi/scsi_types.h>
42 #include <sys/byteorder.h>
43 #include <pthread.h>
44 #include <signal.h>
45 #include <fcntl.h>
46 #include <sys/ctfs.h>
47 #include <libcontract.h>
48 #include <poll.h>
49 #include <sys/contract/device.h>
50 #include <libsysevent.h>
51 #include <sys/sysevent/eventdefs.h>
52 #include <scsi/plugins/ses/vendor/sun.h>
53 
54 #include "disk.h"
55 #include "ses.h"
56 
57 #define	SES_VERSION	1
58 
59 #define	SES_STARTING_SUBCHASSIS 256	/* valid subchassis IDs are uint8_t */
60 #define	NO_SUBCHASSIS	((uint64_t)-1)
61 
62 static int ses_snap_freq = 250;		/* in milliseconds */
63 
64 #define	SES_STATUS_UNAVAIL(s)	\
65 	((s) == SES_ESC_UNSUPPORTED || (s) >= SES_ESC_NOT_INSTALLED)
66 
67 #define	HR_SECOND   1000000000
68 
69 #define	SES_INST_NOTSET		UINT64_MAX
70 
71 /*
72  * Because multiple SES targets can be part of a single chassis, we construct
73  * our own hierarchy that takes this into account.  These SES targets may refer
74  * to the same devices (multiple paths) or to different devices (managing
75  * different portions of the space).  We arrange things into a
76  * ses_enum_enclosure_t, which contains a set of ses targets, and a list of all
77  * nodes found so far.
78  */
79 typedef struct ses_alt_node {
80 	topo_list_t		san_link;
81 	ses_node_t		*san_node;
82 } ses_alt_node_t;
83 
84 typedef struct ses_enum_node {
85 	topo_list_t		sen_link;
86 	ses_node_t		*sen_node;
87 	topo_list_t		sen_alt_nodes;
88 	uint64_t		sen_type;
89 	uint64_t		sen_instance;
90 	ses_enum_target_t	*sen_target;
91 } ses_enum_node_t;
92 
93 typedef struct ses_enum_chassis {
94 	topo_list_t		sec_link;
95 	topo_list_t		sec_subchassis;
96 	topo_list_t		sec_nodes;
97 	topo_list_t		sec_targets;
98 	const char		*sec_csn;
99 	ses_node_t		*sec_enclosure;
100 	ses_enum_target_t	*sec_target;
101 	topo_instance_t		sec_instance;
102 	topo_instance_t		sec_scinstance;
103 	topo_instance_t		sec_maxinstance;
104 	boolean_t		sec_hasdev;
105 	boolean_t		sec_internal;
106 } ses_enum_chassis_t;
107 
108 typedef struct ses_enum_data {
109 	topo_list_t		sed_devs;
110 	topo_list_t		sed_chassis;
111 	ses_enum_chassis_t	*sed_current;
112 	ses_enum_target_t	*sed_target;
113 	int			sed_errno;
114 	char			*sed_name;
115 	topo_mod_t		*sed_mod;
116 	topo_instance_t		sed_instance;
117 } ses_enum_data_t;
118 
119 typedef struct sas_connector_phy_data {
120 	uint64_t    scpd_index;
121 	uint64_t    scpd_pm;
122 } sas_connector_phy_data_t;
123 
124 typedef struct sas_connector_type {
125 	uint64_t    sct_type;
126 	char	    *sct_name;
127 } sas_connector_type_t;
128 
129 static const sas_connector_type_t sas_connector_type_list[] = {
130 	{   0x0, "Information unknown"  },
131 	{   0x1, "External SAS 4x receptacle (see SAS-2 and SFF-8470)"	},
132 	{   0x2, "Exteranl Mini SAS 4x receptacle (see SAS-2 and SFF-8088)" },
133 	{   0x3, "QSFP+ receptacle (see SAS-2.1 and SFF-8436)" },
134 	{   0x4, "Mini SAS 4x active receptacle (see SAS-2.1 and SFF-8088)" },
135 	{   0x5, "Mini SAS HD 4x receptacle (see SAS-2.1 and SFF-8644)" },
136 	{   0x6, "Mini SAS HD 8x receptacle (see SAS-2.1 and SFF-8644)" },
137 	{   0x7, "Mini SAS HD 16x receptacle (see SAS-2.1 and SFF-8644)" },
138 	{   0xF, "Vendor-specific external connector"	},
139 	{   0x10, "Internal wide SAS 4i plug (see SAS-2 and SFF-8484)"	},
140 	{   0x11,
141 	"Internal wide Mini SAS 4i receptacle (see SAS-2 and SFF-8087)"	},
142 	{   0x12, "Mini SAS HD 4i receptacle (see SAS-2.1 and SFF-8643)" },
143 	{   0x20, "Internal SAS Drive receptacle (see SAS-2 and SFF-8482)" },
144 	{   0x21, "Internal SATA host plug (see SAS-2 and SATA-2)"	},
145 	{   0x22, "Internal SAS Drive plug (see SAS-2 and SFF-8482)"	},
146 	{   0x23, "Internal SATA device plug (see SAS-2 and SATA-2)"	},
147 	{   0x24, "Micro SAS receptacle (see SAS-2.14)"	},
148 	{   0x25, "Micro SATA device plug (see SAS-2.1 and SATA)"	},
149 	{   0x26, "Micro SAS plug (see SAS-2.1 and SFF-8486)"	},
150 	{   0x27, "Micro SAS/SATA plug (see SAS-2.1 and SFF-8486)"	},
151 	{   0x28,
152 	"12 Gb/s SAS Drive backplane receptacle (see SAS-34 and SFF-8680)" },
153 	{   0x29, "12Gb/s SAS Drive Plug (see SAS-3 and SFF-8680)"	},
154 	{   0x2A, "Multifunction 12 Gb/s 6x Unshielded receptacle connector "
155 		"receptacle (see SAS-3 and SFF-8639)"	},
156 	{   0x2B, "Multifunction 12 Gb/s 6x Unshielded receptable connector "
157 		"plug (see SAS-3 and SFF-8639)"	},
158 	{   0x2F, "Internal SAS virtual connector"  },
159 	{   0x3F, "Vendor-specific internal connector"	},
160 	{   0x70, "Other Vendor-specific connector"	},
161 	{   0x71, "Other Vendor-specific connector"	},
162 	{   0x72, "Other Vendor-specific connector"	},
163 	{   0x73, "Other Vendor-specific connector"	},
164 	{   0x74, "Other Vendor-specific connector"	},
165 	{   0x75, "Other Vendor-specific connector"	},
166 	{   0x76, "Other Vendor-specific connector"	},
167 	{   0x77, "Other Vendor-specific connector"	},
168 	{   0x78, "Other Vendor-specific connector"	},
169 	{   0x79, "Other Vendor-specific connector"	},
170 	{   0x7A, "Other Vendor-specific connector"	},
171 	{   0x7B, "Other Vendor-specific connector"	},
172 	{   0x7C, "Other Vendor-specific connector"	},
173 	{   0x7D, "Other Vendor-specific connector"	},
174 	{   0x7E, "Other Vendor-specific connector"	},
175 	{   0x7F, "Other Vendor-specific connector"	},
176 	{   0x80, "Not Defined"	}
177 };
178 
179 #define	SAS_CONNECTOR_TYPE_CODE_NOT_DEFINED  0x80
180 #define	SAS_CONNECTOR_TYPE_NOT_DEFINED \
181 	"Connector type not defined by SES-2 standard"
182 #define	SAS_CONNECTOR_TYPE_RESERVED \
183 	"Connector type reserved by SES-2 standard"
184 
185 typedef struct phys_enum_type {
186 	uint64_t    pet_type;
187 	char	    *pet_nodename;
188 	char	    *pet_defaultlabel;
189 	boolean_t   pet_dorange;
190 } phys_enum_type_t;
191 
192 static const phys_enum_type_t phys_enum_type_list[] = {
193 	{   SES_ET_ARRAY_DEVICE, BAY, "BAY", B_TRUE  },
194 	{   SES_ET_COOLING, FAN, "FAN", B_TRUE  },
195 	{   SES_ET_DEVICE, BAY, "BAY", B_TRUE  },
196 	{   SES_ET_ESC_ELECTRONICS, CONTROLLER, "CONTROLLER", B_TRUE  },
197 	{   SES_ET_POWER_SUPPLY, PSU, "PSU", B_TRUE  },
198 	{   SES_ET_SUNW_FANBOARD, FANBOARD, "FANBOARD", B_TRUE  },
199 	{   SES_ET_SUNW_FANMODULE, FANMODULE, "FANMODULE", B_TRUE  },
200 	{   SES_ET_SUNW_POWERBOARD, POWERBOARD, "POWERBOARD", B_TRUE  },
201 	{   SES_ET_SUNW_POWERMODULE, POWERMODULE, "POWERMODULE", B_TRUE  }
202 };
203 
204 #define	N_PHYS_ENUM_TYPES (sizeof (phys_enum_type_list) / \
205 	sizeof (phys_enum_type_list[0]))
206 
207 /*
208  * Structure for the hierarchical tree for element nodes.
209  */
210 typedef struct ses_phys_tree {
211     ses_node_t	*spt_snode;
212     ses_enum_node_t	*spt_senumnode;
213     boolean_t	spt_isfru;
214     uint64_t	spt_eonlyindex;
215     uint64_t	spt_cindex;
216     uint64_t	spt_pindex;
217     uint64_t	spt_maxinst;
218     struct ses_phys_tree    *spt_parent;
219     struct ses_phys_tree    *spt_child;
220     struct ses_phys_tree    *spt_sibling;
221     tnode_t	*spt_tnode;
222 } ses_phys_tree_t;
223 
224 typedef enum {
225 	SES_NEW_CHASSIS		= 0x1,
226 	SES_NEW_SUBCHASSIS	= 0x2,
227 	SES_DUP_CHASSIS		= 0x4,
228 	SES_DUP_SUBCHASSIS	= 0x8
229 } ses_chassis_type_e;
230 
231 static const topo_pgroup_info_t smp_pgroup = {
232 	TOPO_PGROUP_SMP,
233 	TOPO_STABILITY_PRIVATE,
234 	TOPO_STABILITY_PRIVATE,
235 	1
236 };
237 
238 static const topo_pgroup_info_t ses_pgroup = {
239 	TOPO_PGROUP_SES,
240 	TOPO_STABILITY_PRIVATE,
241 	TOPO_STABILITY_PRIVATE,
242 	1
243 };
244 
245 static int ses_present(topo_mod_t *, tnode_t *, topo_version_t, nvlist_t *,
246     nvlist_t **);
247 static int ses_contains(topo_mod_t *, tnode_t *, topo_version_t, nvlist_t *,
248     nvlist_t **);
249 
250 static const topo_method_t ses_component_methods[] = {
251 	{ TOPO_METH_PRESENT, TOPO_METH_PRESENT_DESC,
252 	    TOPO_METH_PRESENT_VERSION0, TOPO_STABILITY_INTERNAL, ses_present },
253 	{ TOPO_METH_FAC_ENUM, TOPO_METH_FAC_ENUM_DESC, 0,
254 	    TOPO_STABILITY_INTERNAL, ses_node_enum_facility },
255 	{ TOPO_METH_SENSOR_FAILURE, TOPO_METH_SENSOR_FAILURE_DESC,
256 	    TOPO_METH_SENSOR_FAILURE_VERSION, TOPO_STABILITY_INTERNAL,
257 	    topo_method_sensor_failure },
258 	{ NULL }
259 };
260 
261 #define	TOPO_METH_SMCI_4U36_LABEL		"smci_4u36_bay_label"
262 #define	TOPO_METH_SMCI_4U36_LABEL_DESC	\
263 	"compute bay labels on SMCI 4U36 storage platform variants"
264 #define	TOPO_METH_SMCI_4U36_LABEL_VERSION	0
265 static int smci_4u36_bay_label(topo_mod_t *, tnode_t *, topo_version_t,
266     nvlist_t *, nvlist_t **);
267 
268 static const topo_method_t ses_bay_methods[] = {
269 	{ TOPO_METH_FAC_ENUM, TOPO_METH_FAC_ENUM_DESC, 0,
270 	    TOPO_STABILITY_INTERNAL, ses_node_enum_facility },
271 	{ TOPO_METH_OCCUPIED, TOPO_METH_OCCUPIED_DESC,
272 	    TOPO_METH_OCCUPIED_VERSION, TOPO_STABILITY_INTERNAL,
273 	    topo_mod_hc_occupied },
274 	{ TOPO_METH_SMCI_4U36_LABEL, TOPO_METH_SMCI_4U36_LABEL_DESC,
275 	    TOPO_METH_SMCI_4U36_LABEL_VERSION, TOPO_STABILITY_INTERNAL,
276 	    smci_4u36_bay_label },
277 	{ NULL }
278 };
279 
280 static const topo_method_t ses_recep_methods[] = {
281 	{ TOPO_METH_OCCUPIED, TOPO_METH_OCCUPIED_DESC,
282 	    TOPO_METH_OCCUPIED_VERSION, TOPO_STABILITY_INTERNAL,
283 	    topo_mod_hc_occupied },
284 	{ NULL }
285 };
286 
287 static const topo_method_t ses_enclosure_methods[] = {
288 	{ TOPO_METH_CONTAINS, TOPO_METH_CONTAINS_DESC,
289 	    TOPO_METH_CONTAINS_VERSION, TOPO_STABILITY_INTERNAL, ses_contains },
290 	{ TOPO_METH_FAC_ENUM, TOPO_METH_FAC_ENUM_DESC, 0,
291 	    TOPO_STABILITY_INTERNAL, ses_enc_enum_facility },
292 	{ NULL }
293 };
294 
295 /*
296  * The bay_label_overrides table can be used to map a server product ID to a
297  * topo method that will be invoked to override the value of the label property
298  * for all bay nodes.  By default the property value is static, derived from
299  * the corresponding SES array device element's descriptor string.
300  */
301 typedef struct ses_label_overrides {
302 	const char *slbl_product;
303 	const char *slbl_mname;
304 } ses_label_overrides_t;
305 
306 /*
307  * This table covers three generations of SMCI's 4U 36-bay storage server
308  * (and the Joyent-branded versions).  There was also an Ivy Bridge variant
309  * which has been omitted due to an inability to find one to test on.
310  */
311 static const ses_label_overrides_t bay_label_overrides[] = {
312 	/* Sandy Bridge variant */
313 	{ "SSG-6047R-E1R36L", TOPO_METH_SMCI_4U36_LABEL },
314 	{ "Joyent-Storage-Platform-5001", TOPO_METH_SMCI_4U36_LABEL },
315 
316 	/* Broadwell variant */
317 	{ "SSG-6048R-E1CR36L", TOPO_METH_SMCI_4U36_LABEL },
318 	{ "Joyent-Storage-Platform-7001", TOPO_METH_SMCI_4U36_LABEL },
319 
320 	/* Skylake variant */
321 	{ "SSG-6049P-E1CR36L", TOPO_METH_SMCI_4U36_LABEL },
322 	{ "Joyent-S10G5", TOPO_METH_SMCI_4U36_LABEL }
323 };
324 
325 #define	N_BAY_LBL_OVERRIDES (sizeof (bay_label_overrides) / \
326 	sizeof (bay_label_overrides[0]))
327 
328 /*
329  * Functions for tracking ses devices which we were unable to open. We retry
330  * these at regular intervals using ses_recheck_dir() and if we find that we
331  * can now open any of them then we send a sysevent to indicate that a new topo
332  * snapshot should be taken.
333  */
334 typedef struct ses_open_fail_list {
335 	struct ses_open_fail_list	*sof_next;
336 	char				*sof_path;
337 } ses_open_fail_list_t;
338 
339 static ses_open_fail_list_t *ses_sofh;
340 static pthread_mutex_t ses_sofmt;
341 static void ses_ct_print(char *ptr);
342 
343 static void
344 ses_recheck_dir()
345 {
346 	ses_target_t *target;
347 	sysevent_id_t eid;
348 	char buf[80];
349 	ses_open_fail_list_t *sof;
350 
351 	/*
352 	 * check list of "unable to open" devices
353 	 */
354 	(void) pthread_mutex_lock(&ses_sofmt);
355 	for (sof = ses_sofh; sof != NULL; sof = sof->sof_next) {
356 		/*
357 		 * see if we can open it now
358 		 */
359 		if ((target = ses_open(LIBSES_VERSION,
360 		    sof->sof_path)) == NULL) {
361 			(void) snprintf(buf, sizeof (buf),
362 			    "recheck_dir - still can't open %s", sof->sof_path);
363 			ses_ct_print(buf);
364 			continue;
365 		}
366 
367 		/*
368 		 * ok - better force a new snapshot
369 		 */
370 		(void) snprintf(buf, sizeof (buf),
371 		    "recheck_dir - can now open %s", sof->sof_path);
372 		ses_ct_print(buf);
373 		(void) sysevent_post_event(EC_PLATFORM, ESC_PLATFORM_SP_RESET,
374 		    SUNW_VENDOR, "fmd", NULL, &eid);
375 		ses_close(target);
376 		break;
377 	}
378 	(void) pthread_mutex_unlock(&ses_sofmt);
379 }
380 
381 static void
382 ses_sof_alloc(topo_mod_t *mod, char *path)
383 {
384 	ses_open_fail_list_t *sof;
385 
386 	(void) pthread_mutex_lock(&ses_sofmt);
387 	sof = topo_mod_zalloc(mod, sizeof (*sof));
388 	topo_mod_dprintf(mod, "sof_alloc %s", path);
389 	sof->sof_path = path;
390 	sof->sof_next = ses_sofh;
391 	ses_sofh = sof;
392 	(void) pthread_mutex_unlock(&ses_sofmt);
393 }
394 
395 static void
396 ses_sof_freeall(topo_mod_t *mod)
397 {
398 	ses_open_fail_list_t *sof, *next_sof;
399 
400 	(void) pthread_mutex_lock(&ses_sofmt);
401 	for (sof = ses_sofh; sof != NULL; sof = next_sof) {
402 		next_sof = sof->sof_next;
403 		topo_mod_dprintf(mod, "sof_freeall %s", sof->sof_path);
404 		topo_mod_strfree(mod, sof->sof_path);
405 		topo_mod_free(mod, sof, sizeof (*sof));
406 	}
407 	ses_sofh = NULL;
408 	(void) pthread_mutex_unlock(&ses_sofmt);
409 }
410 
411 /*
412  * functions for verifying that the ses_enum_target_t held in a device
413  * contract's cookie field is still valid (it may have been freed by
414  * ses_release()).
415  */
416 typedef struct ses_stp_list {
417 	struct ses_stp_list	*ssl_next;
418 	ses_enum_target_t	*ssl_tgt;
419 } ses_stp_list_t;
420 
421 static ses_stp_list_t *ses_sslh;
422 static pthread_mutex_t ses_sslmt;
423 
424 static void
425 ses_ssl_alloc(topo_mod_t *mod, ses_enum_target_t *stp)
426 {
427 	ses_stp_list_t *ssl;
428 
429 	(void) pthread_mutex_lock(&ses_sslmt);
430 	ssl = topo_mod_zalloc(mod, sizeof (*ssl));
431 	topo_mod_dprintf(mod, "ssl_alloc %p", stp);
432 	ssl->ssl_tgt = stp;
433 	ssl->ssl_next = ses_sslh;
434 	ses_sslh = ssl;
435 	(void) pthread_mutex_unlock(&ses_sslmt);
436 }
437 
438 static void
439 ses_ssl_free(topo_mod_t *mod, ses_enum_target_t *stp)
440 {
441 	ses_stp_list_t *ssl, *prev_ssl;
442 
443 	(void) pthread_mutex_lock(&ses_sslmt);
444 	prev_ssl = NULL;
445 	for (ssl = ses_sslh; ssl != NULL; ssl = ssl->ssl_next) {
446 		if (ssl->ssl_tgt == stp) {
447 			topo_mod_dprintf(mod, "ssl_free %p", ssl->ssl_tgt);
448 			if (prev_ssl == NULL)
449 				ses_sslh = ssl->ssl_next;
450 			else
451 				prev_ssl->ssl_next = ssl->ssl_next;
452 			topo_mod_free(mod, ssl, sizeof (*ssl));
453 			break;
454 		}
455 		prev_ssl = ssl;
456 	}
457 	(void) pthread_mutex_unlock(&ses_sslmt);
458 }
459 
460 static int
461 ses_ssl_valid(ses_enum_target_t *stp)
462 {
463 	ses_stp_list_t *ssl;
464 
465 	for (ssl = ses_sslh; ssl != NULL; ssl = ssl->ssl_next)
466 		if (ssl->ssl_tgt == stp)
467 			return (1);
468 	return (0);
469 }
470 
471 /*
472  * Functions for creating and destroying a background thread
473  * (ses_contract_thread) used for detecting when ses devices have been
474  * retired/unretired.
475  */
476 static struct ses_thread_s {
477 	pthread_mutex_t mt;
478 	pthread_t tid;
479 	int thr_sig;
480 	int doexit;
481 	int count;
482 } sesthread = {
483 	PTHREAD_MUTEX_INITIALIZER,
484 	0,
485 	SIGTERM,
486 	0,
487 	0
488 };
489 
490 typedef struct ses_mod_list {
491 	struct ses_mod_list	*smod_next;
492 	topo_mod_t		*smod_mod;
493 } ses_mod_list_t;
494 
495 static ses_mod_list_t *ses_smod;
496 
497 static void
498 ses_ct_print(char *ptr)
499 {
500 	(void) pthread_mutex_lock(&sesthread.mt);
501 	if (ses_smod != NULL && ses_smod->smod_mod != NULL)
502 		topo_mod_dprintf(ses_smod->smod_mod, ptr);
503 	(void) pthread_mutex_unlock(&sesthread.mt);
504 }
505 
506 /*ARGSUSED*/
507 static void *
508 ses_contract_thread(void *arg)
509 {
510 	int efd, ctlfd, statfd;
511 	ct_evthdl_t ev;
512 	ctevid_t evid;
513 	uint_t event;
514 	char path[PATH_MAX];
515 	char buf[80];
516 	ses_enum_target_t *stp;
517 	ct_stathdl_t stathdl;
518 	ctid_t ctid;
519 	struct pollfd fds;
520 	int pollret;
521 	sigset_t sigset;
522 
523 	ses_ct_print("start contract event thread");
524 	efd = open64(CTFS_ROOT "/device/pbundle", O_RDONLY);
525 	fds.fd = efd;
526 	fds.events = POLLIN;
527 	fds.revents = 0;
528 	(void) sigaddset(&sigset, sesthread.thr_sig);
529 	(void) pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);
530 	for (;;) {
531 		/* check if we've been asked to exit */
532 		(void) pthread_mutex_lock(&sesthread.mt);
533 		if (sesthread.doexit) {
534 			(void) pthread_mutex_unlock(&sesthread.mt);
535 			break;
536 		}
537 		(void) pthread_mutex_unlock(&sesthread.mt);
538 
539 		/* poll until an event arrives */
540 		if ((pollret = poll(&fds, 1, 10000)) <= 0) {
541 			if (pollret == 0)
542 				ses_recheck_dir();
543 			continue;
544 		}
545 
546 		/* read the event */
547 		(void) pthread_mutex_lock(&ses_sslmt);
548 		ses_ct_print("read contract event");
549 		if (ct_event_read(efd, &ev) != 0) {
550 			(void) pthread_mutex_unlock(&ses_sslmt);
551 			continue;
552 		}
553 
554 		/* see if it is an event we are expecting */
555 		ctid = ct_event_get_ctid(ev);
556 		(void) snprintf(buf, sizeof (buf),
557 		    "got contract event ctid=%d", ctid);
558 		ses_ct_print(buf);
559 		event = ct_event_get_type(ev);
560 		if (event != CT_DEV_EV_OFFLINE && event != CT_EV_NEGEND) {
561 			(void) snprintf(buf, sizeof (buf),
562 			    "bad contract event %x", event);
563 			ses_ct_print(buf);
564 			ct_event_free(ev);
565 			(void) pthread_mutex_unlock(&ses_sslmt);
566 			continue;
567 		}
568 
569 		/* find target pointer saved in cookie */
570 		evid = ct_event_get_evid(ev);
571 		(void) snprintf(path, PATH_MAX, CTFS_ROOT "/device/%ld/status",
572 		    ctid);
573 		statfd = open64(path, O_RDONLY);
574 		(void) ct_status_read(statfd, CTD_COMMON, &stathdl);
575 		stp = (ses_enum_target_t *)(uintptr_t)
576 		    ct_status_get_cookie(stathdl);
577 		ct_status_free(stathdl);
578 		(void) close(statfd);
579 
580 		/* check if target pointer is still valid */
581 		if (ses_ssl_valid(stp) == 0) {
582 			(void) snprintf(buf, sizeof (buf),
583 			    "contract already abandoned %x", event);
584 			ses_ct_print(buf);
585 			(void) snprintf(path, PATH_MAX,
586 			    CTFS_ROOT "/device/%ld/ctl", ctid);
587 			ctlfd = open64(path, O_WRONLY);
588 			if (event != CT_EV_NEGEND)
589 				(void) ct_ctl_ack(ctlfd, evid);
590 			else
591 				(void) ct_ctl_abandon(ctlfd);
592 			(void) close(ctlfd);
593 			ct_event_free(ev);
594 			(void) pthread_mutex_unlock(&ses_sslmt);
595 			continue;
596 		}
597 
598 		/* find control device for ack/abandon */
599 		(void) pthread_mutex_lock(&stp->set_lock);
600 		(void) snprintf(path, PATH_MAX, CTFS_ROOT "/device/%ld/ctl",
601 		    ctid);
602 		ctlfd = open64(path, O_WRONLY);
603 		if (event != CT_EV_NEGEND) {
604 			/* if this is an offline event, do the offline */
605 			ses_ct_print("got contract offline event");
606 			if (stp->set_target) {
607 				ses_ct_print("contract thread rele");
608 				ses_snap_rele(stp->set_snap);
609 				ses_close(stp->set_target);
610 				stp->set_target = NULL;
611 			}
612 			(void) ct_ctl_ack(ctlfd, evid);
613 		} else {
614 			/* if this is the negend, then abandon the contract */
615 			ses_ct_print("got contract negend");
616 			if (stp->set_ctid) {
617 				(void) snprintf(buf, sizeof (buf),
618 				    "abandon old contract %d", stp->set_ctid);
619 				ses_ct_print(buf);
620 				stp->set_ctid = 0;
621 			}
622 			(void) ct_ctl_abandon(ctlfd);
623 		}
624 		(void) close(ctlfd);
625 		(void) pthread_mutex_unlock(&stp->set_lock);
626 		ct_event_free(ev);
627 		(void) pthread_mutex_unlock(&ses_sslmt);
628 	}
629 	(void) close(efd);
630 	return (NULL);
631 }
632 
633 int
634 find_thr_sig(void)
635 {
636 	int i;
637 	sigset_t oset, rset;
638 	int sig[] = {SIGTERM, SIGUSR1, SIGUSR2};
639 	int sig_sz = sizeof (sig) / sizeof (int);
640 	int rc = SIGTERM;
641 
642 	/* prefered set of signals that are likely used to terminate threads */
643 	(void) sigemptyset(&oset);
644 	(void) pthread_sigmask(SIG_SETMASK, NULL, &oset);
645 	for (i = 0; i < sig_sz; i++) {
646 		if (sigismember(&oset, sig[i]) == 0) {
647 			return (sig[i]);
648 		}
649 	}
650 
651 	/* reserved set of signals that are not allowed to terminate thread */
652 	(void) sigemptyset(&rset);
653 	(void) sigaddset(&rset, SIGABRT);
654 	(void) sigaddset(&rset, SIGKILL);
655 	(void) sigaddset(&rset, SIGSTOP);
656 	(void) sigaddset(&rset, SIGCANCEL);
657 
658 	/* Find signal that is not masked and not in the reserved list. */
659 	for (i = 1; i < MAXSIG; i++) {
660 		if (sigismember(&rset, i) == 1) {
661 			continue;
662 		}
663 		if (sigismember(&oset, i) == 0) {
664 			return (i);
665 		}
666 	}
667 
668 	return (rc);
669 }
670 
671 /*ARGSUSED*/
672 static void
673 ses_handler(int sig)
674 {
675 }
676 
677 static void
678 ses_thread_init(topo_mod_t *mod)
679 {
680 	pthread_attr_t *attr = NULL;
681 	struct sigaction act;
682 	ses_mod_list_t *smod;
683 
684 	(void) pthread_mutex_lock(&sesthread.mt);
685 	sesthread.count++;
686 	smod = topo_mod_zalloc(mod, sizeof (*smod));
687 	smod->smod_mod = mod;
688 	smod->smod_next = ses_smod;
689 	ses_smod = smod;
690 	if (sesthread.tid == 0) {
691 		/* find a suitable signal to use for killing the thread below */
692 		sesthread.thr_sig = find_thr_sig();
693 
694 		/* if don't have a handler for this signal, create one */
695 		(void) sigaction(sesthread.thr_sig, NULL, &act);
696 		if (act.sa_handler == SIG_DFL || act.sa_handler == SIG_IGN)
697 			act.sa_handler = ses_handler;
698 		(void) sigaction(sesthread.thr_sig, &act, NULL);
699 
700 		/* create a thread to listen for offline events */
701 		(void) pthread_create(&sesthread.tid,
702 		    attr, ses_contract_thread, NULL);
703 	}
704 	(void) pthread_mutex_unlock(&sesthread.mt);
705 }
706 
707 static void
708 ses_thread_fini(topo_mod_t *mod)
709 {
710 	ses_mod_list_t *smod, *prev_smod;
711 
712 	(void) pthread_mutex_lock(&sesthread.mt);
713 	prev_smod = NULL;
714 	for (smod = ses_smod; smod != NULL; smod = smod->smod_next) {
715 		if (smod->smod_mod == mod) {
716 			if (prev_smod == NULL)
717 				ses_smod = smod->smod_next;
718 			else
719 				prev_smod->smod_next = smod->smod_next;
720 			topo_mod_free(mod, smod, sizeof (*smod));
721 			break;
722 		}
723 		prev_smod = smod;
724 	}
725 	if (--sesthread.count > 0) {
726 		(void) pthread_mutex_unlock(&sesthread.mt);
727 		return;
728 	}
729 	sesthread.doexit = 1;
730 	(void) pthread_mutex_unlock(&sesthread.mt);
731 	(void) pthread_kill(sesthread.tid, sesthread.thr_sig);
732 	(void) pthread_join(sesthread.tid, NULL);
733 	sesthread.tid = 0;
734 }
735 
736 static void
737 ses_create_contract(topo_mod_t *mod, ses_enum_target_t *stp)
738 {
739 	int tfd, len, rval;
740 	char link_path[PATH_MAX];
741 
742 	stp->set_ctid = 0;
743 
744 	/* convert "/dev" path into "/devices" path */
745 	if ((len = readlink(stp->set_devpath, link_path, PATH_MAX)) < 0) {
746 		topo_mod_dprintf(mod, "readlink failed");
747 		return;
748 	}
749 	link_path[len] = '\0';
750 
751 	/* set up template to create new contract */
752 	tfd = open64(CTFS_ROOT "/device/template", O_RDWR);
753 	(void) ct_tmpl_set_critical(tfd, CT_DEV_EV_OFFLINE);
754 	(void) ct_tmpl_set_cookie(tfd, (uint64_t)(uintptr_t)stp);
755 
756 	/* strip "../../devices" off the front and create the contract */
757 	if ((rval = ct_dev_tmpl_set_minor(tfd, &link_path[13])) != 0)
758 		topo_mod_dprintf(mod, "failed to set minor %s rval = %d",
759 		    &link_path[13], rval);
760 	else if ((rval = ct_tmpl_create(tfd, &stp->set_ctid)) != 0)
761 		topo_mod_dprintf(mod, "failed to create ctid rval = %d", rval);
762 	else
763 		topo_mod_dprintf(mod, "created ctid=%d", stp->set_ctid);
764 	(void) close(tfd);
765 }
766 
767 static void
768 ses_target_free(topo_mod_t *mod, ses_enum_target_t *stp)
769 {
770 	if (--stp->set_refcount == 0) {
771 		/* check if already closed due to contract offline request */
772 		(void) pthread_mutex_lock(&stp->set_lock);
773 		if (stp->set_target) {
774 			ses_snap_rele(stp->set_snap);
775 			ses_close(stp->set_target);
776 			stp->set_target = NULL;
777 		}
778 		if (stp->set_ctid) {
779 			int ctlfd;
780 			char path[PATH_MAX];
781 
782 			topo_mod_dprintf(mod, "abandon old contract %d",
783 			    stp->set_ctid);
784 			(void) snprintf(path, PATH_MAX,
785 			    CTFS_ROOT "/device/%ld/ctl", stp->set_ctid);
786 			ctlfd = open64(path, O_WRONLY);
787 			(void) ct_ctl_abandon(ctlfd);
788 			(void) close(ctlfd);
789 			stp->set_ctid = 0;
790 		}
791 		(void) pthread_mutex_unlock(&stp->set_lock);
792 		ses_ssl_free(mod, stp);
793 		topo_mod_strfree(mod, stp->set_devpath);
794 		topo_mod_free(mod, stp, sizeof (ses_enum_target_t));
795 	}
796 }
797 
798 static void
799 ses_data_free(ses_enum_data_t *sdp, ses_enum_chassis_t *pcp)
800 {
801 	topo_mod_t *mod = sdp->sed_mod;
802 	ses_enum_chassis_t *cp;
803 	ses_enum_node_t *np;
804 	ses_enum_target_t *tp;
805 	ses_alt_node_t *ap;
806 	topo_list_t *cpl;
807 
808 
809 	if (pcp != NULL)
810 		cpl = &pcp->sec_subchassis;
811 	else
812 		cpl = &sdp->sed_chassis;
813 
814 	while ((cp = topo_list_next(cpl)) != NULL) {
815 		topo_list_delete(cpl, cp);
816 
817 		while ((np = topo_list_next(&cp->sec_nodes)) != NULL) {
818 			while ((ap = topo_list_next(&np->sen_alt_nodes)) !=
819 			    NULL) {
820 				topo_list_delete(&np->sen_alt_nodes, ap);
821 				topo_mod_free(mod, ap, sizeof (ses_alt_node_t));
822 			}
823 			topo_list_delete(&cp->sec_nodes, np);
824 			topo_mod_free(mod, np, sizeof (ses_enum_node_t));
825 		}
826 
827 		while ((tp = topo_list_next(&cp->sec_targets)) != NULL) {
828 			topo_list_delete(&cp->sec_targets, tp);
829 			ses_target_free(mod, tp);
830 		}
831 
832 		topo_mod_free(mod, cp, sizeof (ses_enum_chassis_t));
833 	}
834 
835 	if (pcp == NULL) {
836 		dev_list_free(mod, &sdp->sed_devs);
837 		topo_mod_free(mod, sdp, sizeof (ses_enum_data_t));
838 	}
839 }
840 
841 /*
842  * For enclosure nodes, we have a special contains method.  By default, the hc
843  * walker will compare the node name and instance number to determine if an
844  * FMRI matches.  For enclosures where the enumeration order is impossible to
845  * predict, we instead use the chassis-id as a unique identifier, and ignore
846  * the instance number.
847  */
848 static int
849 fmri_contains(topo_mod_t *mod, nvlist_t *nv1, nvlist_t *nv2)
850 {
851 	uint8_t v1, v2;
852 	nvlist_t **hcp1, **hcp2;
853 	int err, i;
854 	uint_t nhcp1, nhcp2;
855 	nvlist_t *a1, *a2;
856 	char *c1, *c2;
857 	int mindepth;
858 
859 	if (nvlist_lookup_uint8(nv1, FM_VERSION, &v1) != 0 ||
860 	    nvlist_lookup_uint8(nv2, FM_VERSION, &v2) != 0 ||
861 	    v1 > FM_HC_SCHEME_VERSION || v2 > FM_HC_SCHEME_VERSION)
862 		return (topo_mod_seterrno(mod, EMOD_FMRI_VERSION));
863 
864 	err = nvlist_lookup_nvlist_array(nv1, FM_FMRI_HC_LIST, &hcp1, &nhcp1);
865 	err |= nvlist_lookup_nvlist_array(nv2, FM_FMRI_HC_LIST, &hcp2, &nhcp2);
866 	if (err != 0)
867 		return (topo_mod_seterrno(mod, EMOD_FMRI_NVL));
868 
869 	/*
870 	 * If the chassis-id doesn't match, then these FMRIs are not
871 	 * equivalent.  If one of the FMRIs doesn't have a chassis ID, then we
872 	 * have no choice but to fall back to the instance ID.
873 	 */
874 	if (nvlist_lookup_nvlist(nv1, FM_FMRI_AUTHORITY, &a1) == 0 &&
875 	    nvlist_lookup_nvlist(nv2, FM_FMRI_AUTHORITY, &a2) == 0 &&
876 	    nvlist_lookup_string(a1, FM_FMRI_AUTH_CHASSIS, &c1) == 0 &&
877 	    nvlist_lookup_string(a2, FM_FMRI_AUTH_CHASSIS, &c2) == 0) {
878 		if (strcmp(c1, c2) != 0)
879 			return (0);
880 
881 		mindepth = 1;
882 	} else {
883 		mindepth = 0;
884 	}
885 
886 	if (nhcp2 < nhcp1)
887 		return (0);
888 
889 	for (i = 0; i < nhcp1; i++) {
890 		char *nm1 = NULL;
891 		char *nm2 = NULL;
892 		char *id1 = NULL;
893 		char *id2 = NULL;
894 
895 		(void) nvlist_lookup_string(hcp1[i], FM_FMRI_HC_NAME, &nm1);
896 		(void) nvlist_lookup_string(hcp2[i], FM_FMRI_HC_NAME, &nm2);
897 		(void) nvlist_lookup_string(hcp1[i], FM_FMRI_HC_ID, &id1);
898 		(void) nvlist_lookup_string(hcp2[i], FM_FMRI_HC_ID, &id2);
899 		if (nm1 == NULL || nm2 == NULL || id1 == NULL || id2 == NULL)
900 			return (topo_mod_seterrno(mod, EMOD_FMRI_NVL));
901 
902 		if (strcmp(nm1, nm2) == 0 &&
903 		    (i < mindepth || strcmp(id1, id2) == 0))
904 			continue;
905 
906 		return (0);
907 	}
908 
909 	return (1);
910 }
911 
912 /*ARGSUSED*/
913 static int
914 ses_contains(topo_mod_t *mod, tnode_t *tn, topo_version_t version,
915     nvlist_t *in, nvlist_t **out)
916 {
917 	int ret;
918 	nvlist_t *nv1, *nv2;
919 
920 	if (version > TOPO_METH_CONTAINS_VERSION)
921 		return (topo_mod_seterrno(mod, EMOD_VER_NEW));
922 
923 	if (nvlist_lookup_nvlist(in, TOPO_METH_FMRI_ARG_FMRI, &nv1) != 0 ||
924 	    nvlist_lookup_nvlist(in, TOPO_METH_FMRI_ARG_SUBFMRI, &nv2) != 0)
925 		return (topo_mod_seterrno(mod, EMOD_METHOD_INVAL));
926 
927 	ret = fmri_contains(mod, nv1, nv2);
928 	if (ret < 0)
929 		return (-1);
930 
931 	if (topo_mod_nvalloc(mod, out, NV_UNIQUE_NAME) == 0) {
932 		if (nvlist_add_uint32(*out, TOPO_METH_CONTAINS_RET,
933 		    ret) == 0)
934 			return (0);
935 		else
936 			nvlist_free(*out);
937 	}
938 
939 	return (-1);
940 
941 }
942 
943 /*
944  * Return a current instance of the node.  This is somewhat complicated because
945  * we need to take a new snapshot in order to get the new data, but we don't
946  * want to be constantly taking SES snapshots if the consumer is going to do a
947  * series of queries.  So we adopt the strategy of assuming that the SES state
948  * is not going to be rapidly changing, and limit our snapshot frequency to
949  * some defined bounds.
950  */
951 ses_node_t *
952 ses_node_lock(topo_mod_t *mod, tnode_t *tn)
953 {
954 	ses_enum_target_t *tp = topo_node_getspecific(tn);
955 	hrtime_t now;
956 	ses_snap_t *snap;
957 	int err;
958 	uint64_t nodeid;
959 	ses_node_t *np;
960 
961 	if (tp == NULL) {
962 		(void) topo_mod_seterrno(mod, EMOD_METHOD_NOTSUP);
963 		return (NULL);
964 	}
965 
966 	(void) pthread_mutex_lock(&tp->set_lock);
967 
968 	/*
969 	 * Determine if we need to take a new snapshot.
970 	 */
971 	now = gethrtime();
972 
973 	if (tp->set_target == NULL) {
974 		/*
975 		 * We may have closed the device but not yet abandoned the
976 		 * contract (ie we've had the offline event but not yet the
977 		 * negend). If so, just return failure.
978 		 */
979 		if (tp->set_ctid != 0) {
980 			(void) topo_mod_seterrno(mod, EMOD_METHOD_NOTSUP);
981 			(void) pthread_mutex_unlock(&tp->set_lock);
982 			return (NULL);
983 		}
984 
985 		/*
986 		 * The device has been closed due to a contract offline
987 		 * request, then we need to reopen it and create a new contract.
988 		 */
989 		if ((tp->set_target =
990 		    ses_open(LIBSES_VERSION, tp->set_devpath)) == NULL) {
991 			sysevent_id_t eid;
992 
993 			(void) topo_mod_seterrno(mod, EMOD_METHOD_NOTSUP);
994 			(void) pthread_mutex_unlock(&tp->set_lock);
995 			topo_mod_dprintf(mod, "recheck_dir - "
996 			    "can no longer open %s", tp->set_devpath);
997 			(void) sysevent_post_event(EC_PLATFORM,
998 			    ESC_PLATFORM_SP_RESET, SUNW_VENDOR, "fmd", NULL,
999 			    &eid);
1000 			return (NULL);
1001 		}
1002 		topo_mod_dprintf(mod, "reopen contract");
1003 		ses_create_contract(mod, tp);
1004 		tp->set_snap = ses_snap_hold(tp->set_target);
1005 		tp->set_snaptime = gethrtime();
1006 	} else if (now - tp->set_snaptime > (ses_snap_freq * 1000 * 1000) &&
1007 	    (snap = ses_snap_new(tp->set_target)) != NULL) {
1008 		if (ses_snap_generation(snap) !=
1009 		    ses_snap_generation(tp->set_snap)) {
1010 			/*
1011 			 * If we find ourselves in this situation, we're in
1012 			 * trouble.  The generation count has changed, which
1013 			 * indicates that our current topology is out of date.
1014 			 * But we need to consult the new topology in order to
1015 			 * determine presence at this moment in time.  We can't
1016 			 * go back and change the topo snapshot in situ, so
1017 			 * we'll just have to fail the call in this unlikely
1018 			 * scenario.
1019 			 */
1020 			ses_snap_rele(snap);
1021 			(void) topo_mod_seterrno(mod, EMOD_METHOD_NOTSUP);
1022 			(void) pthread_mutex_unlock(&tp->set_lock);
1023 			return (NULL);
1024 		} else {
1025 			ses_snap_rele(tp->set_snap);
1026 			tp->set_snap = snap;
1027 		}
1028 		tp->set_snaptime = gethrtime();
1029 	}
1030 
1031 	snap = tp->set_snap;
1032 
1033 	verify(topo_prop_get_uint64(tn, TOPO_PGROUP_SES,
1034 	    TOPO_PROP_NODE_ID, &nodeid, &err) == 0);
1035 	verify((np = ses_node_lookup(snap, nodeid)) != NULL);
1036 
1037 	return (np);
1038 }
1039 
1040 /*ARGSUSED*/
1041 void
1042 ses_node_unlock(topo_mod_t *mod, tnode_t *tn)
1043 {
1044 	ses_enum_target_t *tp = topo_node_getspecific(tn);
1045 
1046 	verify(tp != NULL);
1047 
1048 	(void) pthread_mutex_unlock(&tp->set_lock);
1049 }
1050 
1051 /*
1052  * Determine if the element is present.
1053  */
1054 /*ARGSUSED*/
1055 static int
1056 ses_present(topo_mod_t *mod, tnode_t *tn, topo_version_t version,
1057     nvlist_t *in, nvlist_t **out)
1058 {
1059 	boolean_t present;
1060 	ses_node_t *np;
1061 	nvlist_t *props, *nvl;
1062 	uint64_t status;
1063 
1064 	if ((np = ses_node_lock(mod, tn)) == NULL)
1065 		return (-1);
1066 
1067 	verify((props = ses_node_props(np)) != NULL);
1068 	verify(nvlist_lookup_uint64(props,
1069 	    SES_PROP_STATUS_CODE, &status) == 0);
1070 
1071 	ses_node_unlock(mod, tn);
1072 
1073 	present = (status != SES_ESC_NOT_INSTALLED);
1074 
1075 	if (topo_mod_nvalloc(mod, &nvl, NV_UNIQUE_NAME) != 0)
1076 		return (topo_mod_seterrno(mod, EMOD_FMRI_NVL));
1077 
1078 	if (nvlist_add_uint32(nvl, TOPO_METH_PRESENT_RET,
1079 	    present) != 0) {
1080 		nvlist_free(nvl);
1081 		return (topo_mod_seterrno(mod, EMOD_FMRI_NVL));
1082 	}
1083 
1084 	*out = nvl;
1085 
1086 	return (0);
1087 }
1088 
1089 /*
1090  * Sets standard properties for a ses node (enclosure, bay, controller
1091  * or expander).
1092  * This includes setting the FRU, as well as setting the
1093  * authority information.  When  the fru topo node(frutn) is not NULL
1094  * its resouce should be used as FRU.
1095  */
1096 static int
1097 ses_set_standard_props(topo_mod_t *mod, tnode_t *frutn, tnode_t *tn,
1098     nvlist_t *auth, uint64_t nodeid, const char *path)
1099 {
1100 	int err;
1101 	char *product, *chassis;
1102 	nvlist_t *fmri;
1103 
1104 	/*
1105 	 * Set the authority explicitly if specified.
1106 	 */
1107 	if (auth) {
1108 		verify(nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT,
1109 		    &product) == 0);
1110 		verify(nvlist_lookup_string(auth, FM_FMRI_AUTH_CHASSIS,
1111 		    &chassis) == 0);
1112 		if (topo_prop_set_string(tn, FM_FMRI_AUTHORITY,
1113 		    FM_FMRI_AUTH_PRODUCT, TOPO_PROP_IMMUTABLE, product,
1114 		    &err) != 0 ||
1115 		    topo_prop_set_string(tn, FM_FMRI_AUTHORITY,
1116 		    FM_FMRI_AUTH_CHASSIS, TOPO_PROP_IMMUTABLE, chassis,
1117 		    &err) != 0 ||
1118 		    topo_prop_set_string(tn, FM_FMRI_AUTHORITY,
1119 		    FM_FMRI_AUTH_SERVER, TOPO_PROP_IMMUTABLE, "",
1120 		    &err) != 0) {
1121 			topo_mod_dprintf(mod, "failed to add authority "
1122 			    "properties: %s\n", topo_strerror(err));
1123 			return (topo_mod_seterrno(mod, err));
1124 		}
1125 	}
1126 
1127 	/*
1128 	 * Copy the resource and set that as the FRU.
1129 	 */
1130 	if (frutn != NULL) {
1131 		if (topo_node_resource(frutn, &fmri, &err) != 0) {
1132 			topo_mod_dprintf(mod,
1133 			    "topo_node_resource() failed : %s\n",
1134 			    topo_strerror(err));
1135 			return (topo_mod_seterrno(mod, err));
1136 		}
1137 	} else {
1138 		if (topo_node_resource(tn, &fmri, &err) != 0) {
1139 			topo_mod_dprintf(mod,
1140 			    "topo_node_resource() failed : %s\n",
1141 			    topo_strerror(err));
1142 			return (topo_mod_seterrno(mod, err));
1143 		}
1144 	}
1145 
1146 	if (topo_node_fru_set(tn, fmri, 0, &err) != 0) {
1147 		topo_mod_dprintf(mod,
1148 		    "topo_node_fru_set() failed : %s\n",
1149 		    topo_strerror(err));
1150 		nvlist_free(fmri);
1151 		return (topo_mod_seterrno(mod, err));
1152 	}
1153 
1154 	nvlist_free(fmri);
1155 
1156 	/*
1157 	 * Set the SES-specific properties so that consumers can query
1158 	 * additional information about the particular SES element.
1159 	 */
1160 	if (topo_pgroup_create(tn, &ses_pgroup, &err) != 0) {
1161 		topo_mod_dprintf(mod, "failed to create propgroup "
1162 		    "%s: %s\n", TOPO_PGROUP_SES, topo_strerror(err));
1163 		return (-1);
1164 	}
1165 
1166 	if (topo_prop_set_uint64(tn, TOPO_PGROUP_SES,
1167 	    TOPO_PROP_NODE_ID, TOPO_PROP_IMMUTABLE,
1168 	    nodeid, &err) != 0) {
1169 		topo_mod_dprintf(mod,
1170 		    "failed to create property %s: %s\n",
1171 		    TOPO_PROP_NODE_ID, topo_strerror(err));
1172 		return (-1);
1173 	}
1174 
1175 	if (topo_prop_set_string(tn, TOPO_PGROUP_SES,
1176 	    TOPO_PROP_TARGET_PATH, TOPO_PROP_IMMUTABLE,
1177 	    path, &err) != 0) {
1178 		topo_mod_dprintf(mod,
1179 		    "failed to create property %s: %s\n",
1180 		    TOPO_PROP_TARGET_PATH, topo_strerror(err));
1181 		return (-1);
1182 	}
1183 
1184 	return (0);
1185 }
1186 
1187 /*
1188  * Iterate over the SES phy information. If any of the ports indicates that it's
1189  * a SATA device and we haven't matched any disk devices yet, that means
1190  * that the HBA was able to create a WWN for the SATA device based on its GUID,
1191  * which is good. However, SES includes the WWN for the device's STP bridge. In
1192  * theory, if the driver includes the WWN based on the SATA guid then it should
1193  * also set the bridge-port property indicating the WWN that should match the
1194  * SATA device.
1195  */
1196 static int
1197 ses_create_disk_bridge(ses_enum_data_t *sdp, tnode_t *pnode, nvlist_t *props,
1198     tnode_t **child)
1199 {
1200 	nvlist_t **phys;
1201 	uint_t i, n_phys;
1202 	topo_mod_t *mod = sdp->sed_mod;
1203 
1204 	if (nvlist_lookup_nvlist_array(props, SES_SAS_PROP_PHYS, &phys,
1205 	    &n_phys) != 0)
1206 		return (1);
1207 
1208 	for (i = 0; i < n_phys; i++) {
1209 		uint64_t wwn;
1210 		boolean_t sata;
1211 		char wwnstr[64];
1212 
1213 		if (nvlist_lookup_uint64(phys[i], SES_SAS_PROP_ADDR,
1214 		    &wwn) != 0 || wwn == 0) {
1215 			continue;
1216 		}
1217 
1218 		if (nvlist_lookup_boolean_value(phys[i],
1219 		    SES_SAS_PROP_SATA_DEVICE, &sata) != 0 || !sata) {
1220 			continue;
1221 		}
1222 
1223 		if (scsi_wwn_to_wwnstr(wwn, 0, wwnstr) == NULL)
1224 			continue;
1225 
1226 		if (disk_declare_bridge(mod, pnode, &sdp->sed_devs,
1227 		    wwnstr, child) == 0) {
1228 			return (0);
1229 		}
1230 
1231 	}
1232 
1233 	return (1);
1234 }
1235 
1236 /*
1237  * Callback to add a disk to a given bay.  We first check the status-code to
1238  * determine if a disk is present, ignoring those that aren't in an appropriate
1239  * state.  We then scan the parent bay node's SAS address array to determine
1240  * possible attached SAS addresses.  We create a disk node if the disk is not
1241  * SAS or the SES target does not support the necessary pages for this; if we
1242  * find the SAS address, we create a disk node and also correlate it with
1243  * the corresponding Solaris device node to fill in the rest of the data.
1244  */
1245 static int
1246 ses_create_disk(ses_enum_data_t *sdp, tnode_t *pnode, nvlist_t *props)
1247 {
1248 	topo_mod_t *mod = sdp->sed_mod;
1249 	uint64_t status;
1250 	uint_t s, nsas;
1251 	char **paths;
1252 	int err, ret;
1253 	tnode_t *child = NULL;
1254 
1255 	/*
1256 	 * Skip devices that are not in a present (and possibly damaged) state.
1257 	 * Also, skip devices that this expander is either not fully wired to,
1258 	 * or are hidden due to SAS zoning, as indicated by the
1259 	 * SES_ESC_NO_ACCESS state.
1260 	 */
1261 	if (nvlist_lookup_uint64(props, SES_PROP_STATUS_CODE, &status) != 0)
1262 		return (0);
1263 
1264 	if (status != SES_ESC_UNSUPPORTED &&
1265 	    status != SES_ESC_OK &&
1266 	    status != SES_ESC_CRITICAL &&
1267 	    status != SES_ESC_NONCRITICAL &&
1268 	    status != SES_ESC_UNRECOVERABLE &&
1269 	    status != SES_ESC_UNKNOWN)
1270 		return (0);
1271 
1272 	topo_mod_dprintf(mod, "found attached disk");
1273 
1274 	/*
1275 	 * Create the disk range.
1276 	 */
1277 	if (topo_node_range_create(mod, pnode, DISK, 0, 0) != 0) {
1278 		topo_mod_dprintf(mod,
1279 		    "topo_node_create_range() failed: %s",
1280 		    topo_mod_errmsg(mod));
1281 		return (-1);
1282 	}
1283 
1284 	/*
1285 	 * Look through all SAS addresses and attempt to correlate them to a
1286 	 * known Solaris device.  If we don't find a matching node, then we
1287 	 * don't enumerate the disk node.
1288 	 * Note that TOPO_PROP_SAS_ADDR prop includes SAS address from
1289 	 * alternate elements that represent the same device.
1290 	 */
1291 	if (topo_prop_get_string_array(pnode, TOPO_PGROUP_SES,
1292 	    TOPO_PROP_SAS_ADDR, &paths, &nsas, &err) != 0)
1293 		return (0);
1294 
1295 	err = 0;
1296 
1297 	for (s = 0; s < nsas; s++) {
1298 		ret = disk_declare_addr(mod, pnode, &sdp->sed_devs, paths[s],
1299 		    &child);
1300 		if (ret == 0) {
1301 			break;
1302 		} else if (ret < 0) {
1303 			err = -1;
1304 			break;
1305 		}
1306 	}
1307 
1308 	/*
1309 	 * We need to take another pass through the properties for this bay by
1310 	 * iterating over the phys and noting if any of these are SATA. Note,
1311 	 * this information isn't commonly part of the topo tree at this time,
1312 	 * hence why we end up going back and iterating over the properties
1313 	 * ourselves.
1314 	 */
1315 	if (s == nsas) {
1316 		if (ses_create_disk_bridge(sdp, pnode, props, &child) != 0)
1317 			(void) disk_declare_non_enumerated(mod, pnode, &child);
1318 	}
1319 
1320 	/* copy sas_addresses (target-ports) from parent (with 'w'added) */
1321 	if (child != NULL) {
1322 		int i;
1323 		char **tports;
1324 		uint64_t wwn;
1325 
1326 		tports = topo_mod_zalloc(mod, sizeof (char *) * nsas);
1327 		if (tports != NULL) {
1328 			for (i = 0; i < nsas; i++) {
1329 				if (scsi_wwnstr_to_wwn(paths[i], &wwn) !=
1330 				    DDI_SUCCESS)
1331 					break;
1332 				tports[i] = scsi_wwn_to_wwnstr(wwn, 1, NULL);
1333 				if (tports[i] == NULL)
1334 					break;
1335 			}
1336 			/* if they all worked then create the property */
1337 			if (i == nsas)
1338 				(void) topo_prop_set_string_array(child,
1339 				    TOPO_PGROUP_STORAGE,
1340 				    TOPO_STORAGE_TARGET_PORT_L0IDS,
1341 				    TOPO_PROP_IMMUTABLE, (const char **)tports,
1342 				    nsas, &err);
1343 
1344 			for (i = 0; i < nsas; i++)
1345 				if (tports[i] != NULL)
1346 					scsi_free_wwnstr(tports[i]);
1347 			topo_mod_free(mod, tports, sizeof (char *) * nsas);
1348 		}
1349 	}
1350 
1351 	for (s = 0; s < nsas; s++)
1352 		topo_mod_free(mod, paths[s], strlen(paths[s]) + 1);
1353 	topo_mod_free(mod, paths, nsas * sizeof (char *));
1354 
1355 	return (err);
1356 }
1357 
1358 static int
1359 ses_add_bay_props(topo_mod_t *mod, tnode_t *tn, ses_enum_node_t *snp)
1360 {
1361 	ses_alt_node_t *ap;
1362 	ses_node_t *np;
1363 	nvlist_t *props;
1364 
1365 	nvlist_t **phys;
1366 	uint_t i, j, n_phys, all_phys = 0;
1367 	char **paths;
1368 	uint64_t addr;
1369 	size_t len;
1370 	int terr, err = -1;
1371 
1372 	for (ap = topo_list_next(&snp->sen_alt_nodes); ap != NULL;
1373 	    ap = topo_list_next(ap)) {
1374 		np = ap->san_node;
1375 		props = ses_node_props(np);
1376 
1377 		if (nvlist_lookup_nvlist_array(props, SES_SAS_PROP_PHYS,
1378 		    &phys, &n_phys) != 0)
1379 			continue;
1380 
1381 		all_phys += n_phys;
1382 	}
1383 
1384 	if (all_phys == 0)
1385 		return (0);
1386 
1387 	if ((paths = topo_mod_zalloc(mod, all_phys * sizeof (char *))) == NULL)
1388 		return (-1);
1389 
1390 	for (i = 0, ap = topo_list_next(&snp->sen_alt_nodes); ap != NULL;
1391 	    ap = topo_list_next(ap)) {
1392 		np = ap->san_node;
1393 		props = ses_node_props(np);
1394 
1395 		if (nvlist_lookup_nvlist_array(props, SES_SAS_PROP_PHYS,
1396 		    &phys, &n_phys) != 0)
1397 			continue;
1398 
1399 		for (j = 0; j < n_phys; j++) {
1400 			if (nvlist_lookup_uint64(phys[j], SES_SAS_PROP_ADDR,
1401 			    &addr) != 0)
1402 				continue;
1403 
1404 			len = snprintf(NULL, 0, "%016llx", addr) + 1;
1405 			if ((paths[i] = topo_mod_alloc(mod, len)) == NULL)
1406 				goto error;
1407 
1408 			(void) snprintf(paths[i], len, "%016llx", addr);
1409 
1410 			++i;
1411 		}
1412 	}
1413 
1414 	err = topo_prop_set_string_array(tn, TOPO_PGROUP_SES,
1415 	    TOPO_PROP_SAS_ADDR, TOPO_PROP_IMMUTABLE,
1416 	    (const char **)paths, i, &terr);
1417 	if (err != 0)
1418 		err = topo_mod_seterrno(mod, terr);
1419 
1420 error:
1421 	for (i = 0; i < all_phys && paths[i] != NULL; i++)
1422 		topo_mod_free(mod, paths[i], strlen(paths[i]) + 1);
1423 	topo_mod_free(mod, paths, all_phys * sizeof (char *));
1424 
1425 	return (err);
1426 }
1427 
1428 static const char *
1429 lookup_bay_override(const char *product_id)
1430 {
1431 	for (uint_t i = 0; i < N_BAY_LBL_OVERRIDES; i++) {
1432 		if (strcmp(product_id,
1433 		    bay_label_overrides[i].slbl_product) == 0) {
1434 			return (bay_label_overrides[i].slbl_mname);
1435 		}
1436 	}
1437 	return (NULL);
1438 }
1439 
1440 /*
1441  * Callback to create a basic node (bay, psu, fan, or controller and expander).
1442  */
1443 static int
1444 ses_create_generic(ses_enum_data_t *sdp, ses_enum_node_t *snp, tnode_t *pnode,
1445     tnode_t *frutn, const char *nodename, const char *labelname,
1446     tnode_t **node)
1447 {
1448 	ses_node_t *np = snp->sen_node;
1449 	ses_node_t *parent;
1450 	uint64_t instance = snp->sen_instance;
1451 	topo_mod_t *mod = sdp->sed_mod;
1452 	nvlist_t *props, *aprops;
1453 	nvlist_t *auth = NULL, *fmri = NULL;
1454 	tnode_t *tn = NULL;
1455 	char *clean_label = NULL, label[128];
1456 	int err;
1457 	char *part = NULL, *serial = NULL, *revision = NULL;
1458 	char *desc;
1459 	boolean_t report;
1460 
1461 	props = ses_node_props(np);
1462 
1463 	(void) nvlist_lookup_string(props, LIBSES_PROP_PART, &part);
1464 	(void) nvlist_lookup_string(props, LIBSES_PROP_SERIAL, &serial);
1465 
1466 	topo_mod_dprintf(mod, "adding %s %llu", nodename, instance);
1467 
1468 	/*
1469 	 * Create the node.  The interesting information is all copied from the
1470 	 * parent enclosure node, so there is not much to do.
1471 	 */
1472 	if ((auth = topo_mod_auth(mod, pnode)) == NULL)
1473 		goto error;
1474 
1475 	/*
1476 	 * We want to report revision information for the controller nodes, but
1477 	 * we do not get per-element revision information.  However, we do have
1478 	 * revision information for the entire enclosure, and we can use the
1479 	 * 'reported-via' property to know that this controller corresponds to
1480 	 * the given revision information.  This means we cannot get revision
1481 	 * information for targets we are not explicitly connected to, but
1482 	 * there is little we can do about the situation.
1483 	 */
1484 	if (strcmp(nodename, CONTROLLER) == 0 &&
1485 	    nvlist_lookup_boolean_value(props, SES_PROP_REPORT, &report) == 0 &&
1486 	    report) {
1487 		for (parent = ses_node_parent(np); parent != NULL;
1488 		    parent = ses_node_parent(parent)) {
1489 			if (ses_node_type(parent) == SES_NODE_ENCLOSURE) {
1490 				(void) nvlist_lookup_string(
1491 				    ses_node_props(parent),
1492 				    SES_EN_PROP_REV, &revision);
1493 				break;
1494 			}
1495 		}
1496 	}
1497 
1498 	if ((fmri = topo_mod_hcfmri(mod, pnode, FM_HC_SCHEME_VERSION,
1499 	    nodename, (topo_instance_t)instance, NULL, auth, part, revision,
1500 	    serial)) == NULL) {
1501 		topo_mod_dprintf(mod, "topo_mod_hcfmri() failed: %s",
1502 		    topo_mod_errmsg(mod));
1503 		goto error;
1504 	}
1505 
1506 	if ((tn = topo_node_bind(mod, pnode, nodename,
1507 	    instance, fmri)) == NULL) {
1508 		topo_mod_dprintf(mod, "topo_node_bind() failed: %s",
1509 		    topo_mod_errmsg(mod));
1510 		goto error;
1511 	}
1512 
1513 	/*
1514 	 * For the node label, we look for the following in order:
1515 	 *
1516 	 * <ses-description>
1517 	 * <ses-class-description> <instance>
1518 	 * <default-type-label> <instance>
1519 	 */
1520 	if (nvlist_lookup_string(props, SES_PROP_DESCRIPTION, &desc) != 0 ||
1521 	    desc[0] == '\0') {
1522 		parent = ses_node_parent(np);
1523 		aprops = ses_node_props(parent);
1524 		if (nvlist_lookup_string(aprops, SES_PROP_CLASS_DESCRIPTION,
1525 		    &desc) != 0 || desc[0] == '\0')
1526 			desc = (char *)labelname;
1527 		(void) snprintf(label, sizeof (label), "%s %llu", desc,
1528 		    instance);
1529 		desc = label;
1530 	}
1531 
1532 	if ((clean_label = topo_mod_clean_str(mod, desc)) == NULL)
1533 		goto error;
1534 
1535 	if (topo_prop_set_string(tn, TOPO_PGROUP_PROTOCOL, TOPO_PROP_LABEL,
1536 	    TOPO_PROP_MUTABLE, clean_label, &err) < 0)
1537 		goto error;
1538 
1539 	if (ses_set_standard_props(mod, frutn, tn, NULL, ses_node_id(np),
1540 	    snp->sen_target->set_devpath) != 0)
1541 		goto error;
1542 
1543 	if (strcmp(nodename, BAY) == 0) {
1544 		const char *label_method;
1545 		char *product;
1546 		nvlist_t *args = NULL;
1547 
1548 		if (ses_add_bay_props(mod, tn, snp) != 0)
1549 			goto error;
1550 
1551 		if (topo_method_register(mod, tn, ses_bay_methods) != 0) {
1552 			topo_mod_dprintf(mod,
1553 			    "topo_method_register() failed: %s",
1554 			    topo_mod_errmsg(mod));
1555 			goto error;
1556 		}
1557 
1558 		/*
1559 		 * Ideally we'd perform this sort of override with a platform
1560 		 * specific XML map file, and that would work here if we only
1561 		 * wanted to override the bay node label.  However, we'd also
1562 		 * like the disk node label (if the bay is occupied) to inherit
1563 		 * the overriden bay label.  So we need to ensure the
1564 		 * propmethod is registered before we create the child disk
1565 		 * node.
1566 		 */
1567 		if ((product = topo_mod_product(mod)) == NULL) {
1568 			(void) topo_mod_seterrno(mod, EMOD_NOMEM);
1569 			goto error;
1570 		}
1571 		if ((label_method = lookup_bay_override(product)) != NULL) {
1572 			if (topo_mod_nvalloc(mod, &args, NV_UNIQUE_NAME) != 0 ||
1573 			    topo_prop_method_register(tn, TOPO_PGROUP_PROTOCOL,
1574 			    TOPO_PROP_LABEL, TOPO_TYPE_STRING, label_method,
1575 			    args, &err)) {
1576 				topo_mod_dprintf(mod,
1577 				    "Failed to register method: %s on %s=%"
1578 				    PRIu64, label_method, BAY,
1579 				    topo_node_instance(tn));
1580 				topo_mod_strfree(mod, product);
1581 				nvlist_free(args);
1582 				goto error;
1583 			}
1584 			nvlist_free(args);
1585 		}
1586 		topo_mod_strfree(mod, product);
1587 
1588 		if (ses_create_disk(sdp, tn, props) != 0)
1589 			goto error;
1590 	} else if ((strcmp(nodename, FAN) == 0) ||
1591 	    (strcmp(nodename, PSU) == 0) ||
1592 	    (strcmp(nodename, CONTROLLER) == 0)) {
1593 		/*
1594 		 * Only fan, psu, and controller nodes have a 'present' method.
1595 		 * Bay nodes are always present, and disk nodes are present by
1596 		 * virtue of being enumerated and SAS expander nodes and
1597 		 * SAS connector nodes are also always present once
1598 		 * the parent controller is found.
1599 		 */
1600 		if (topo_method_register(mod, tn, ses_component_methods) != 0) {
1601 			topo_mod_dprintf(mod,
1602 			    "topo_method_register() failed: %s",
1603 			    topo_mod_errmsg(mod));
1604 			goto error;
1605 		}
1606 
1607 	}
1608 
1609 	snp->sen_target->set_refcount++;
1610 	topo_node_setspecific(tn, snp->sen_target);
1611 
1612 	nvlist_free(auth);
1613 	nvlist_free(fmri);
1614 	topo_mod_strfree(mod, clean_label);
1615 	if (node != NULL) *node = tn;
1616 	return (0);
1617 
1618 error:
1619 	nvlist_free(auth);
1620 	nvlist_free(fmri);
1621 	topo_mod_strfree(mod, clean_label);
1622 	return (-1);
1623 }
1624 
1625 /*
1626  * Create SAS expander specific props.
1627  */
1628 /*ARGSUSED*/
1629 static int
1630 ses_set_expander_props(ses_enum_data_t *sdp, ses_enum_node_t *snp,
1631     tnode_t *ptnode, tnode_t *tnode, int *phycount, int64_t *connlist)
1632 {
1633 	ses_node_t *np = snp->sen_node;
1634 	topo_mod_t *mod = sdp->sed_mod;
1635 	nvlist_t *auth = NULL, *fmri = NULL;
1636 	nvlist_t *props, **phylist;
1637 	int err, i;
1638 	uint_t pcount;
1639 	uint64_t sasaddr, connidx;
1640 	char sasaddr_str[17];
1641 	boolean_t found = B_FALSE, ses_found = B_FALSE;
1642 	dev_di_node_t *dnode, *sesdnode;
1643 
1644 	props = ses_node_props(np);
1645 
1646 	/*
1647 	 * the uninstalled expander is not enumerated by checking
1648 	 * the element status code.  No present present' method provided.
1649 	 */
1650 	/*
1651 	 * Get the Expander SAS address.  It should exist.
1652 	 */
1653 	if (nvlist_lookup_uint64(props, SES_EXP_PROP_SAS_ADDR,
1654 	    &sasaddr) != 0) {
1655 		topo_mod_dprintf(mod,
1656 		    "Failed to get prop %s.", SES_EXP_PROP_SAS_ADDR);
1657 		goto error;
1658 	}
1659 
1660 	(void) sprintf(sasaddr_str, "%llx", sasaddr);
1661 
1662 	/* search matching dev_di_node. */
1663 	for (dnode = topo_list_next(&sdp->sed_devs); dnode != NULL;
1664 	    dnode = topo_list_next(dnode)) {
1665 		for (i = 0; i < dnode->ddn_ppath_count; i++) {
1666 			if ((dnode->ddn_target_port[i] != NULL) &&
1667 			    (strstr(dnode->ddn_target_port[i],
1668 			    sasaddr_str) != NULL)) {
1669 				found = B_TRUE;
1670 				break;
1671 			}
1672 		}
1673 		if (found)
1674 			break;
1675 	}
1676 
1677 	if (!found) {
1678 		topo_mod_dprintf(mod,
1679 		    "ses_set_expander_props: Failed to find matching "
1680 		    "devinfo node for Exapnder SAS address %s",
1681 		    SES_EXP_PROP_SAS_ADDR);
1682 		/* continue on to get storage group props. */
1683 	} else {
1684 		/* create/set the devfs-path and devid in the smp group */
1685 		if (topo_pgroup_create(tnode, &smp_pgroup, &err) != 0) {
1686 			topo_mod_dprintf(mod, "ses_set_expander_props: "
1687 			    "failed to create smp property group %s\n",
1688 			    topo_strerror(err));
1689 			goto error;
1690 		} else {
1691 			if (topo_prop_set_string(tnode, TOPO_PGROUP_SMP,
1692 			    TOPO_PROP_SMP_TARGET_PORT, TOPO_PROP_IMMUTABLE,
1693 			    dnode->ddn_target_port[i], &err) != 0) {
1694 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1695 				    "set %S error %s\n", TOPO_PROP_SAS_ADDR,
1696 				    topo_strerror(err));
1697 			}
1698 			if (topo_prop_set_string(tnode, TOPO_PGROUP_SMP,
1699 			    TOPO_PROP_SMP_DEV_PATH, TOPO_PROP_IMMUTABLE,
1700 			    dnode->ddn_dpath, &err) != 0) {
1701 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1702 				    "set dev error %s\n", topo_strerror(err));
1703 			}
1704 			if (topo_prop_set_string(tnode, TOPO_PGROUP_SMP,
1705 			    TOPO_PROP_SMP_DEVID, TOPO_PROP_IMMUTABLE,
1706 			    dnode->ddn_devid, &err) != 0) {
1707 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1708 				    "set devid error %s\n", topo_strerror(err));
1709 			}
1710 			if (dnode->ddn_ppath_count != 0 &&
1711 			    topo_prop_set_string_array(tnode, TOPO_PGROUP_SMP,
1712 			    TOPO_PROP_SMP_PHYS_PATH, TOPO_PROP_IMMUTABLE,
1713 			    (const char **)dnode->ddn_ppath,
1714 			    dnode->ddn_ppath_count, &err) != 0) {
1715 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1716 				    "set phys-path error %s\n",
1717 				    topo_strerror(err));
1718 			}
1719 		}
1720 	}
1721 
1722 	/* update the ses property group with SES target info */
1723 	if ((topo_pgroup_create(tnode, &ses_pgroup, &err) != 0) &&
1724 	    (err != ETOPO_PROP_DEFD)) {
1725 		/* SES prop group doesn't exist but failed to be created. */
1726 		topo_mod_dprintf(mod, "ses_set_expander_props: "
1727 		    "ses pgroup create error %s\n", topo_strerror(err));
1728 		goto error;
1729 	} else {
1730 		/* locate assciated enclosure dev_di_node. */
1731 		for (sesdnode = topo_list_next(&sdp->sed_devs);
1732 		    sesdnode != NULL; sesdnode = topo_list_next(sesdnode)) {
1733 			for (i = 0; i < sesdnode->ddn_ppath_count; i++) {
1734 				/*
1735 				 * check if attached port exists and
1736 				 * its node type is enclosure and
1737 				 * attached port is same as sas address of
1738 				 * the expander and
1739 				 * bridge port for virtual phy indication
1740 				 * exist.
1741 				 */
1742 				if ((sesdnode->ddn_attached_port[i] != NULL) &&
1743 				    (sesdnode->ddn_dtype == DTYPE_ESI) &&
1744 				    (strstr(sesdnode->ddn_attached_port[i],
1745 				    sasaddr_str) != NULL) &&
1746 				    (sesdnode->ddn_bridge_port[i] != NULL)) {
1747 					ses_found = B_TRUE;
1748 					break;
1749 				}
1750 			}
1751 			if (ses_found) break;
1752 		}
1753 
1754 		if (ses_found) {
1755 			if (topo_prop_set_string(tnode, TOPO_PGROUP_SES,
1756 			    TOPO_PROP_SES_TARGET_PORT, TOPO_PROP_IMMUTABLE,
1757 			    sesdnode->ddn_target_port[i], &err) != 0) {
1758 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1759 				    "set ses %S error %s\n", TOPO_PROP_SAS_ADDR,
1760 				    topo_strerror(err));
1761 			}
1762 			if (topo_prop_set_string(tnode, TOPO_PGROUP_SES,
1763 			    TOPO_PROP_SES_DEV_PATH, TOPO_PROP_IMMUTABLE,
1764 			    sesdnode->ddn_dpath, &err) != 0) {
1765 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1766 				    "set ses dev error %s\n",
1767 				    topo_strerror(err));
1768 			}
1769 			if (topo_prop_set_string(tnode, TOPO_PGROUP_SES,
1770 			    TOPO_PROP_SES_DEVID, TOPO_PROP_IMMUTABLE,
1771 			    sesdnode->ddn_devid, &err) != 0) {
1772 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1773 				    "set ses devid error %s\n",
1774 				    topo_strerror(err));
1775 			}
1776 			if (sesdnode->ddn_ppath_count != 0 &&
1777 			    topo_prop_set_string_array(tnode, TOPO_PGROUP_SES,
1778 			    TOPO_PROP_SES_PHYS_PATH, TOPO_PROP_IMMUTABLE,
1779 			    (const char **)sesdnode->ddn_ppath,
1780 			    sesdnode->ddn_ppath_count, &err) != 0) {
1781 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1782 				    "set ses phys-path error %s\n",
1783 				    topo_strerror(err));
1784 			}
1785 
1786 		}
1787 	}
1788 
1789 	/* create the storage group */
1790 	if (topo_pgroup_create(tnode, &storage_pgroup, &err) != 0) {
1791 		topo_mod_dprintf(mod, "ses_set_expander_props: "
1792 		    "create storage error %s\n", topo_strerror(err));
1793 		goto error;
1794 	} else {
1795 		/* set the SAS address prop out of expander element status. */
1796 		if (topo_prop_set_string(tnode, TOPO_PGROUP_STORAGE,
1797 		    TOPO_PROP_SAS_ADDR, TOPO_PROP_IMMUTABLE, sasaddr_str,
1798 		    &err) != 0) {
1799 			topo_mod_dprintf(mod, "ses_set_expander_props: "
1800 			    "set %S error %s\n", TOPO_PROP_SAS_ADDR,
1801 			    topo_strerror(err));
1802 		}
1803 
1804 		/* Get the phy information for the expander */
1805 		if (nvlist_lookup_nvlist_array(props, SES_SAS_PROP_PHYS,
1806 		    &phylist, &pcount) != 0) {
1807 			topo_mod_dprintf(mod,
1808 			    "Failed to get prop %s.", SES_SAS_PROP_PHYS);
1809 		} else {
1810 			/*
1811 			 * For each phy, get the connector element index and
1812 			 * stores into connector element index array.
1813 			 */
1814 			*phycount = pcount;
1815 			for (i = 0; i < pcount; i++) {
1816 				if (nvlist_lookup_uint64(phylist[i],
1817 				    SES_PROP_CE_IDX, &connidx) == 0) {
1818 					if (connidx != 0xff) {
1819 						connlist[i] = connidx;
1820 					} else {
1821 						connlist[i] = -1;
1822 					}
1823 				} else {
1824 					/* Fail to get the index. set to -1. */
1825 					connlist[i] = -1;
1826 				}
1827 			}
1828 
1829 			/* set the phy count prop of the expander. */
1830 			if (topo_prop_set_uint64(tnode, TOPO_PGROUP_STORAGE,
1831 			    TOPO_PROP_PHY_COUNT, TOPO_PROP_IMMUTABLE, pcount,
1832 			    &err) != 0) {
1833 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1834 				    "set %S error %s\n", TOPO_PROP_PHY_COUNT,
1835 				    topo_strerror(err));
1836 			}
1837 
1838 			/*
1839 			 * set the connector element index of
1840 			 * the expander phys.
1841 			 */
1842 		}
1843 
1844 		/* populate other misc storage group properties */
1845 		if (found) {
1846 			if (dnode->ddn_mfg && (topo_prop_set_string(tnode,
1847 			    TOPO_PGROUP_STORAGE, TOPO_STORAGE_MANUFACTURER,
1848 			    TOPO_PROP_IMMUTABLE, dnode->ddn_mfg, &err) != 0)) {
1849 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1850 				    "set mfg error %s\n", topo_strerror(err));
1851 			}
1852 
1853 			if (dnode->ddn_model && (topo_prop_set_string(tnode,
1854 			    TOPO_PGROUP_STORAGE, TOPO_STORAGE_MODEL,
1855 			    TOPO_PROP_IMMUTABLE,
1856 			    dnode->ddn_model, &err) != 0)) {
1857 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1858 				    "set model error %s\n", topo_strerror(err));
1859 			}
1860 
1861 			if (dnode->ddn_serial && (topo_prop_set_string(tnode,
1862 			    TOPO_PGROUP_STORAGE, TOPO_STORAGE_SERIAL_NUM,
1863 			    TOPO_PROP_IMMUTABLE,
1864 			    dnode->ddn_serial, &err) != 0)) {
1865 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1866 				    "set serial error %s\n",
1867 				    topo_strerror(err));
1868 			}
1869 
1870 			if (dnode->ddn_firm && (topo_prop_set_string(tnode,
1871 			    TOPO_PGROUP_STORAGE,
1872 			    TOPO_STORAGE_FIRMWARE_REV, TOPO_PROP_IMMUTABLE,
1873 			    dnode->ddn_firm, &err) != 0)) {
1874 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1875 				    "set firm error %s\n", topo_strerror(err));
1876 			}
1877 		}
1878 	}
1879 
1880 	return (0);
1881 
1882 error:
1883 	nvlist_free(auth);
1884 	nvlist_free(fmri);
1885 	return (-1);
1886 }
1887 
1888 /*
1889  * Create SAS expander specific props.
1890  */
1891 /*ARGSUSED*/
1892 static int
1893 ses_set_connector_props(ses_enum_data_t *sdp, ses_enum_node_t *snp,
1894     tnode_t *tnode, int64_t phy_mask)
1895 {
1896 	ses_node_t *np = snp->sen_node;
1897 	topo_mod_t *mod = sdp->sed_mod;
1898 	nvlist_t *props;
1899 	int err, i;
1900 	uint64_t conntype;
1901 	char phymask_str[17], *conntype_str;
1902 	boolean_t   found;
1903 
1904 	props = ses_node_props(np);
1905 
1906 	/*
1907 	 * convert phy mask to string.
1908 	 */
1909 	(void) snprintf(phymask_str, 17, "%llx", phy_mask);
1910 
1911 	/* create the storage group */
1912 	if (topo_pgroup_create(tnode, &storage_pgroup, &err) != 0) {
1913 		topo_mod_dprintf(mod, "ses_set_expander_props: "
1914 		    "create storage error %s\n", topo_strerror(err));
1915 		return (-1);
1916 	} else {
1917 		/* set the SAS address prop of the expander. */
1918 		if (topo_prop_set_string(tnode, TOPO_PGROUP_STORAGE,
1919 		    TOPO_STORAGE_SAS_PHY_MASK, TOPO_PROP_IMMUTABLE,
1920 		    phymask_str, &err) != 0) {
1921 			topo_mod_dprintf(mod, "ses_set_expander_props: "
1922 			    "set %S error %s\n", TOPO_STORAGE_SAS_PHY_MASK,
1923 			    topo_strerror(err));
1924 		}
1925 
1926 		/* Get the connector type information for the expander */
1927 		if (nvlist_lookup_uint64(props,
1928 		    SES_SC_PROP_CONNECTOR_TYPE, &conntype) != 0) {
1929 			topo_mod_dprintf(mod, "Failed to get prop %s.",
1930 			    TOPO_STORAGE_SAS_PHY_MASK);
1931 		} else {
1932 			found = B_FALSE;
1933 			for (i = 0; ; i++) {
1934 				if (sas_connector_type_list[i].sct_type ==
1935 				    SAS_CONNECTOR_TYPE_CODE_NOT_DEFINED) {
1936 					break;
1937 				}
1938 				if (sas_connector_type_list[i].sct_type ==
1939 				    conntype) {
1940 					conntype_str =
1941 					    sas_connector_type_list[i].sct_name;
1942 					found = B_TRUE;
1943 					break;
1944 				}
1945 			}
1946 
1947 			if (!found) {
1948 				if (conntype <
1949 				    SAS_CONNECTOR_TYPE_CODE_NOT_DEFINED) {
1950 					conntype_str =
1951 					    SAS_CONNECTOR_TYPE_RESERVED;
1952 				} else {
1953 					conntype_str =
1954 					    SAS_CONNECTOR_TYPE_NOT_DEFINED;
1955 				}
1956 			}
1957 
1958 			/* set the phy count prop of the expander. */
1959 			if (topo_prop_set_string(tnode, TOPO_PGROUP_STORAGE,
1960 			    TOPO_STORAGE_SAS_CONNECTOR_TYPE,
1961 			    TOPO_PROP_IMMUTABLE, conntype_str, &err) != 0) {
1962 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1963 				    "set %S error %s\n", TOPO_PROP_PHY_COUNT,
1964 				    topo_strerror(err));
1965 			}
1966 		}
1967 	}
1968 
1969 	return (0);
1970 }
1971 
1972 /*
1973  * Instantiate SAS expander nodes for a given ESC Electronics node(controller)
1974  * nodes.
1975  */
1976 /*ARGSUSED*/
1977 static int
1978 ses_create_esc_sasspecific(ses_enum_data_t *sdp, ses_enum_node_t *snp,
1979     tnode_t *pnode, ses_enum_chassis_t *cp,
1980     boolean_t dorange)
1981 {
1982 	topo_mod_t *mod = sdp->sed_mod;
1983 	tnode_t	*exptn, *contn;
1984 	boolean_t found;
1985 	sas_connector_phy_data_t connectors[64] = {0};
1986 	uint64_t max;
1987 	ses_enum_node_t *ctlsnp, *xsnp, *consnp;
1988 	ses_node_t *np = snp->sen_node;
1989 	nvlist_t *props, *psprops;
1990 	uint64_t index, psindex, conindex, psstatus, i, j, count;
1991 	int64_t cidxlist[256] = {0};
1992 	int phycount;
1993 
1994 	props = ses_node_props(np);
1995 
1996 	if (nvlist_lookup_uint64(props, SES_PROP_ELEMENT_ONLY_INDEX,
1997 	    &index) != 0)
1998 		return (-1);
1999 
2000 	/*
2001 	 * For SES constroller node, check to see if there are
2002 	 * associated SAS expanders.
2003 	 */
2004 	found = B_FALSE;
2005 	max = 0;
2006 	for (ctlsnp = topo_list_next(&cp->sec_nodes); ctlsnp != NULL;
2007 	    ctlsnp = topo_list_next(ctlsnp)) {
2008 		if (ctlsnp->sen_type == SES_ET_SAS_EXPANDER) {
2009 			found = B_TRUE;
2010 			if (ctlsnp->sen_instance > max)
2011 				max = ctlsnp->sen_instance;
2012 		}
2013 	}
2014 
2015 	/*
2016 	 * No SAS expander found notthing to process.
2017 	 */
2018 	if (!found)
2019 		return (0);
2020 
2021 	topo_mod_dprintf(mod, "%s Controller %d: creating "
2022 	    "%llu %s nodes", cp->sec_csn, index, max + 1, SASEXPANDER);
2023 
2024 	/*
2025 	 * The max number represent the number of elements
2026 	 * deducted from the highest SES_PROP_ELEMENT_CLASS_INDEX
2027 	 * of SET_ET_SAS_EXPANDER type element.
2028 	 *
2029 	 * There may be multiple ESC Electronics element(controllers)
2030 	 * within JBOD(typicall two for redundancy) and SAS expander
2031 	 * elements are associated with only one of them.  We are
2032 	 * still creating the range based max number here.
2033 	 * That will cover the case that all expanders are associated
2034 	 * with one SES controller.
2035 	 */
2036 	if (dorange && topo_node_range_create(mod, pnode,
2037 	    SASEXPANDER, 0, max) != 0) {
2038 		topo_mod_dprintf(mod,
2039 		    "topo_node_create_range() failed: %s",
2040 		    topo_mod_errmsg(mod));
2041 		return (-1);
2042 	}
2043 
2044 	/*
2045 	 * Search exapnders with the parent index matching with
2046 	 * ESC Electronics element index.
2047 	 * Note the index used here is a global index across
2048 	 * SES elements.
2049 	 */
2050 	for (xsnp = topo_list_next(&cp->sec_nodes); xsnp != NULL;
2051 	    xsnp = topo_list_next(xsnp)) {
2052 		if (xsnp->sen_type == SES_ET_SAS_EXPANDER) {
2053 			/*
2054 			 * get the parent ESC controller.
2055 			 */
2056 			psprops = ses_node_props(xsnp->sen_node);
2057 			if (nvlist_lookup_uint64(psprops,
2058 			    SES_PROP_STATUS_CODE, &psstatus) == 0) {
2059 				if (psstatus == SES_ESC_NOT_INSTALLED) {
2060 					/*
2061 					 * Not installed.
2062 					 * Don't create a ndoe.
2063 					 */
2064 					continue;
2065 				}
2066 			} else {
2067 				/*
2068 				 * The element should have status code.
2069 				 * If not there is no way to find
2070 				 * out if the expander element exist or
2071 				 * not.
2072 				 */
2073 				continue;
2074 			}
2075 
2076 			/* Get the physical parent index to compare. */
2077 			if (nvlist_lookup_uint64(psprops,
2078 			    LIBSES_PROP_PHYS_PARENT, &psindex) == 0) {
2079 				if (index == psindex) {
2080 		/* indentation moved forward */
2081 		/*
2082 		 * Handle basic node information of SAS expander
2083 		 * element - binding to parent node and
2084 		 * allocating FMRI...
2085 		 */
2086 		if (ses_create_generic(sdp, xsnp, pnode, pnode, SASEXPANDER,
2087 		    "SAS-EXPANDER", &exptn) != 0)
2088 			continue;
2089 		/*
2090 		 * Now handle SAS expander unique portion of node creation.
2091 		 * The max nubmer of the phy count is 256 since SES-2
2092 		 * defines as 1 byte field.  The cidxlist has the same
2093 		 * number of elements.
2094 		 *
2095 		 * We use size 64 array to store the connectors.
2096 		 * Typically a connectors associated with 4 phys so that
2097 		 * matches with the max number of connecters associated
2098 		 * with an expander.
2099 		 * The phy count goes up to 38 for Sun supported
2100 		 * JBOD.
2101 		 */
2102 		(void) memset(cidxlist, 0, sizeof (int64_t) * 64);
2103 		if (ses_set_expander_props(sdp, xsnp, pnode, exptn, &phycount,
2104 		    cidxlist) != 0) {
2105 			/*
2106 			 * error on getting specific prop failed.
2107 			 * continue on.  Note that the node is
2108 			 * left bound.
2109 			 */
2110 			continue;
2111 		}
2112 
2113 		/*
2114 		 * count represetns the number of connectors discovered so far.
2115 		 */
2116 		count = 0;
2117 		(void) memset(connectors, 0,
2118 		    sizeof (sas_connector_phy_data_t) * 64);
2119 		for (i = 0; i < phycount; i++) {
2120 			if (cidxlist[i] != -1) {
2121 				/* connector index is valid. */
2122 				for (j = 0; j < count; j++) {
2123 					if (connectors[j].scpd_index ==
2124 					    cidxlist[i]) {
2125 						/*
2126 						 * Just update phy mask.
2127 						 * The postion for connector
2128 						 * index lists(cidxlist index)
2129 						 * is set.
2130 						 */
2131 						connectors[j].scpd_pm =
2132 						    connectors[j].scpd_pm |
2133 						    (1ULL << i);
2134 						break;
2135 					}
2136 				}
2137 				/*
2138 				 * If j and count matche a  new connector
2139 				 * index is found.
2140 				 */
2141 				if (j == count) {
2142 					/* add a new index and phy mask. */
2143 					connectors[count].scpd_index =
2144 					    cidxlist[i];
2145 					connectors[count].scpd_pm =
2146 					    connectors[count].scpd_pm |
2147 					    (1ULL << i);
2148 					count++;
2149 				}
2150 			}
2151 		}
2152 
2153 		/*
2154 		 * create range for the connector nodes.
2155 		 * The class index of the ses connector element
2156 		 * is set as the instance nubmer for the node.
2157 		 * Even though one expander may not have all connectors
2158 		 * are associated with we are creating the range with
2159 		 * max possible instance number.
2160 		 */
2161 		found = B_FALSE;
2162 		max = 0;
2163 		for (consnp = topo_list_next(&cp->sec_nodes);
2164 		    consnp != NULL; consnp = topo_list_next(consnp)) {
2165 			if (consnp->sen_type == SES_ET_SAS_CONNECTOR) {
2166 				psprops = ses_node_props(consnp->sen_node);
2167 				found = B_TRUE;
2168 				if (consnp->sen_instance > max)
2169 					max = consnp->sen_instance;
2170 			}
2171 		}
2172 
2173 		/*
2174 		 * No SAS connector found nothing to process.
2175 		 */
2176 		if (!found)
2177 			return (0);
2178 
2179 		if (dorange && topo_node_range_create(mod, exptn,
2180 		    RECEPTACLE, 0, max) != 0) {
2181 			topo_mod_dprintf(mod,
2182 			    "topo_node_create_range() failed: %s",
2183 			    topo_mod_errmsg(mod));
2184 			return (-1);
2185 		}
2186 
2187 		/* search matching connector element using the index. */
2188 		for (i = 0; i < count; i++) {
2189 			found = B_FALSE;
2190 			for (consnp = topo_list_next(&cp->sec_nodes);
2191 			    consnp != NULL; consnp = topo_list_next(consnp)) {
2192 				if (consnp->sen_type == SES_ET_SAS_CONNECTOR) {
2193 					psprops = ses_node_props(
2194 					    consnp->sen_node);
2195 					/*
2196 					 * Get the physical parent index to
2197 					 * compare.
2198 					 * The connector elements are children
2199 					 * of ESC Electronics element even
2200 					 * though we enumerate them under
2201 					 * an expander in libtopo.
2202 					 */
2203 					if (nvlist_lookup_uint64(psprops,
2204 					    SES_PROP_ELEMENT_ONLY_INDEX,
2205 					    &conindex) == 0) {
2206 						if (conindex ==
2207 						    connectors[i].scpd_index) {
2208 							found = B_TRUE;
2209 							break;
2210 						}
2211 					}
2212 				}
2213 			}
2214 
2215 			/* now create a libtopo node. */
2216 			if (found) {
2217 				/* Create generic props. */
2218 				if (ses_create_generic(sdp, consnp, exptn,
2219 				    topo_node_parent(exptn),
2220 				    RECEPTACLE, "RECEPTACLE", &contn) !=
2221 				    0) {
2222 					continue;
2223 				}
2224 				/* Create connector specific props. */
2225 				if (ses_set_connector_props(sdp, consnp,
2226 				    contn, connectors[i].scpd_pm) != 0) {
2227 					continue;
2228 				}
2229 				if (topo_method_register(mod, contn,
2230 				    ses_recep_methods) != 0) {
2231 					topo_mod_dprintf(mod,
2232 					    "topo_method_register() failed: "
2233 					    "%s",
2234 					    topo_mod_errmsg(mod));
2235 					continue;
2236 				}
2237 
2238 			}
2239 		}
2240 		/* end indentation change */
2241 				}
2242 			}
2243 		}
2244 	}
2245 
2246 	return (0);
2247 }
2248 
2249 /*
2250  * Instantiate any protocol specific portion of a node.
2251  */
2252 /*ARGSUSED*/
2253 static int
2254 ses_create_protocol_specific(ses_enum_data_t *sdp, ses_enum_node_t *snp,
2255     tnode_t *pnode, uint64_t type, ses_enum_chassis_t *cp,
2256     boolean_t dorange)
2257 {
2258 
2259 	if (type == SES_ET_ESC_ELECTRONICS) {
2260 		/* create SAS specific children(expanders and connectors. */
2261 		return (ses_create_esc_sasspecific(sdp, snp, pnode, cp,
2262 		    dorange));
2263 	}
2264 
2265 	return (0);
2266 }
2267 
2268 /*
2269  * Instantiate any children of a given type.
2270  */
2271 static int
2272 ses_create_children(ses_enum_data_t *sdp, tnode_t *pnode, uint64_t type,
2273     const char *nodename, const char *defaultlabel, ses_enum_chassis_t *cp,
2274     boolean_t dorange)
2275 {
2276 	topo_mod_t *mod = sdp->sed_mod;
2277 	boolean_t found;
2278 	uint64_t max;
2279 	ses_enum_node_t *snp;
2280 	tnode_t	*tn;
2281 
2282 	/*
2283 	 * First go through and count how many matching nodes we have.
2284 	 */
2285 	max = 0;
2286 	found = B_FALSE;
2287 	for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
2288 	    snp = topo_list_next(snp)) {
2289 		if (snp->sen_type == type) {
2290 			found = B_TRUE;
2291 			if (snp->sen_instance > max)
2292 				max = snp->sen_instance;
2293 		}
2294 	}
2295 
2296 	/*
2297 	 * No enclosure should export both DEVICE and ARRAY_DEVICE elements.
2298 	 * Since we map both of these to 'disk', if an enclosure does this, we
2299 	 * just ignore the array elements.
2300 	 */
2301 	if (!found ||
2302 	    (type == SES_ET_ARRAY_DEVICE && cp->sec_hasdev))
2303 		return (0);
2304 
2305 	topo_mod_dprintf(mod, "%s: creating %llu %s nodes",
2306 	    cp->sec_csn, max + 1, nodename);
2307 
2308 	if (dorange && topo_node_range_create(mod, pnode,
2309 	    nodename, 0, max) != 0) {
2310 		topo_mod_dprintf(mod,
2311 		    "topo_node_create_range() failed: %s",
2312 		    topo_mod_errmsg(mod));
2313 		return (-1);
2314 	}
2315 
2316 	for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
2317 	    snp = topo_list_next(snp)) {
2318 		if (snp->sen_type == type) {
2319 			/*
2320 			 * With flat layout of ses nodes there is no
2321 			 * way to find out the direct FRU for a node.
2322 			 * Passing NULL for fru topo node.  Note that
2323 			 * ses_create_children_from_phys_tree() provides
2324 			 * the actual direct FRU for a node.
2325 			 */
2326 			if (ses_create_generic(sdp, snp, pnode, NULL,
2327 			    nodename, defaultlabel, &tn) != 0)
2328 				return (-1);
2329 			/*
2330 			 * For some SES element there may be protocol specific
2331 			 * information to process.   Here we are processing
2332 			 * the association between enclosure controller and
2333 			 * SAS expanders.
2334 			 */
2335 			if (type == SES_ET_ESC_ELECTRONICS) {
2336 				/* create SAS expander node */
2337 				if (ses_create_protocol_specific(sdp, snp,
2338 				    tn, type, cp, dorange) != 0) {
2339 					return (-1);
2340 				}
2341 			}
2342 
2343 		}
2344 	}
2345 
2346 	return (0);
2347 }
2348 
2349 /*
2350  * Instantiate a new subchassis instance in the topology.
2351  */
2352 static int
2353 ses_create_subchassis(ses_enum_data_t *sdp, tnode_t *pnode,
2354     ses_enum_chassis_t *scp)
2355 {
2356 	topo_mod_t *mod = sdp->sed_mod;
2357 	tnode_t *tn;
2358 	nvlist_t *props;
2359 	nvlist_t *auth = NULL, *fmri = NULL;
2360 	uint64_t instance = scp->sec_instance;
2361 	char *desc;
2362 	char *clean_label = NULL, label[128];
2363 	char **paths;
2364 	int i, err;
2365 	ses_enum_target_t *stp;
2366 	int ret = -1;
2367 
2368 	/*
2369 	 * Copy authority information from parent enclosure node
2370 	 */
2371 	if ((auth = topo_mod_auth(mod, pnode)) == NULL)
2372 		goto error;
2373 
2374 	/*
2375 	 * Record the subchassis serial number in the FMRI.
2376 	 * For now, we assume that logical id is the subchassis serial number.
2377 	 * If this assumption changes in future, then the following
2378 	 * piece of code will need to be updated via an RFE.
2379 	 */
2380 	if ((fmri = topo_mod_hcfmri(mod, pnode, FM_HC_SCHEME_VERSION,
2381 	    SUBCHASSIS, (topo_instance_t)instance, NULL, auth, NULL, NULL,
2382 	    NULL)) == NULL) {
2383 		topo_mod_dprintf(mod, "topo_mod_hcfmri() failed: %s",
2384 		    topo_mod_errmsg(mod));
2385 		goto error;
2386 	}
2387 
2388 	if ((tn = topo_node_bind(mod, pnode, SUBCHASSIS,
2389 	    instance, fmri)) == NULL) {
2390 		topo_mod_dprintf(mod, "topo_node_bind() failed: %s",
2391 		    topo_mod_errmsg(mod));
2392 		goto error;
2393 	}
2394 
2395 	props = ses_node_props(scp->sec_enclosure);
2396 
2397 	/*
2398 	 * Look for the subchassis label in the following order:
2399 	 *	<ses-description>
2400 	 *	<ses-class-description> <instance>
2401 	 *	<default-type-label> <instance>
2402 	 *
2403 	 * For subchassis, the default label is "SUBCHASSIS"
2404 	 */
2405 	if (nvlist_lookup_string(props, SES_PROP_DESCRIPTION, &desc) != 0 ||
2406 	    desc[0] == '\0') {
2407 		if (nvlist_lookup_string(props, SES_PROP_CLASS_DESCRIPTION,
2408 		    &desc) == 0 && desc[0] != '\0')
2409 			(void) snprintf(label, sizeof (label), "%s %llu", desc,
2410 			    instance);
2411 		else
2412 			(void) snprintf(label, sizeof (label),
2413 			    "SUBCHASSIS %llu", instance);
2414 		desc = label;
2415 	}
2416 
2417 	if ((clean_label = topo_mod_clean_str(mod, desc)) == NULL)
2418 		goto error;
2419 
2420 	if (topo_prop_set_string(tn, TOPO_PGROUP_PROTOCOL, TOPO_PROP_LABEL,
2421 	    TOPO_PROP_MUTABLE, clean_label, &err) < 0)
2422 		goto error;
2423 
2424 	if (ses_set_standard_props(mod, NULL, tn, NULL,
2425 	    ses_node_id(scp->sec_enclosure), scp->sec_target->set_devpath) != 0)
2426 		goto error;
2427 
2428 	/*
2429 	 * Set the 'chassis-type' property for this subchassis.  This is either
2430 	 * 'ses-class-description' or 'subchassis'.
2431 	 */
2432 	if (nvlist_lookup_string(props, SES_PROP_CLASS_DESCRIPTION, &desc) != 0)
2433 		desc = "subchassis";
2434 
2435 	if (topo_prop_set_string(tn, TOPO_PGROUP_SES,
2436 	    TOPO_PROP_CHASSIS_TYPE, TOPO_PROP_IMMUTABLE, desc, &err) != 0) {
2437 		topo_mod_dprintf(mod, "failed to create property %s: %s\n",
2438 		    TOPO_PROP_CHASSIS_TYPE, topo_strerror(err));
2439 		goto error;
2440 	}
2441 
2442 	/*
2443 	 * For enclosures, we want to include all possible targets (for upgrade
2444 	 * purposes).
2445 	 */
2446 	for (i = 0, stp = topo_list_next(&scp->sec_targets); stp != NULL;
2447 	    stp = topo_list_next(stp), i++)
2448 		;
2449 
2450 	verify(i != 0);
2451 	paths = alloca(i * sizeof (char *));
2452 
2453 	for (i = 0, stp = topo_list_next(&scp->sec_targets); stp != NULL;
2454 	    stp = topo_list_next(stp), i++)
2455 		paths[i] = stp->set_devpath;
2456 
2457 	if (topo_prop_set_string_array(tn, TOPO_PGROUP_SES,
2458 	    TOPO_PROP_PATHS, TOPO_PROP_IMMUTABLE, (const char **)paths,
2459 	    i, &err) != 0) {
2460 		topo_mod_dprintf(mod, "failed to create property %s: %s\n",
2461 		    TOPO_PROP_PATHS, topo_strerror(err));
2462 		goto error;
2463 	}
2464 
2465 	if (topo_method_register(mod, tn, ses_enclosure_methods) != 0) {
2466 		topo_mod_dprintf(mod, "topo_method_register() failed: %s",
2467 		    topo_mod_errmsg(mod));
2468 		goto error;
2469 	}
2470 
2471 	/*
2472 	 * Create the nodes for controllers and bays.
2473 	 */
2474 	if (ses_create_children(sdp, tn, SES_ET_ESC_ELECTRONICS,
2475 	    CONTROLLER, "CONTROLLER", scp, B_TRUE) != 0 ||
2476 	    ses_create_children(sdp, tn, SES_ET_DEVICE,
2477 	    BAY, "BAY", scp, B_TRUE) != 0 ||
2478 	    ses_create_children(sdp, tn, SES_ET_ARRAY_DEVICE,
2479 	    BAY, "BAY", scp, B_TRUE) != 0)
2480 		goto error;
2481 
2482 	ret = 0;
2483 
2484 error:
2485 	nvlist_free(auth);
2486 	nvlist_free(fmri);
2487 	topo_mod_strfree(mod, clean_label);
2488 	return (ret);
2489 }
2490 
2491 /*
2492  * Function we use to insert a node.
2493  */
2494 static int
2495 ses_phys_tree_insert(topo_mod_t *mod, ses_phys_tree_t **sproot,
2496     ses_phys_tree_t *child)
2497 {
2498 	uint64_t ppindex, eindex, pindex;
2499 	ses_phys_tree_t *node_ptr;
2500 	int ret = 0;
2501 
2502 	assert(sproot != NULL);
2503 	assert(child != NULL);
2504 
2505 	if (*sproot == NULL) {
2506 		*sproot = child;
2507 		return (0);
2508 	}
2509 
2510 	pindex = child->spt_pindex;
2511 	ppindex = (*sproot)->spt_pindex;
2512 	eindex = (*sproot)->spt_eonlyindex;
2513 
2514 	/*
2515 	 * If the element only index of the root is same as the physical
2516 	 * parent index of a node to be added, add the node as a child of
2517 	 * the current root.
2518 	 */
2519 	if (eindex == pindex) {
2520 		(void) ses_phys_tree_insert(mod, &(*sproot)->spt_child, child);
2521 		child->spt_parent = *sproot;
2522 	} else if (ppindex == pindex) {
2523 		/*
2524 		 * if the physical parent of the current root and the child
2525 		 * is same, then this should be a sibling node.
2526 		 * Siblings can be different element types and arrange
2527 		 * them by group.
2528 		 */
2529 		if ((*sproot)->spt_senumnode->sen_type ==
2530 		    child->spt_senumnode->sen_type) {
2531 			child->spt_sibling = *sproot;
2532 			*sproot = child;
2533 		} else {
2534 			/* add a node in front of matching element type. */
2535 			node_ptr = *sproot;
2536 			while (node_ptr->spt_sibling != NULL) {
2537 				if (node_ptr->spt_sibling->
2538 				    spt_senumnode->sen_type ==
2539 				    child->spt_senumnode->sen_type) {
2540 					child->spt_sibling =
2541 					    node_ptr->spt_sibling;
2542 					node_ptr->spt_sibling = child;
2543 					break;
2544 				}
2545 				node_ptr = node_ptr->spt_sibling;
2546 			}
2547 			/* no matching.  Add the child at the end. */
2548 			if (node_ptr->spt_sibling == NULL) {
2549 				node_ptr->spt_sibling = child;
2550 			}
2551 		}
2552 		child->spt_parent = (*sproot)->spt_parent;
2553 	} else {
2554 		/*
2555 		 * The root and the node is not directly related.
2556 		 * Try to insert to the child sub-tree first and then try to
2557 		 * insert to the sibling sub-trees.  If fails for both
2558 		 * the caller will retry insertion later.
2559 		 */
2560 		if ((*sproot)->spt_child) {
2561 			ret = ses_phys_tree_insert(mod, &(*sproot)->spt_child,
2562 			    child);
2563 		}
2564 		if ((*sproot)->spt_child == NULL || ret != 0) {
2565 			if ((*sproot)->spt_sibling) {
2566 				ret = ses_phys_tree_insert(mod,
2567 				    &(*sproot)->spt_sibling, child);
2568 			} else {
2569 				ret = 1;
2570 			}
2571 		}
2572 		return (ret);
2573 	}
2574 	return (0);
2575 }
2576 
2577 /*
2578  * Construct tree view of ses elements through parent phyiscal element index.
2579  * The root of tree is already constructed using the enclosure element.
2580  */
2581 static int
2582 ses_construct_phys_tree(ses_enum_data_t *sdp, ses_enum_chassis_t *cp,
2583     ses_phys_tree_t *sproot)
2584 {
2585 	ses_enum_node_t *snp;
2586 	ses_phys_tree_t	*child;
2587 	ses_phys_tree_t	*u_watch = NULL;
2588 	ses_phys_tree_t	*u_head = NULL;
2589 	ses_phys_tree_t	*u_tail = NULL;
2590 	int u_inserted = 0, u_left = 0;
2591 	nvlist_t *props;
2592 	topo_mod_t *mod = sdp->sed_mod;
2593 
2594 	for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
2595 	    snp = topo_list_next(snp)) {
2596 		if ((child = topo_mod_zalloc(mod,
2597 		    sizeof (ses_phys_tree_t))) == NULL) {
2598 			topo_mod_dprintf(mod,
2599 			    "failed to allocate root.");
2600 			return (-1);
2601 		}
2602 		child->spt_snode = snp->sen_node;
2603 		props = ses_node_props(snp->sen_node);
2604 		if (nvlist_lookup_uint64(props,
2605 		    LIBSES_PROP_PHYS_PARENT, &child->spt_pindex) != 0) {
2606 			/*
2607 			 * the prop should exist. continue to see if
2608 			 * we can build a partial tree with other elements.
2609 			 */
2610 			topo_mod_dprintf(mod,
2611 			    "ses_construct_phys_tree(): Failed to find prop %s "
2612 			    "on ses element type %d and instance %d "
2613 			    "(CSN %s).", LIBSES_PROP_PHYS_PARENT,
2614 			    snp->sen_type, snp->sen_instance, cp->sec_csn);
2615 			topo_mod_free(mod, child, sizeof (ses_phys_tree_t));
2616 			continue;
2617 		} else {
2618 			if (nvlist_lookup_boolean_value(props,
2619 			    LIBSES_PROP_FRU, &child->spt_isfru) != 0) {
2620 				topo_mod_dprintf(mod,
2621 				    "ses_construct_phys_tree(): Failed to "
2622 				    "find prop %s on ses element type %d "
2623 				    "and instance %d (CSN %s).",
2624 				    LIBSES_PROP_FRU,
2625 				    snp->sen_type, snp->sen_instance,
2626 				    cp->sec_csn);
2627 				/*
2628 				 * Ignore if the prop doesn't exist.
2629 				 * Note that the enclosure itself should be
2630 				 * a FRU so if no FRU found the enclosure FRU
2631 				 * can be a direct FRU.
2632 				 */
2633 			}
2634 			verify(nvlist_lookup_uint64(props,
2635 			    SES_PROP_ELEMENT_ONLY_INDEX,
2636 			    &child->spt_eonlyindex) == 0);
2637 			verify(nvlist_lookup_uint64(props,
2638 			    SES_PROP_ELEMENT_CLASS_INDEX,
2639 			    &child->spt_cindex) == 0);
2640 		}
2641 		child->spt_senumnode = snp;
2642 		if (ses_phys_tree_insert(mod, &sproot, child) != 0) {
2643 			/* collect unresolved element to process later. */
2644 			if (u_head == NULL) {
2645 				u_head = child;
2646 				u_tail = child;
2647 			} else {
2648 				child->spt_sibling = u_head;
2649 				u_head = child;
2650 			}
2651 		}
2652 	}
2653 
2654 	/*
2655 	 * The parent of a child node may not be inserted yet.
2656 	 * Trying to insert the child until no child is left or
2657 	 * no child is not added further.  For the latter
2658 	 * the hierarchical relationship between elements
2659 	 * should be checked through SUNW,FRUID page.
2660 	 * u_watch is a watch dog to check the prgress of unresolved
2661 	 * node.
2662 	 */
2663 	u_watch = u_tail;
2664 	while (u_head) {
2665 		child = u_head;
2666 		u_head = u_head->spt_sibling;
2667 		if (u_head == NULL)
2668 			u_tail = NULL;
2669 		child->spt_sibling = NULL;
2670 		if (ses_phys_tree_insert(mod, &sproot, child) != 0) {
2671 			u_tail->spt_sibling = child;
2672 			u_tail = child;
2673 			if (child == u_watch) {
2674 				/*
2675 				 * We just scanned one round for the
2676 				 * unresolved list. Check to see whether we
2677 				 * have nodes inserted, if none, we should
2678 				 * break in case of an indefinite loop.
2679 				 */
2680 				if (u_inserted == 0) {
2681 					/*
2682 					 * Indicate there is unhandled node.
2683 					 * Chain free the whole unsolved
2684 					 * list here.
2685 					 */
2686 					u_left++;
2687 					break;
2688 				} else {
2689 					u_inserted = 0;
2690 					u_watch = u_tail;
2691 				}
2692 			}
2693 		} else {
2694 			/*
2695 			 * We just inserted one rpnode, increment the
2696 			 * unsolved_inserted counter. We will utilize this
2697 			 * counter to detect an indefinite insertion loop.
2698 			 */
2699 			u_inserted++;
2700 			if (child == u_watch) {
2701 				/*
2702 				 * watch dog node itself is inserted.
2703 				 * Set it to the tail and refresh the watching.
2704 				 */
2705 				u_watch = u_tail;
2706 				u_inserted = 0;
2707 				u_left = 0;
2708 			}
2709 		}
2710 	}
2711 
2712 	/* check if there is left out unresolved nodes. */
2713 	if (u_left) {
2714 		topo_mod_dprintf(mod, "ses_construct_phys_tree(): "
2715 		    "Failed to construct physical view of the following "
2716 		    "ses elements of Chassis CSN %s.", cp->sec_csn);
2717 		while (u_head) {
2718 			u_tail = u_head->spt_sibling;
2719 			topo_mod_dprintf(mod,
2720 			    "\telement type (%d) and instance (%d)",
2721 			    u_head->spt_senumnode->sen_type,
2722 			    u_head->spt_senumnode->sen_instance);
2723 			topo_mod_free(mod, u_head, sizeof (ses_phys_tree_t));
2724 			u_head = u_tail;
2725 		}
2726 		return (-1);
2727 	}
2728 
2729 	return (0);
2730 }
2731 
2732 /*
2733  * Free the whole phys tree.
2734  */
2735 static void ses_phys_tree_free(topo_mod_t *mod, ses_phys_tree_t *sproot)
2736 {
2737 	if (sproot == NULL)
2738 		return;
2739 
2740 	/* Free child tree. */
2741 	if (sproot->spt_child) {
2742 		ses_phys_tree_free(mod, sproot->spt_child);
2743 	}
2744 
2745 	/* Free sibling trees. */
2746 	if (sproot->spt_sibling) {
2747 		ses_phys_tree_free(mod, sproot->spt_sibling);
2748 	}
2749 
2750 	/* Free root node itself. */
2751 	topo_mod_free(mod, sproot, sizeof (ses_phys_tree_t));
2752 }
2753 
2754 /*
2755  * Parses phys_enum_type table to get the index of the given type.
2756  */
2757 static boolean_t
2758 is_type_enumerated(ses_phys_tree_t *node, int *index)
2759 {
2760 	int i;
2761 
2762 	for (i = 0; i < N_PHYS_ENUM_TYPES; i++) {
2763 		if (node->spt_senumnode->sen_type ==
2764 		    phys_enum_type_list[i].pet_type) {
2765 			*index = i;
2766 			return (B_TRUE);
2767 		}
2768 	}
2769 	return (B_FALSE);
2770 }
2771 
2772 /*
2773  * Recusrive routine for top-down enumeration of the tree.
2774  */
2775 static int
2776 ses_enumerate_node(ses_enum_data_t *sdp, tnode_t *pnode, ses_enum_chassis_t *cp,
2777     ses_phys_tree_t *parent, int mrange[])
2778 {
2779 	topo_mod_t *mod = sdp->sed_mod;
2780 	ses_phys_tree_t *child = NULL;
2781 	int i, ret = 0, ret_ch;
2782 	uint64_t prevtype = SES_ET_UNSPECIFIED;
2783 	ses_phys_tree_t *dirfru = NULL;
2784 	tnode_t *tn = NULL, *frutn = NULL;
2785 
2786 	if (parent == NULL) {
2787 		return (0);
2788 	}
2789 
2790 	for (child = parent->spt_child; child != NULL;
2791 	    child = child->spt_sibling) {
2792 		if (is_type_enumerated(child, &i)) {
2793 			if (prevtype != phys_enum_type_list[i].pet_type) {
2794 				/* check if range needs to be created. */
2795 				if (phys_enum_type_list[i].pet_dorange &&
2796 				    topo_node_range_create(mod, pnode,
2797 				    phys_enum_type_list[i].pet_nodename, 0,
2798 				    mrange[i]) != 0) {
2799 					topo_mod_dprintf(mod,
2800 					    "topo_node_create_range() failed: "
2801 					    "%s", topo_mod_errmsg(mod));
2802 					return (-1);
2803 				}
2804 				prevtype = phys_enum_type_list[i].pet_type;
2805 			}
2806 
2807 			if (!(child->spt_isfru)) {
2808 				for (dirfru = parent; dirfru != NULL;
2809 				    dirfru = dirfru->spt_parent) {
2810 					if (dirfru->spt_isfru) {
2811 						break;
2812 					}
2813 				}
2814 				/* found direct FRU node. */
2815 				if (dirfru) {
2816 					frutn = dirfru->spt_tnode;
2817 				} else {
2818 					frutn = NULL;
2819 				}
2820 			} else {
2821 				frutn = NULL;
2822 			}
2823 
2824 			if (ses_create_generic(sdp, child->spt_senumnode,
2825 			    pnode, frutn, phys_enum_type_list[i].pet_nodename,
2826 			    phys_enum_type_list[i].pet_defaultlabel, &tn) != 0)
2827 				return (-1);
2828 
2829 			child->spt_tnode = tn;
2830 			/*
2831 			 * For some SES element there may be protocol specific
2832 			 * information to process.   Here we are processing
2833 			 * the association between enclosure controller and
2834 			 * SAS expanders.
2835 			 */
2836 			if (phys_enum_type_list[i].pet_type ==
2837 			    SES_ET_ESC_ELECTRONICS) {
2838 				/* create SAS expander node */
2839 				if (ses_create_protocol_specific(sdp,
2840 				    child->spt_senumnode, tn,
2841 				    phys_enum_type_list[i].pet_type,
2842 				    cp, phys_enum_type_list[i].pet_dorange) !=
2843 				    0) {
2844 					return (-1);
2845 				}
2846 			}
2847 		} else {
2848 			continue;
2849 		}
2850 		ret_ch = ses_enumerate_node(sdp, tn, cp, child, mrange);
2851 		if (ret_ch)
2852 			ret = ret_ch; /* there was an error and set the ret. */
2853 	}
2854 
2855 	return (ret);
2856 }
2857 
2858 /*
2859  * Instantiate types of nodes that are specified in the hierarchy
2860  * element type list.
2861  */
2862 static int
2863 ses_create_children_from_phys_tree(ses_enum_data_t *sdp, tnode_t *pnode,
2864     ses_enum_chassis_t *cp, ses_phys_tree_t *phys_tree)
2865 {
2866 	topo_mod_t *mod = sdp->sed_mod;
2867 	int mrange[N_PHYS_ENUM_TYPES] = { 0 };
2868 	ses_enum_node_t *snp;
2869 	int i, ret;
2870 
2871 	/*
2872 	 * First get max range for each type of element to be enumerated.
2873 	 */
2874 	for (i = 0; i < N_PHYS_ENUM_TYPES; i++) {
2875 		if (phys_enum_type_list[i].pet_dorange) {
2876 			for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
2877 			    snp = topo_list_next(snp)) {
2878 				if (snp->sen_type ==
2879 				    phys_enum_type_list[i].pet_type) {
2880 					if (snp->sen_instance > mrange[i])
2881 						mrange[i] =
2882 						    snp->sen_instance;
2883 				}
2884 			}
2885 		}
2886 	}
2887 
2888 	topo_mod_dprintf(mod, "%s: creating nodes from FRU hierarchy tree.",
2889 	    cp->sec_csn);
2890 
2891 	if ((ret = ses_enumerate_node(sdp, pnode, cp, phys_tree, mrange)) !=
2892 	    0) {
2893 		topo_mod_dprintf(mod,
2894 		    "ses_create_children_from_phys_tree() failed: ");
2895 		return (ret);
2896 	}
2897 
2898 	return (0);
2899 }
2900 
2901 /*
2902  * Instantiate a new chassis instance in the topology.
2903  */
2904 static int
2905 ses_create_chassis(ses_enum_data_t *sdp, tnode_t *pnode, ses_enum_chassis_t *cp)
2906 {
2907 	topo_mod_t *mod = sdp->sed_mod;
2908 	nvlist_t *props;
2909 	char *raw_manufacturer, *raw_model, *raw_revision;
2910 	char *manufacturer = NULL, *model = NULL, *product = NULL;
2911 	char *revision = NULL;
2912 	char *serial;
2913 	char **paths;
2914 	size_t prodlen;
2915 	tnode_t *tn;
2916 	nvlist_t *fmri = NULL, *auth = NULL;
2917 	int ret = -1;
2918 	ses_enum_node_t *snp;
2919 	ses_enum_target_t *stp;
2920 	ses_enum_chassis_t *scp;
2921 	int i, err;
2922 	uint64_t sc_count = 0, pindex;
2923 	ses_phys_tree_t	*sproot = NULL;
2924 	hrtime_t start;
2925 	hrtime_t end;
2926 	double duration;
2927 
2928 	/*
2929 	 * Ignore any internal enclosures.
2930 	 */
2931 	if (cp->sec_internal)
2932 		return (0);
2933 
2934 	/*
2935 	 * Check to see if there are any devices presennt in the chassis.  If
2936 	 * not, ignore the chassis alltogether.  This is most useful for
2937 	 * ignoring internal HBAs that present a SES target but don't actually
2938 	 * manage any of the devices.
2939 	 */
2940 	for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
2941 	    snp = topo_list_next(snp)) {
2942 		if (snp->sen_type == SES_ET_DEVICE ||
2943 		    snp->sen_type == SES_ET_ARRAY_DEVICE)
2944 			break;
2945 	}
2946 
2947 	if (snp == NULL)
2948 		return (0);
2949 
2950 	props = ses_node_props(cp->sec_enclosure);
2951 
2952 	/*
2953 	 * We use the following property mappings:
2954 	 *
2955 	 * manufacturer		vendor-id
2956 	 * model		product-id
2957 	 * serial-number	libses-chassis-serial
2958 	 */
2959 	verify(nvlist_lookup_string(props, SES_EN_PROP_VID,
2960 	    &raw_manufacturer) == 0);
2961 	verify(nvlist_lookup_string(props, SES_EN_PROP_PID, &raw_model) == 0);
2962 	verify(nvlist_lookup_string(props, SES_EN_PROP_REV,
2963 	    &raw_revision) == 0);
2964 	verify(nvlist_lookup_string(props, LIBSES_EN_PROP_CSN, &serial) == 0);
2965 
2966 	/*
2967 	 * To construct the authority information, we 'clean' each string by
2968 	 * removing any offensive characters and trimmming whitespace.  For the
2969 	 * 'product-id', we use a concatenation of 'manufacturer-model'.  We
2970 	 * also take the numerical serial number and convert it to a string.
2971 	 */
2972 	if ((manufacturer = topo_mod_clean_str(mod, raw_manufacturer)) ==
2973 	    NULL || (model = topo_mod_clean_str(mod, raw_model)) == NULL ||
2974 	    (revision = topo_mod_clean_str(mod, raw_revision)) == NULL) {
2975 		goto error;
2976 	}
2977 
2978 	prodlen = strlen(manufacturer) + strlen(model) + 2;
2979 	if ((product = topo_mod_alloc(mod, prodlen)) == NULL)
2980 		goto error;
2981 
2982 	(void) snprintf(product, prodlen, "%s-%s", manufacturer, model);
2983 
2984 	/*
2985 	 * Construct the topo node and bind it to our parent.
2986 	 */
2987 	if (topo_mod_nvalloc(mod, &auth, NV_UNIQUE_NAME) != 0)
2988 		goto error;
2989 
2990 	if (nvlist_add_string(auth, FM_FMRI_AUTH_PRODUCT, product) != 0 ||
2991 	    nvlist_add_string(auth, FM_FMRI_AUTH_CHASSIS, serial) != 0) {
2992 		(void) topo_mod_seterrno(mod, EMOD_NVL_INVAL);
2993 		goto error;
2994 	}
2995 
2996 	/*
2997 	 * We pass NULL for the parent FMRI because there is no resource
2998 	 * associated with it.  For the toplevel enclosure, we leave the
2999 	 * serial/part/revision portions empty, which are reserved for
3000 	 * individual components within the chassis.
3001 	 */
3002 	if ((fmri = topo_mod_hcfmri(mod, NULL, FM_HC_SCHEME_VERSION,
3003 	    SES_ENCLOSURE, cp->sec_instance, NULL, auth,
3004 	    model, revision, serial)) == NULL) {
3005 		topo_mod_dprintf(mod, "topo_mod_hcfmri() failed: %s",
3006 		    topo_mod_errmsg(mod));
3007 		goto error;
3008 	}
3009 
3010 	if ((tn = topo_node_bind(mod, pnode, SES_ENCLOSURE,
3011 	    cp->sec_instance, fmri)) == NULL) {
3012 		topo_mod_dprintf(mod, "topo_node_bind() failed: %s",
3013 		    topo_mod_errmsg(mod));
3014 		goto error;
3015 	}
3016 
3017 	if (topo_method_register(mod, tn, ses_enclosure_methods) != 0) {
3018 		topo_mod_dprintf(mod,
3019 		    "topo_method_register() failed: %s",
3020 		    topo_mod_errmsg(mod));
3021 		goto error;
3022 	}
3023 
3024 	if (ses_set_standard_props(mod, NULL, tn, auth,
3025 	    ses_node_id(cp->sec_enclosure), cp->sec_target->set_devpath) != 0)
3026 		goto error;
3027 
3028 	/*
3029 	 * For enclosures, we want to include all possible targets (for upgrade
3030 	 * purposes).
3031 	 */
3032 	for (i = 0, stp = topo_list_next(&cp->sec_targets); stp != NULL;
3033 	    stp = topo_list_next(stp), i++)
3034 		;
3035 
3036 	verify(i != 0);
3037 	paths = alloca(i * sizeof (char *));
3038 
3039 	for (i = 0, stp = topo_list_next(&cp->sec_targets); stp != NULL;
3040 	    stp = topo_list_next(stp), i++)
3041 		paths[i] = stp->set_devpath;
3042 
3043 
3044 	if (topo_prop_set_string_array(tn, TOPO_PGROUP_SES,
3045 	    TOPO_PROP_PATHS, TOPO_PROP_IMMUTABLE, (const char **)paths,
3046 	    i, &err) != 0) {
3047 		topo_mod_dprintf(mod,
3048 		    "failed to create property %s: %s\n",
3049 		    TOPO_PROP_PATHS, topo_strerror(err));
3050 		goto error;
3051 	}
3052 
3053 	if (nvlist_lookup_uint64(props,
3054 	    LIBSES_PROP_PHYS_PARENT, &pindex) == 0) {
3055 		start = gethrtime(); /* to mearusre performance */
3056 		/*
3057 		 * The enclosure is supported through SUNW,FRUID.
3058 		 * Need to enumerate the nodes through hierarchical order.
3059 		 */
3060 		if ((sproot = topo_mod_zalloc(mod,
3061 		    sizeof (ses_phys_tree_t))) == NULL) {
3062 			topo_mod_dprintf(mod,
3063 			    "failed to allocate root: %s\n",
3064 			    topo_strerror(err));
3065 			goto error;
3066 		}
3067 		sproot->spt_pindex = pindex;
3068 		if (nvlist_lookup_boolean_value(props,
3069 		    LIBSES_PROP_FRU, &sproot->spt_isfru) != 0) {
3070 			topo_mod_dprintf(mod,
3071 			    "ses_create_chassis(): Failed to find prop %s "
3072 			    "on enclosure element (CSN %s).",
3073 			    LIBSES_PROP_FRU, cp->sec_csn);
3074 			/* an enclosure should be a FRU. continue to process. */
3075 			sproot->spt_isfru = B_TRUE;
3076 		}
3077 		if (nvlist_lookup_uint64(props,
3078 		    SES_PROP_ELEMENT_ONLY_INDEX,
3079 		    &sproot->spt_eonlyindex) != 0) {
3080 			topo_mod_dprintf(mod,
3081 			    "ses_create_chassis(): Failed to find prop %s "
3082 			    "on enclosure element (CSN %s).",
3083 			    LIBSES_PROP_PHYS_PARENT, cp->sec_csn);
3084 			topo_mod_free(mod, sproot, sizeof (ses_phys_tree_t));
3085 			goto error;
3086 		}
3087 		if (sproot->spt_pindex != sproot->spt_eonlyindex) {
3088 			topo_mod_dprintf(mod, "ses_create_chassis(): "
3089 			    "Enclosure element(CSN %s) should have "
3090 			    "itself as the parent to be the root node "
3091 			    "of FRU hierarchical tree.)", cp->sec_csn);
3092 			topo_mod_free(mod, sproot, sizeof (ses_phys_tree_t));
3093 			goto error;
3094 		} else {
3095 			sproot->spt_snode = cp->sec_enclosure;
3096 			sproot->spt_tnode = tn;
3097 			/* construct a tree. */
3098 			if (ses_construct_phys_tree(sdp, cp, sproot) != 0) {
3099 				topo_mod_dprintf(mod, "ses_create_chassis(): "
3100 				    "Failed to construct FRU hierarchical "
3101 				    "tree on enclosure (CSN %s.)",
3102 				    cp->sec_csn);
3103 			}
3104 
3105 			/* enumerate elements from the tree. */
3106 			if (ses_create_children_from_phys_tree(sdp, tn, cp,
3107 			    sproot) != 0) {
3108 				topo_mod_dprintf(mod, "ses_create_chassis(): "
3109 				    "Failed to create children topo nodes out "
3110 				    "of FRU hierarchical tree on enclosure "
3111 				    "(CSN %s).", cp->sec_csn);
3112 			}
3113 			/* destroy the phys tree. */
3114 			ses_phys_tree_free(mod, sproot);
3115 		}
3116 
3117 		end = gethrtime();
3118 		duration = end - start;
3119 		duration /= HR_SECOND;
3120 		topo_mod_dprintf(mod,
3121 		    "FRU boundary tree based enumeration: %.6f seconds",
3122 		    duration);
3123 	} else {
3124 		/*
3125 		 * Create the nodes for power supplies, fans, controllers and
3126 		 * devices.  Note that SAS exopander nodes and connector nodes
3127 		 * are handled through protocol specific processing of
3128 		 * controllers.
3129 		 */
3130 		if (ses_create_children(sdp, tn, SES_ET_POWER_SUPPLY,
3131 		    PSU, "PSU", cp, B_TRUE) != 0 ||
3132 		    ses_create_children(sdp, tn, SES_ET_COOLING,
3133 		    FAN, "FAN", cp, B_TRUE) != 0 ||
3134 		    ses_create_children(sdp, tn, SES_ET_ESC_ELECTRONICS,
3135 		    CONTROLLER, "CONTROLLER", cp, B_TRUE) != 0 ||
3136 		    ses_create_children(sdp, tn, SES_ET_DEVICE,
3137 		    BAY, "BAY", cp, B_TRUE) != 0 ||
3138 		    ses_create_children(sdp, tn, SES_ET_ARRAY_DEVICE,
3139 		    BAY, "BAY", cp, B_TRUE) != 0)
3140 			goto error;
3141 	}
3142 
3143 	if (cp->sec_maxinstance != SES_INST_NOTSET &&
3144 	    (topo_node_range_create(mod, tn, SUBCHASSIS, 0,
3145 	    cp->sec_maxinstance) != 0)) {
3146 		topo_mod_dprintf(mod, "topo_node_create_range() failed: %s",
3147 		    topo_mod_errmsg(mod));
3148 		goto error;
3149 	}
3150 
3151 	for (scp = topo_list_next(&cp->sec_subchassis); scp != NULL;
3152 	    scp = topo_list_next(scp)) {
3153 
3154 		if (ses_create_subchassis(sdp, tn, scp) != 0)
3155 			goto error;
3156 
3157 		topo_mod_dprintf(mod, "created Subchassis node with "
3158 		    "instance %u\nand target (%s) under Chassis with CSN %s",
3159 		    scp->sec_instance, scp->sec_target->set_devpath,
3160 		    cp->sec_csn);
3161 
3162 		sc_count++;
3163 	}
3164 
3165 	topo_mod_dprintf(mod, "%s: created %llu %s nodes",
3166 	    cp->sec_csn, sc_count, SUBCHASSIS);
3167 
3168 	cp->sec_target->set_refcount++;
3169 	topo_node_setspecific(tn, cp->sec_target);
3170 
3171 	ret = 0;
3172 error:
3173 	topo_mod_strfree(mod, manufacturer);
3174 	topo_mod_strfree(mod, model);
3175 	topo_mod_strfree(mod, revision);
3176 	topo_mod_strfree(mod, product);
3177 
3178 	nvlist_free(fmri);
3179 	nvlist_free(auth);
3180 	return (ret);
3181 }
3182 
3183 /*
3184  * Create a bay node explicitly enumerated via XML.
3185  */
3186 static int
3187 ses_create_bays(ses_enum_data_t *sdp, tnode_t *pnode)
3188 {
3189 	topo_mod_t *mod = sdp->sed_mod;
3190 	ses_enum_chassis_t *cp;
3191 
3192 	/*
3193 	 * Iterate over chassis looking for an internal enclosure.  This
3194 	 * property is set via a vendor-specific plugin, and there should only
3195 	 * ever be a single internal chassis in a system.
3196 	 */
3197 	for (cp = topo_list_next(&sdp->sed_chassis); cp != NULL;
3198 	    cp = topo_list_next(cp)) {
3199 		if (cp->sec_internal)
3200 			break;
3201 	}
3202 
3203 	if (cp == NULL) {
3204 		topo_mod_dprintf(mod, "failed to find internal chassis\n");
3205 		return (-1);
3206 	}
3207 
3208 	if (ses_create_children(sdp, pnode, SES_ET_DEVICE,
3209 	    BAY, "BAY", cp, B_FALSE) != 0 ||
3210 	    ses_create_children(sdp, pnode, SES_ET_ARRAY_DEVICE,
3211 	    BAY, "BAY", cp, B_FALSE) != 0)
3212 		return (-1);
3213 
3214 	return (0);
3215 }
3216 
3217 /*
3218  * Initialize chassis or subchassis.
3219  */
3220 static int
3221 ses_init_chassis(topo_mod_t *mod, ses_enum_data_t *sdp, ses_enum_chassis_t *pcp,
3222     ses_enum_chassis_t *cp, ses_node_t *np, nvlist_t *props,
3223     uint64_t subchassis, ses_chassis_type_e flags)
3224 {
3225 	boolean_t internal, ident;
3226 
3227 	assert((flags & (SES_NEW_CHASSIS | SES_NEW_SUBCHASSIS |
3228 	    SES_DUP_CHASSIS | SES_DUP_SUBCHASSIS)) != 0);
3229 
3230 	assert(cp != NULL);
3231 	assert(np != NULL);
3232 	assert(props != NULL);
3233 
3234 	if (flags & (SES_NEW_SUBCHASSIS | SES_DUP_SUBCHASSIS))
3235 		assert(pcp != NULL);
3236 
3237 	topo_mod_dprintf(mod, "ses_init_chassis: %s: index %llu, flags (%d)",
3238 	    sdp->sed_name, subchassis, flags);
3239 
3240 	if (flags & (SES_NEW_CHASSIS | SES_NEW_SUBCHASSIS)) {
3241 
3242 		topo_mod_dprintf(mod, "new chassis/subchassis");
3243 		if (nvlist_lookup_boolean_value(props,
3244 		    LIBSES_EN_PROP_INTERNAL, &internal) == 0)
3245 			cp->sec_internal = internal;
3246 
3247 		cp->sec_enclosure = np;
3248 		cp->sec_target = sdp->sed_target;
3249 
3250 		if (flags & SES_NEW_CHASSIS) {
3251 			if (!cp->sec_internal)
3252 				cp->sec_instance = sdp->sed_instance++;
3253 			topo_list_append(&sdp->sed_chassis, cp);
3254 		} else {
3255 			if (subchassis != NO_SUBCHASSIS)
3256 				cp->sec_instance = subchassis;
3257 			else
3258 				cp->sec_instance = pcp->sec_scinstance++;
3259 
3260 			if (cp->sec_instance > pcp->sec_maxinstance)
3261 				pcp->sec_maxinstance = cp->sec_instance;
3262 
3263 			topo_list_append(&pcp->sec_subchassis, cp);
3264 		}
3265 
3266 	} else {
3267 		topo_mod_dprintf(mod, "dup chassis/subchassis");
3268 		if (nvlist_lookup_boolean_value(props,
3269 		    SES_PROP_IDENT, &ident) == 0) {
3270 			topo_mod_dprintf(mod,  "overriding enclosure node");
3271 
3272 			cp->sec_enclosure = np;
3273 			cp->sec_target = sdp->sed_target;
3274 		}
3275 	}
3276 
3277 	topo_list_append(&cp->sec_targets, sdp->sed_target);
3278 	sdp->sed_current = cp;
3279 
3280 	return (0);
3281 }
3282 
3283 /*
3284  * Gather nodes from the current SES target into our chassis list, merging the
3285  * results if necessary.
3286  */
3287 static ses_walk_action_t
3288 ses_enum_gather(ses_node_t *np, void *data)
3289 {
3290 	nvlist_t *props = ses_node_props(np);
3291 	ses_enum_data_t *sdp = data;
3292 	topo_mod_t *mod = sdp->sed_mod;
3293 	ses_enum_chassis_t *cp, *scp;
3294 	ses_enum_node_t *snp;
3295 	ses_alt_node_t *sap;
3296 	char *csn;
3297 	uint64_t instance, type;
3298 	uint64_t prevstatus, status;
3299 	boolean_t report;
3300 	uint64_t subchassis = NO_SUBCHASSIS;
3301 
3302 	if (ses_node_type(np) == SES_NODE_ENCLOSURE) {
3303 		/*
3304 		 * If we have already identified the chassis for this target,
3305 		 * then this is a secondary enclosure and we should ignore it,
3306 		 * along with the rest of the tree (since this is depth-first).
3307 		 */
3308 		if (sdp->sed_current != NULL)
3309 			return (SES_WALK_ACTION_TERMINATE);
3310 
3311 		/*
3312 		 * Go through the list of chassis we have seen so far and see
3313 		 * if this serial number matches one of the known values.
3314 		 * If so, check whether this enclosure is a subchassis.
3315 		 */
3316 		if (nvlist_lookup_string(props, LIBSES_EN_PROP_CSN,
3317 		    &csn) != 0)
3318 			return (SES_WALK_ACTION_TERMINATE);
3319 
3320 		(void) nvlist_lookup_uint64(props, LIBSES_EN_PROP_SUBCHASSIS_ID,
3321 		    &subchassis);
3322 
3323 		topo_mod_dprintf(mod, "ses_enum_gather: Enclosure Node (%s) "
3324 		    "CSN (%s), subchassis (%llu)", sdp->sed_name, csn,
3325 		    subchassis);
3326 
3327 		/*
3328 		 * We need to determine whether this enclosure node
3329 		 * represents a chassis or a subchassis. Since we may
3330 		 * receive the enclosure nodes in a non-deterministic
3331 		 * manner, we need to account for all possible combinations:
3332 		 *	1. Chassis for the current CSN has not yet been
3333 		 *	   allocated
3334 		 *		1.1 This is a new chassis:
3335 		 *			allocate and instantiate the chassis
3336 		 *		1.2 This is a new subchassis:
3337 		 *			allocate a placeholder chassis
3338 		 *			allocate and instantiate the subchassis
3339 		 *			link the subchassis to the chassis
3340 		 *	2. Chassis for the current CSN has been allocated
3341 		 *		2.1 This is a duplicate chassis enclosure
3342 		 *			check whether to override old chassis
3343 		 *			append to chassis' target list
3344 		 *		2.2 Only placeholder chassis exists
3345 		 *			fill in the chassis fields
3346 		 *		2.3 This is a new subchassis
3347 		 *			allocate and instantiate the subchassis
3348 		 *			link the subchassis to the chassis
3349 		 *		2.4 This is a duplicate subchassis enclosure
3350 		 *			 check whether to override old chassis
3351 		 *			 append to chassis' target list
3352 		 */
3353 
3354 		for (cp = topo_list_next(&sdp->sed_chassis); cp != NULL;
3355 		    cp = topo_list_next(cp))
3356 			if (strcmp(cp->sec_csn, csn) == 0)
3357 				break;
3358 
3359 		if (cp == NULL) {
3360 			/* 1. Haven't seen a chassis with this CSN before */
3361 
3362 			if ((cp = topo_mod_zalloc(mod,
3363 			    sizeof (ses_enum_chassis_t))) == NULL)
3364 				goto error;
3365 
3366 			cp->sec_scinstance = SES_STARTING_SUBCHASSIS;
3367 			cp->sec_maxinstance = SES_INST_NOTSET;
3368 			cp->sec_csn = csn;
3369 
3370 			if (subchassis == NO_SUBCHASSIS) {
3371 				/* 1.1 This is a new chassis */
3372 
3373 				topo_mod_dprintf(mod, "%s: Initialize new "
3374 				    "chassis with CSN %s", sdp->sed_name, csn);
3375 
3376 				if (ses_init_chassis(mod, sdp, NULL, cp,
3377 				    np, props, NO_SUBCHASSIS,
3378 				    SES_NEW_CHASSIS) < 0)
3379 					goto error;
3380 			} else {
3381 				/* 1.2 This is a new subchassis */
3382 
3383 				topo_mod_dprintf(mod, "%s: Initialize new "
3384 				    "subchassis with CSN %s and index %llu",
3385 				    sdp->sed_name, csn, subchassis);
3386 
3387 				if ((scp = topo_mod_zalloc(mod,
3388 				    sizeof (ses_enum_chassis_t))) == NULL)
3389 					goto error;
3390 
3391 				scp->sec_csn = csn;
3392 
3393 				if (ses_init_chassis(mod, sdp, cp, scp, np,
3394 				    props, subchassis, SES_NEW_SUBCHASSIS) < 0)
3395 					goto error;
3396 			}
3397 		} else {
3398 			/*
3399 			 * We have a chassis or subchassis with this CSN.  If
3400 			 * it's a chassis, we must check to see whether it is
3401 			 * a placeholder previously created because we found a
3402 			 * subchassis with this CSN.  We will know that because
3403 			 * the sec_target value will not be set; it is set only
3404 			 * in ses_init_chassis().  In that case, initialise it
3405 			 * as a new chassis; otherwise, it's a duplicate and we
3406 			 * need to append only.
3407 			 */
3408 			if (subchassis == NO_SUBCHASSIS) {
3409 				if (cp->sec_target != NULL) {
3410 					/* 2.1 This is a duplicate chassis */
3411 
3412 					topo_mod_dprintf(mod, "%s: Append "
3413 					    "duplicate chassis with CSN (%s)",
3414 					    sdp->sed_name, csn);
3415 
3416 					if (ses_init_chassis(mod, sdp, NULL, cp,
3417 					    np, props, NO_SUBCHASSIS,
3418 					    SES_DUP_CHASSIS) < 0)
3419 						goto error;
3420 				} else {
3421 					/* Placeholder chassis - init it up */
3422 					topo_mod_dprintf(mod, "%s: Initialize"
3423 					    "placeholder chassis with CSN %s",
3424 					    sdp->sed_name, csn);
3425 
3426 					if (ses_init_chassis(mod, sdp, NULL,
3427 					    cp, np, props, NO_SUBCHASSIS,
3428 					    SES_NEW_CHASSIS) < 0)
3429 						goto error;