xref: /illumos-gate/usr/src/cmd/fm/modules/sun4v/etm/etm.c (revision bbf21555)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * etm.c	FMA Event Transport Module implementation, a plugin of FMD
29  *		for sun4v/Ontario
30  *
31  * plugin for sending/receiving FMA events to/from service processor
32  */
33 
34 /*
35  * --------------------------------- includes --------------------------------
36  */
37 
38 #include <sys/fm/protocol.h>
39 #include <sys/fm/util.h>
40 #include <sys/fm/ldom.h>
41 #include <sys/strlog.h>
42 #include <sys/syslog.h>
43 #include <sys/libds.h>
44 #include <netinet/in.h>
45 #include <fm/fmd_api.h>
46 
47 #include "etm_xport_api.h"
48 #include "etm_etm_proto.h"
49 #include "etm_impl.h"
50 #include "etm_iosvc.h"
51 #include "etm_filter.h"
52 #include "etm_ckpt.h"
53 
54 #include <pthread.h>
55 #include <signal.h>
56 #include <stropts.h>
57 #include <locale.h>
58 #include <strings.h>
59 #include <stdlib.h>
60 #include <unistd.h>
61 #include <limits.h>
62 #include <values.h>
63 #include <alloca.h>
64 #include <errno.h>
65 #include <dlfcn.h>
66 #include <link.h>
67 #include <fcntl.h>
68 #include <time.h>
69 
70 /*
71  * ----------------------------- forward decls -------------------------------
72  */
73 
74 static void
75 etm_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class);
76 
77 static int
78 etm_send(fmd_hdl_t *hdl, fmd_xprt_t *xp, fmd_event_t *event, nvlist_t *nvl);
79 
80 static void
81 etm_send_to_remote_root(void *arg);
82 
83 static void
84 etm_recv_from_remote_root(void *arg);
85 
86 static void
87 etm_ckpt_remove(fmd_hdl_t *hdl, etm_iosvc_q_ele_t *ele);
88 
89 /*
90  * ------------------------- data structs for FMD ----------------------------
91  */
92 
93 static const fmd_hdl_ops_t fmd_ops = {
94 	etm_recv,	/* fmdo_recv */
95 	NULL,		/* fmdo_timeout */
96 	NULL,		/* fmdo_close */
97 	NULL,		/* fmdo_stats */
98 	NULL,		/* fmdo_gc */
99 	etm_send,	/* fmdo_send */
100 };
101 
102 static const fmd_prop_t fmd_props[] = {
103 	{ ETM_PROP_NM_XPORT_ADDRS,		FMD_TYPE_STRING, "" },
104 	{ ETM_PROP_NM_DEBUG_LVL,		FMD_TYPE_INT32, "0" },
105 	{ ETM_PROP_NM_DEBUG_MAX_EV_CNT,		FMD_TYPE_INT32, "-1" },
106 	{ ETM_PROP_NM_CONSOLE,			FMD_TYPE_BOOL, "false" },
107 	{ ETM_PROP_NM_SYSLOGD,			FMD_TYPE_BOOL, "true" },
108 	{ ETM_PROP_NM_FACILITY,			FMD_TYPE_STRING, "LOG_DAEMON" },
109 	{ ETM_PROP_NM_MAX_RESP_Q_LEN,		FMD_TYPE_UINT32, "32" },
110 	{ ETM_PROP_NM_BAD_ACC_TO_SEC,		FMD_TYPE_UINT32, "1" },
111 	{ ETM_PROP_NM_FMA_RESP_WAIT_TIME,	FMD_TYPE_INT32, "240" },
112 	{ NULL, 0, NULL }
113 };
114 
115 
116 static const fmd_hdl_info_t fmd_info = {
117 	"FMA Event Transport Module", "1.2", &fmd_ops, fmd_props
118 };
119 
120 /*
121  * ----------------------- private consts and defns --------------------------
122  */
123 
124 /* misc buffer for variable sized protocol header fields */
125 
126 #define	ETM_MISC_BUF_SZ	(4 * 1024)
127 
128 static uint32_t
129 etm_ldom_type = LDOM_TYPE_LEGACY;
130 
131 /* try limit for IO operations w/ capped exp backoff sleep on retry */
132 
133 /*
134  * Design_Note:	ETM will potentially retry forever IO operations that the
135  *		transport fails with EAGAIN (aka EWOULDBLOCK) rather than
136  *		giving up after some number of seconds. This avoids
137  *		dropping FMA events while the service processor is down,
138  *		but at the risk of pending fmdo_recv() forever and
139  *		overflowing FMD's event queue for ETM.
140  *		A future TBD enhancement would be to always recv
141  *		and send each ETM msg in a single read/write() to reduce
142  *		the risk of failure between ETM msg hdr and body,
143  *		assuming the MTU_SZ is large enough.
144  */
145 
146 #define	ETM_TRY_MAX_CNT		(MAXINT - 1)
147 #define	ETM_TRY_BACKOFF_RATE	(4)
148 #define	ETM_TRY_BACKOFF_CAP	(60)
149 
150 /* amount to increment protocol transaction id on each new send */
151 
152 #define	ETM_XID_INC		(2)
153 
154 typedef struct etm_resp_q_ele {
155 
156 	etm_xport_conn_t	rqe_conn;	/* open connection to send on */
157 	etm_proto_v1_pp_t	*rqe_hdrp;	/* ptr to ETM msg hdr */
158 	size_t			rqe_hdr_sz;	/* sizeof ETM msg hdr */
159 	int32_t			rqe_resp_code;	/* response code to send */
160 
161 	struct etm_resp_q_ele	*rqe_nextp;	/* PRIVATE - next ele ptr */
162 
163 } etm_resp_q_ele_t;	/* responder queue element */
164 
165 /*
166  * ---------------------------- global data ----------------------------------
167  */
168 
169 static fmd_hdl_t
170 *init_hdl = NULL;	/* used in mem allocator and several other places */
171 
172 static int
173 etm_debug_lvl = 0;	/* debug level: 0 is off, 1 is on, 2 is more, etc */
174 
175 static int
176 etm_debug_max_ev_cnt = -1; /* max allowed event count for debugging */
177 
178 static fmd_xprt_t
179 *etm_fmd_xprt = NULL;	/* FMD transport layer handle */
180 
181 static pthread_t
182 etm_svr_tid = 0;	/* thread id of connection acceptance server */
183 
184 static pthread_t
185 etm_resp_tid = 0;	/* thread id of msg responder */
186 
187 static etm_resp_q_ele_t
188 *etm_resp_q_head = NULL; /* ptr to cur head of responder queue */
189 
190 static etm_resp_q_ele_t
191 *etm_resp_q_tail = NULL; /* ptr to cur tail of responder queue */
192 
193 static uint32_t
194 etm_resp_q_cur_len = 0;	/* cur length (ele cnt) of responder queue */
195 
196 static uint32_t
197 etm_resp_q_max_len = 0;	/* max length (ele cnt) of responder queue */
198 
199 static uint32_t
200 etm_bad_acc_to_sec = 0;	/* sleep timeout (in sec) after bad conn accept */
201 
202 static pthread_mutex_t
203 etm_resp_q_lock = PTHREAD_MUTEX_INITIALIZER;	/* protects responder queue */
204 
205 static pthread_cond_t
206 etm_resp_q_cv = PTHREAD_COND_INITIALIZER;	/* nudges msg responder */
207 
208 static volatile int
209 etm_is_dying = 0;	/* bool for dying (killing self) */
210 
211 static uint32_t
212 etm_xid_cur = 0;	/* current transaction id for sends */
213 
214 static uint32_t
215 etm_xid_ping = 0;	/* xid of last CONTROL msg sent requesting ping */
216 
217 static uint32_t
218 etm_xid_ver_negot = 0;	/* xid of last CONTROL msg sent requesting ver negot */
219 
220 static uint32_t
221 etm_xid_posted_logged_ev = 0;
222 			/* xid of last FMA_EVENT msg/event posted OK to FMD */
223 
224 static uint32_t
225 etm_xid_posted_sa = 0;	/* xid of last ALERT msg/event posted OK to syslog */
226 
227 static uint8_t
228 etm_resp_ver = ETM_PROTO_V1; /* proto ver [negotiated] for msg sends */
229 
230 static uint32_t
231 etm_fma_resp_wait_time = 30;	/*  time (sec) wait for fma event resp */
232 
233 static pthread_mutex_t
234 etm_write_lock = PTHREAD_MUTEX_INITIALIZER;	/* for write operations */
235 
236 static log_ctl_t syslog_ctl;	/* log(4D) meta-data for each msg */
237 static int syslog_facility;	/* log(4D) facility (part of priority) */
238 static int syslog_logfd = -1;	/* log(4D) file descriptor */
239 static int syslog_msgfd = -1;	/* sysmsg(4D) file descriptor */
240 static int syslog_file = 0;	/* log to syslog_logfd */
241 static int syslog_cons = 0;	/* log to syslog_msgfd */
242 
243 static const struct facility {
244 	const char *fac_name;
245 	int fac_value;
246 } syslog_facs[] = {
247 	{ "LOG_DAEMON", LOG_DAEMON },
248 	{ "LOG_LOCAL0", LOG_LOCAL0 },
249 	{ "LOG_LOCAL1", LOG_LOCAL1 },
250 	{ "LOG_LOCAL2", LOG_LOCAL2 },
251 	{ "LOG_LOCAL3", LOG_LOCAL3 },
252 	{ "LOG_LOCAL4", LOG_LOCAL4 },
253 	{ "LOG_LOCAL5", LOG_LOCAL5 },
254 	{ "LOG_LOCAL6", LOG_LOCAL6 },
255 	{ "LOG_LOCAL7", LOG_LOCAL7 },
256 	{ NULL, 0 }
257 };
258 
259 static struct stats {
260 
261 	/* ETM msg counters */
262 
263 	fmd_stat_t etm_rd_hdr_fmaevent;
264 	fmd_stat_t etm_rd_hdr_control;
265 	fmd_stat_t etm_rd_hdr_alert;
266 	fmd_stat_t etm_rd_hdr_response;
267 	fmd_stat_t etm_rd_body_fmaevent;
268 	fmd_stat_t etm_rd_body_control;
269 	fmd_stat_t etm_rd_body_alert;
270 	fmd_stat_t etm_rd_body_response;
271 	fmd_stat_t etm_wr_hdr_fmaevent;
272 	fmd_stat_t etm_wr_hdr_control;
273 	fmd_stat_t etm_wr_hdr_response;
274 	fmd_stat_t etm_wr_body_fmaevent;
275 	fmd_stat_t etm_wr_body_control;
276 	fmd_stat_t etm_wr_body_response;
277 
278 	fmd_stat_t etm_rd_max_ev_per_msg;
279 	fmd_stat_t etm_wr_max_ev_per_msg;
280 
281 	fmd_stat_t etm_resp_q_cur_len;
282 	fmd_stat_t etm_resp_q_max_len;
283 
284 	/* ETM byte counters */
285 
286 	fmd_stat_t etm_wr_fmd_bytes;
287 	fmd_stat_t etm_rd_fmd_bytes;
288 	fmd_stat_t etm_wr_xport_bytes;
289 	fmd_stat_t etm_rd_xport_bytes;
290 
291 	fmd_stat_t etm_magic_drop_bytes;
292 
293 	/* ETM [dropped] FMA event counters */
294 
295 	fmd_stat_t etm_rd_fmd_fmaevent;
296 	fmd_stat_t etm_wr_fmd_fmaevent;
297 
298 	fmd_stat_t etm_rd_drop_fmaevent;
299 	fmd_stat_t etm_wr_drop_fmaevent;
300 
301 	fmd_stat_t etm_rd_dup_fmaevent;
302 	fmd_stat_t etm_wr_dup_fmaevent;
303 
304 	fmd_stat_t etm_rd_dup_alert;
305 	fmd_stat_t etm_wr_dup_alert;
306 
307 	fmd_stat_t etm_enq_drop_resp_q;
308 	fmd_stat_t etm_deq_drop_resp_q;
309 
310 	/* ETM protocol failures */
311 
312 	fmd_stat_t etm_magic_bad;
313 	fmd_stat_t etm_ver_bad;
314 	fmd_stat_t etm_msgtype_bad;
315 	fmd_stat_t etm_subtype_bad;
316 	fmd_stat_t etm_xid_bad;
317 	fmd_stat_t etm_fmaeventlen_bad;
318 	fmd_stat_t etm_respcode_bad;
319 	fmd_stat_t etm_timeout_bad;
320 	fmd_stat_t etm_evlens_bad;
321 
322 	/* IO operation failures */
323 
324 	fmd_stat_t etm_xport_wr_fail;
325 	fmd_stat_t etm_xport_rd_fail;
326 	fmd_stat_t etm_xport_pk_fail;
327 
328 	/* IO operation retries */
329 
330 	fmd_stat_t etm_xport_wr_retry;
331 	fmd_stat_t etm_xport_rd_retry;
332 	fmd_stat_t etm_xport_pk_retry;
333 
334 	/* system and library failures */
335 
336 	fmd_stat_t etm_os_nvlist_pack_fail;
337 	fmd_stat_t etm_os_nvlist_unpack_fail;
338 	fmd_stat_t etm_os_nvlist_size_fail;
339 	fmd_stat_t etm_os_pthread_create_fail;
340 
341 	/* xport API failures */
342 
343 	fmd_stat_t etm_xport_get_ev_addrv_fail;
344 	fmd_stat_t etm_xport_open_fail;
345 	fmd_stat_t etm_xport_close_fail;
346 	fmd_stat_t etm_xport_accept_fail;
347 	fmd_stat_t etm_xport_open_retry;
348 
349 	/* FMD entry point bad arguments */
350 
351 	fmd_stat_t etm_fmd_init_badargs;
352 	fmd_stat_t etm_fmd_fini_badargs;
353 
354 	/* Alert logging errors */
355 
356 	fmd_stat_t etm_log_err;
357 	fmd_stat_t etm_msg_err;
358 
359 	/* miscellaneous stats */
360 
361 	fmd_stat_t etm_reset_xport;
362 
363 } etm_stats = {
364 
365 	/* ETM msg counters */
366 
367 	{ "etm_rd_hdr_fmaevent", FMD_TYPE_UINT64,
368 		"ETM fmaevent msg headers rcvd from xport" },
369 	{ "etm_rd_hdr_control", FMD_TYPE_UINT64,
370 		"ETM control msg headers rcvd from xport" },
371 	{ "etm_rd_hdr_alert", FMD_TYPE_UINT64,
372 		"ETM alert msg headers rcvd from xport" },
373 	{ "etm_rd_hdr_response", FMD_TYPE_UINT64,
374 		"ETM response msg headers rcvd from xport" },
375 	{ "etm_rd_body_fmaevent", FMD_TYPE_UINT64,
376 		"ETM fmaevent msg bodies rcvd from xport" },
377 	{ "etm_rd_body_control", FMD_TYPE_UINT64,
378 		"ETM control msg bodies rcvd from xport" },
379 	{ "etm_rd_body_alert", FMD_TYPE_UINT64,
380 		"ETM alert msg bodies rcvd from xport" },
381 	{ "etm_rd_body_response", FMD_TYPE_UINT64,
382 		"ETM response msg bodies rcvd from xport" },
383 	{ "etm_wr_hdr_fmaevent", FMD_TYPE_UINT64,
384 		"ETM fmaevent msg headers sent to xport" },
385 	{ "etm_wr_hdr_control", FMD_TYPE_UINT64,
386 		"ETM control msg headers sent to xport" },
387 	{ "etm_wr_hdr_response", FMD_TYPE_UINT64,
388 		"ETM response msg headers sent to xport" },
389 	{ "etm_wr_body_fmaevent", FMD_TYPE_UINT64,
390 		"ETM fmaevent msg bodies sent to xport" },
391 	{ "etm_wr_body_control", FMD_TYPE_UINT64,
392 		"ETM control msg bodies sent to xport" },
393 	{ "etm_wr_body_response", FMD_TYPE_UINT64,
394 		"ETM response msg bodies sent to xport" },
395 
396 	{ "etm_rd_max_ev_per_msg", FMD_TYPE_UINT64,
397 		"max FMA events per ETM msg from xport" },
398 	{ "etm_wr_max_ev_per_msg", FMD_TYPE_UINT64,
399 		"max FMA events per ETM msg to xport" },
400 
401 	{ "etm_resp_q_cur_len", FMD_TYPE_UINT64,
402 		"cur enqueued response msgs to xport" },
403 	{ "etm_resp_q_max_len", FMD_TYPE_UINT64,
404 		"max enqueable response msgs to xport" },
405 
406 	/* ETM byte counters */
407 
408 	{ "etm_wr_fmd_bytes", FMD_TYPE_UINT64,
409 		"bytes of FMA events sent to FMD" },
410 	{ "etm_rd_fmd_bytes", FMD_TYPE_UINT64,
411 		"bytes of FMA events rcvd from FMD" },
412 	{ "etm_wr_xport_bytes", FMD_TYPE_UINT64,
413 		"bytes of FMA events sent to xport" },
414 	{ "etm_rd_xport_bytes", FMD_TYPE_UINT64,
415 		"bytes of FMA events rcvd from xport" },
416 
417 	{ "etm_magic_drop_bytes", FMD_TYPE_UINT64,
418 		"bytes dropped from xport pre magic num" },
419 
420 	/* ETM [dropped] FMA event counters */
421 
422 	{ "etm_rd_fmd_fmaevent", FMD_TYPE_UINT64,
423 		"FMA events rcvd from FMD" },
424 	{ "etm_wr_fmd_fmaevent", FMD_TYPE_UINT64,
425 		"FMA events sent to FMD" },
426 
427 	{ "etm_rd_drop_fmaevent", FMD_TYPE_UINT64,
428 		"dropped FMA events from xport" },
429 	{ "etm_wr_drop_fmaevent", FMD_TYPE_UINT64,
430 		"dropped FMA events to xport" },
431 
432 	{ "etm_rd_dup_fmaevent", FMD_TYPE_UINT64,
433 	    "duplicate FMA events rcvd from xport" },
434 	{ "etm_wr_dup_fmaevent", FMD_TYPE_UINT64,
435 	    "duplicate FMA events sent to xport" },
436 
437 	{ "etm_rd_dup_alert", FMD_TYPE_UINT64,
438 	    "duplicate ALERTs rcvd from xport" },
439 	{ "etm_wr_dup_alert", FMD_TYPE_UINT64,
440 	    "duplicate ALERTs sent to xport" },
441 
442 	{ "etm_enq_drop_resp_q", FMD_TYPE_UINT64,
443 	    "dropped response msgs on enq" },
444 	{ "etm_deq_drop_resp_q", FMD_TYPE_UINT64,
445 	    "dropped response msgs on deq" },
446 
447 	/* ETM protocol failures */
448 
449 	{ "etm_magic_bad", FMD_TYPE_UINT64,
450 		"ETM msgs w/ invalid magic num" },
451 	{ "etm_ver_bad", FMD_TYPE_UINT64,
452 		"ETM msgs w/ invalid protocol version" },
453 	{ "etm_msgtype_bad", FMD_TYPE_UINT64,
454 		"ETM msgs w/ invalid message type" },
455 	{ "etm_subtype_bad", FMD_TYPE_UINT64,
456 		"ETM msgs w/ invalid sub type" },
457 	{ "etm_xid_bad", FMD_TYPE_UINT64,
458 		"ETM msgs w/ unmatched xid" },
459 	{ "etm_fmaeventlen_bad", FMD_TYPE_UINT64,
460 		"ETM msgs w/ invalid FMA event length" },
461 	{ "etm_respcode_bad", FMD_TYPE_UINT64,
462 		"ETM msgs w/ invalid response code" },
463 	{ "etm_timeout_bad", FMD_TYPE_UINT64,
464 		"ETM msgs w/ invalid timeout value" },
465 	{ "etm_evlens_bad", FMD_TYPE_UINT64,
466 		"ETM msgs w/ too many event lengths" },
467 
468 	/* IO operation failures */
469 
470 	{ "etm_xport_wr_fail", FMD_TYPE_UINT64,
471 		"xport write failures" },
472 	{ "etm_xport_rd_fail", FMD_TYPE_UINT64,
473 		"xport read failures" },
474 	{ "etm_xport_pk_fail", FMD_TYPE_UINT64,
475 		"xport peek failures" },
476 
477 	/* IO operation retries */
478 
479 	{ "etm_xport_wr_retry", FMD_TYPE_UINT64,
480 		"xport write retries" },
481 	{ "etm_xport_rd_retry", FMD_TYPE_UINT64,
482 		"xport read retries" },
483 	{ "etm_xport_pk_retry", FMD_TYPE_UINT64,
484 		"xport peek retries" },
485 
486 	/* system and library failures */
487 
488 	{ "etm_os_nvlist_pack_fail", FMD_TYPE_UINT64,
489 		"nvlist_pack failures" },
490 	{ "etm_os_nvlist_unpack_fail", FMD_TYPE_UINT64,
491 		"nvlist_unpack failures" },
492 	{ "etm_os_nvlist_size_fail", FMD_TYPE_UINT64,
493 		"nvlist_size failures" },
494 	{ "etm_os_pthread_create_fail", FMD_TYPE_UINT64,
495 		"pthread_create failures" },
496 
497 	/* transport API failures */
498 
499 	{ "etm_xport_get_ev_addrv_fail", FMD_TYPE_UINT64,
500 		"xport get event addrv API failures" },
501 	{ "etm_xport_open_fail", FMD_TYPE_UINT64,
502 		"xport open API failures" },
503 	{ "etm_xport_close_fail", FMD_TYPE_UINT64,
504 		"xport close API failures" },
505 	{ "etm_xport_accept_fail", FMD_TYPE_UINT64,
506 		"xport accept API failures" },
507 	{ "etm_xport_open_retry", FMD_TYPE_UINT64,
508 		"xport open API retries" },
509 
510 	/* FMD entry point bad arguments */
511 
512 	{ "etm_fmd_init_badargs", FMD_TYPE_UINT64,
513 	    "bad arguments from fmd_init entry point" },
514 	{ "etm_fmd_fini_badargs", FMD_TYPE_UINT64,
515 	    "bad arguments from fmd_fini entry point" },
516 
517 	/* Alert logging errors */
518 
519 	{ "etm_log_err", FMD_TYPE_UINT64,
520 		"failed to log message to log(4D)" },
521 	{ "etm_msg_err", FMD_TYPE_UINT64,
522 		"failed to log message to sysmsg(4D)" },
523 
524 	/* miscellaneous stats */
525 
526 	{ "etm_reset_xport", FMD_TYPE_UINT64,
527 		"xport resets after xport API failure" }
528 };
529 
530 
531 /*
532  * -------------------- global data for Root ldom-------------------------
533  */
534 
535 ldom_hdl_t
536 *etm_lhp = NULL;		/* ldom pointer */
537 
538 static void *etm_dl_hdl = (void *)NULL;
539 static const char *etm_dl_path = "libds.so.1";
540 static int etm_dl_mode = (RTLD_NOW | RTLD_LOCAL);
541 
542 static int(*etm_ds_svc_reg)(ds_capability_t *cap, ds_ops_t *ops) =
543 	(int (*)(ds_capability_t *cap, ds_ops_t *ops))NULL;
544 static int(*etm_ds_clnt_reg)(ds_capability_t *cap, ds_ops_t *ops) =
545 	(int (*)(ds_capability_t *cap, ds_ops_t *ops))NULL;
546 static int(*etm_ds_send_msg)(ds_hdl_t hdl, void *buf, size_t buflen) =
547 	(int (*)(ds_hdl_t hdl, void *buf, size_t buflen))NULL;
548 static int(*etm_ds_recv_msg)(ds_hdl_t hdl, void *buf, size_t buflen,
549     size_t *msglen) =
550 	(int (*)(ds_hdl_t hdl, void *buf, size_t buflen, size_t *msglen))NULL;
551 static int (*etm_ds_fini)(void) = (int (*)(void))NULL;
552 
553 static pthread_mutex_t
554 iosvc_list_lock =  PTHREAD_MUTEX_INITIALIZER;
555 
556 static pthread_t
557 etm_async_e_tid = 0;	/* thread id of io svc async event handler */
558 
559 static etm_proto_v1_ev_hdr_t iosvc_hdr = {
560 	ETM_PROTO_MAGIC_NUM,	/* magic number */
561 	ETM_PROTO_V1,		/* default to V1, not checked */
562 	ETM_MSG_TYPE_FMA_EVENT,	/* Root Domain inteoduces only FMA events */
563 	0,			/* sub-type */
564 	0,			/* pad */
565 	0,			/* add the xid at the Q send time */
566 	ETM_PROTO_V1_TIMEOUT_NONE,
567 	0			/* ev_lens, 0-termed, after 1 FMA event */
568 };
569 
570 /*
571  * static iosvc_list
572  */
573 static etm_iosvc_t iosvc_list[NUM_OF_ROOT_DOMAINS] = {
574 	{"", 0}, {"", 0}, {"", 0}, {"", 0}, {"", 0}, {"", 0},
575 	{"", 0}, {"", 0}
576 };
577 
578 static etm_iosvc_t io_svc = {
579 	"\0",				/* ldom_name */
580 	PTHREAD_COND_INITIALIZER,	/* nudges */
581 	PTHREAD_MUTEX_INITIALIZER,	/* protects the iosvc msg Q */
582 	NULL,				/* iosvc msg Q head */
583 	NULL,				/* iosvc msg Q tail */
584 	0,				/* msg Q current length */
585 	100,				/* msg Q max length */
586 	0,				/* current transaction id */
587 	0,				/* xid of last event posted to FMD */
588 	DS_INVALID_HDL,			/* DS handle */
589 	NULL,				/* fmd xprt handle */
590 	0,				/* tid 4 send to remote RootDomain */
591 	0,				/* tid 4 recv from remote RootDomain */
592 	PTHREAD_COND_INITIALIZER,	/* nudges etm_send_to_remote_root */
593 	PTHREAD_MUTEX_INITIALIZER,	/* protects msg_ack_cv */
594 	0,				/* send/recv threads are not dying */
595 	0,				/* flag for start sending msg Q */
596 	0				/* indicate if the ACK has come  */
597 };
598 etm_iosvc_t *io_svc_p = &io_svc;
599 
600 
601 static uint32_t
602 flags;					/* flags for fmd_xprt_open */
603 
604 static etm_async_event_ele_t
605 async_event_q[ASYNC_EVENT_Q_SIZE];	/* holds the async events */
606 
607 static uint32_t
608 etm_async_q_head = 0;		/* ptr to cur head of async event queue */
609 
610 static uint32_t
611 etm_async_q_tail = 0;		/* ptr to cur tail of async event queue */
612 
613 static uint32_t
614 etm_async_q_cur_len = 0;	/* cur length (ele cnt) of async event queue */
615 
616 static uint32_t
617 etm_async_q_max_len = ASYNC_EVENT_Q_SIZE;
618 				/* max length (ele cnt) of async event queue */
619 
620 static pthread_cond_t
621 etm_async_event_q_cv = PTHREAD_COND_INITIALIZER;
622 				/* nudges  async event handler */
623 
624 static pthread_mutex_t
625 etm_async_event_q_lock = PTHREAD_MUTEX_INITIALIZER;
626 				/* protects async event q */
627 
628 static ds_ver_t
629 etm_iosvc_vers[] = { { 1, 0} };
630 
631 #define	ETM_NVERS	(sizeof (etm_iosvc_vers) / sizeof (ds_ver_t))
632 
633 static ds_capability_t
634 iosvc_caps = {
635 	"ETM",				/* svc_id */
636 	etm_iosvc_vers,			/* vers */
637 	ETM_NVERS			/* number of vers */
638 };
639 
640 static void
641 etm_iosvc_reg_handler(ds_hdl_t hdl, ds_cb_arg_t arg, ds_ver_t *ver,
642     ds_domain_hdl_t did);
643 
644 static void
645 etm_iosvc_unreg_handler(ds_hdl_t hdl, ds_cb_arg_t arg);
646 
647 static ds_ops_t
648 iosvc_ops = {
649 	etm_iosvc_reg_handler,		/* ds_reg_cb */
650 	etm_iosvc_unreg_handler,	/* ds_unreg_cb */
651 	NULL,				/* ds_data_cb */
652 	NULL				/* cb_arg */
653 };
654 
655 
656 /*
657  * -------------------------- support functions ------------------------------
658  */
659 
660 /*
661  * Design_Note:	Each failure worth reporting to FMD should be done using
662  *		a single call to fmd_hdl_error() as it logs an FMA event
663  *		for each call. Also be aware that all the fmd_hdl_*()
664  *		format strings currently use platform specific *printf()
665  *		routines; so "%p" under Solaris does not prepend "0x" to
666  *		the outputted hex digits, while Linux and VxWorks do.
667  */
668 
669 
670 /*
671  * etm_show_time - display the current time of day (for debugging) using
672  *		the given FMD module handle and annotation string
673  */
674 
675 static void
etm_show_time(fmd_hdl_t * hdl,char * note_str)676 etm_show_time(fmd_hdl_t *hdl, char *note_str)
677 {
678 	struct timeval		tmv;		/* timeval */
679 
680 	(void) gettimeofday(&tmv, NULL);
681 	fmd_hdl_debug(hdl, "info: %s: cur Unix Epoch time %d.%06d\n",
682 	    note_str, tmv.tv_sec, tmv.tv_usec);
683 
684 } /* etm_show_time() */
685 
686 /*
687  * etm_hexdump - hexdump the given buffer (for debugging) using
688  *		the given FMD module handle
689  */
690 
691 static void
etm_hexdump(fmd_hdl_t * hdl,void * buf,size_t byte_cnt)692 etm_hexdump(fmd_hdl_t *hdl, void *buf, size_t byte_cnt)
693 {
694 	uint8_t		*bp;		/* byte ptr */
695 	int		i, j;		/* index */
696 	char		cb[80];		/* char buf */
697 	unsigned int	n;		/* a byte of data for sprintf() */
698 
699 	bp = buf;
700 	j = 0;
701 
702 	/*
703 	 * Design_Note:	fmd_hdl_debug() auto adds a newline if missing;
704 	 *		hence cb exists to accumulate a longer string.
705 	 */
706 
707 	for (i = 1; i <= byte_cnt; i++) {
708 		n = *bp++;
709 		(void) sprintf(&cb[j], "%2.2x ", n);
710 		j += 3;
711 		/* add a newline every 16 bytes or at the buffer's end */
712 		if (((i % 16) == 0) || (i >= byte_cnt)) {
713 			cb[j-1] = '\0';
714 			fmd_hdl_debug(hdl, "%s\n", cb);
715 			j = 0;
716 		}
717 	} /* for each byte in the buffer */
718 
719 } /* etm_hexdump() */
720 
721 /*
722  * etm_sleep - sleep the caller for the given number of seconds,
723  *		return 0 or -errno value
724  *
725  * Design_Note:	To avoid interfering with FMD's signal mask (SIGALRM)
726  *		do not use [Solaris] sleep(3C) and instead use
727  *		pthread_cond_wait() or nanosleep(), both of which
728  *		are POSIX spec-ed to leave signal masks alone.
729  *		This is needed for Solaris and Linux (domain and SP).
730  */
731 
732 static int
etm_sleep(unsigned sleep_sec)733 etm_sleep(unsigned sleep_sec)
734 {
735 	struct timespec	tms;	/* for nanosleep() */
736 
737 	tms.tv_sec = sleep_sec;
738 	tms.tv_nsec = 0;
739 
740 	if (nanosleep(&tms, NULL) < 0) {
741 		/* errno assumed set by above call */
742 		return (-errno);
743 	}
744 	return (0);
745 
746 } /* etm_sleep() */
747 
748 /*
749  * etm_conn_open - open a connection to the given transport address,
750  *		return 0 and the opened connection handle
751  *		or -errno value
752  *
753  * caveats:	the err_substr is used in failure cases for calling
754  *		fmd_hdl_error()
755  */
756 
757 static int
etm_conn_open(fmd_hdl_t * hdl,char * err_substr,etm_xport_addr_t addr,etm_xport_conn_t * connp)758 etm_conn_open(fmd_hdl_t *hdl, char *err_substr,
759     etm_xport_addr_t addr, etm_xport_conn_t *connp)
760 {
761 	etm_xport_conn_t	conn;	/* connection to return */
762 	int			nev;	/* -errno value */
763 
764 	if ((conn = etm_xport_open(hdl, addr)) == NULL) {
765 		nev = (-errno);
766 		fmd_hdl_error(hdl, "error: %s: errno %d\n",
767 		    err_substr, errno);
768 		etm_stats.etm_xport_open_fail.fmds_value.ui64++;
769 		return (nev);
770 	} else {
771 		*connp = conn;
772 		return (0);
773 	}
774 } /* etm_conn_open() */
775 
776 /*
777  * etm_conn_close - close the given connection,
778  *		return 0 or -errno value
779  *
780  * caveats:	the err_substr is used in failure cases for calling
781  *		fmd_hdl_error()
782  */
783 
784 static int
etm_conn_close(fmd_hdl_t * hdl,char * err_substr,etm_xport_conn_t conn)785 etm_conn_close(fmd_hdl_t *hdl, char *err_substr, etm_xport_conn_t conn)
786 {
787 	int	nev;	/* -errno value */
788 
789 	if (etm_xport_close(hdl, conn) == NULL) {
790 		nev = (-errno);
791 		fmd_hdl_error(hdl, "warning: %s: errno %d\n",
792 		    err_substr, errno);
793 		etm_stats.etm_xport_close_fail.fmds_value.ui64++;
794 		return (nev);
795 	} else {
796 		return (0);
797 	}
798 } /* etm_conn_close() */
799 
800 /*
801  * etm_io_op - perform an IO operation on the given connection
802  *		with the given buffer,
803  *		accommodating MTU size and retrying op if needed,
804  *		return how many bytes actually done by the op
805  *		or -errno value
806  *
807  * caveats:	the err_substr is used in failure cases for calling
808  *		fmd_hdl_error()
809  */
810 
811 static ssize_t
etm_io_op(fmd_hdl_t * hdl,char * err_substr,etm_xport_conn_t conn,void * buf,size_t byte_cnt,int io_op)812 etm_io_op(fmd_hdl_t *hdl, char *err_substr, etm_xport_conn_t conn,
813     void *buf, size_t byte_cnt, int io_op)
814 {
815 	ssize_t		rv;		/* ret val / byte count */
816 	ssize_t		n;		/* gen use */
817 	uint8_t		*datap;		/* ptr to data */
818 	size_t		mtu_sz;		/* MTU size in bytes */
819 	int		(*io_func_ptr)(fmd_hdl_t *, etm_xport_conn_t,
820 	    void *, size_t);
821 	size_t		io_sz;		/* byte count for io_func_ptr */
822 	int		try_cnt;	/* number of tries done */
823 	int		sleep_sec;	/* exp backoff sleep period in sec */
824 	int		sleep_rv;	/* ret val from sleeping */
825 	fmd_stat_t	io_retry_stat;	/* IO retry stat to update */
826 	fmd_stat_t	io_fail_stat;	/* IO failure stat to update */
827 
828 	if ((conn == NULL) || (buf == NULL)) {
829 		return (-EINVAL);
830 	}
831 	switch (io_op) {
832 	case ETM_IO_OP_RD:
833 		io_func_ptr = etm_xport_read;
834 		io_retry_stat = etm_stats.etm_xport_rd_retry;
835 		io_fail_stat = etm_stats.etm_xport_rd_fail;
836 		break;
837 	case ETM_IO_OP_WR:
838 		io_func_ptr = etm_xport_write;
839 		io_retry_stat = etm_stats.etm_xport_wr_retry;
840 		io_fail_stat = etm_stats.etm_xport_wr_fail;
841 		break;
842 	default:
843 		return (-EINVAL);
844 	}
845 	if (byte_cnt == 0) {
846 		return (byte_cnt);	/* nop */
847 	}
848 
849 	/* obtain [current] MTU size */
850 
851 	if ((n = etm_xport_get_opt(hdl, conn, ETM_XPORT_OPT_MTU_SZ)) < 0) {
852 		mtu_sz = ETM_XPORT_MTU_SZ_DEF;
853 	} else {
854 		mtu_sz = n;
855 	}
856 
857 	/* loop until all IO done, try limit exceeded, or real failure */
858 
859 	rv = 0;
860 	datap = buf;
861 	while (rv < byte_cnt) {
862 		io_sz = MIN((byte_cnt - rv), mtu_sz);
863 		try_cnt = 0;
864 		sleep_sec = 0;
865 
866 		/* when give up, return -errno value even if partly done */
867 
868 		while ((n = (*io_func_ptr)(hdl, conn, datap, io_sz)) ==
869 		    (-EAGAIN)) {
870 			try_cnt++;
871 			if (try_cnt > ETM_TRY_MAX_CNT) {
872 				rv = n;
873 				goto func_ret;
874 			}
875 			if (etm_is_dying) {
876 				rv = (-EINTR);
877 				goto func_ret;
878 			}
879 			if ((sleep_rv = etm_sleep(sleep_sec)) < 0) {
880 				rv = sleep_rv;
881 				goto func_ret;
882 			}
883 			sleep_sec = ((sleep_sec == 0) ? 1 :
884 			    (sleep_sec * ETM_TRY_BACKOFF_RATE));
885 			sleep_sec = MIN(sleep_sec, ETM_TRY_BACKOFF_CAP);
886 			io_retry_stat.fmds_value.ui64++;
887 			if (etm_debug_lvl >= 1) {
888 				fmd_hdl_debug(hdl, "info: retrying io op %d "
889 				    "due to EAGAIN\n", io_op);
890 			}
891 		} /* while trying the io operation */
892 
893 		if (etm_is_dying) {
894 			rv = (-EINTR);
895 			goto func_ret;
896 		}
897 		if (n < 0) {
898 			rv = n;
899 			goto func_ret;
900 		}
901 		/* avoid spinning CPU when given 0 bytes but no error */
902 		if (n == 0) {
903 			if ((sleep_rv = etm_sleep(ETM_SLEEP_QUIK)) < 0) {
904 				rv = sleep_rv;
905 				goto func_ret;
906 			}
907 		}
908 		rv += n;
909 		datap += n;
910 	} /* while still have more data */
911 
912 func_ret:
913 
914 	if (rv < 0) {
915 		io_fail_stat.fmds_value.ui64++;
916 		fmd_hdl_debug(hdl, "error: %s: errno %d\n",
917 		    err_substr, (int)(-rv));
918 	}
919 	if (etm_debug_lvl >= 3) {
920 		fmd_hdl_debug(hdl, "info: io op %d ret %d of %d\n",
921 		    io_op, (int)rv, (int)byte_cnt);
922 	}
923 	return (rv);
924 
925 } /* etm_io_op() */
926 
927 /*
928  * etm_magic_read - read the magic number of an ETM message header
929  *		from the given connection into the given buffer,
930  *		return 0 or -errno value
931  *
932  * Design_Note:	This routine is intended to help protect ETM from protocol
933  *		framing errors as might be caused by an SP reset / crash in
934  *		the middle of an ETM message send; the connection will be
935  *		read from for as many bytes as needed until the magic number
936  *		is found using a sliding buffer for comparisons.
937  */
938 
939 static int
etm_magic_read(fmd_hdl_t * hdl,etm_xport_conn_t conn,uint32_t * magic_ptr)940 etm_magic_read(fmd_hdl_t *hdl, etm_xport_conn_t conn, uint32_t *magic_ptr)
941 {
942 	int		rv;		/* ret val */
943 	uint32_t	magic_num;	/* magic number */
944 	int		byte_cnt;	/* count of bytes read */
945 	uint8_t		buf5[4+1];	/* sliding input buffer */
946 	int		i, j;		/* indices into buf5 */
947 	ssize_t		n;		/* gen use */
948 	uint8_t		drop_buf[1024];	/* dropped bytes buffer */
949 
950 	rv = 0;		/* assume success */
951 	magic_num = 0;
952 	byte_cnt = 0;
953 	j = 0;
954 
955 	/* magic number bytes are sent in network (big endian) order */
956 
957 	while (magic_num != ETM_PROTO_MAGIC_NUM) {
958 		if ((n = etm_io_op(hdl, "bad io read on magic",
959 		    conn, &buf5[j], 1, ETM_IO_OP_RD)) < 0) {
960 			rv = n;
961 			goto func_ret;
962 		}
963 		byte_cnt++;
964 		j = MIN((j + 1), sizeof (magic_num));
965 		if (byte_cnt < sizeof (magic_num)) {
966 			continue;
967 		}
968 
969 		if (byte_cnt > sizeof (magic_num)) {
970 			etm_stats.etm_magic_drop_bytes.fmds_value.ui64++;
971 			i = MIN(byte_cnt - j - 1, sizeof (drop_buf) - 1);
972 			drop_buf[i] = buf5[0];
973 			for (i = 0; i < j; i++) {
974 				buf5[i] = buf5[i+1];
975 			} /* for sliding the buffer contents */
976 		}
977 		(void) memcpy(&magic_num, &buf5[0], sizeof (magic_num));
978 		magic_num = ntohl(magic_num);
979 	} /* for reading bytes until find magic number */
980 
981 func_ret:
982 
983 	if (byte_cnt != sizeof (magic_num)) {
984 		fmd_hdl_debug(hdl, "warning: bad proto frame "
985 		    "implies corrupt/lost msg(s)\n");
986 	}
987 	if ((byte_cnt > sizeof (magic_num)) && (etm_debug_lvl >= 2)) {
988 		i = MIN(byte_cnt - sizeof (magic_num), sizeof (drop_buf));
989 		fmd_hdl_debug(hdl, "info: magic drop hexdump "
990 		    "first %d of %d bytes:\n", i,
991 		    byte_cnt - sizeof (magic_num));
992 		etm_hexdump(hdl, drop_buf, i);
993 	}
994 
995 	if (rv == 0) {
996 		*magic_ptr = magic_num;
997 	}
998 	return (rv);
999 
1000 } /* etm_magic_read() */
1001 
1002 /*
1003  * etm_hdr_read - allocate, read, and validate a [variable sized]
1004  *		ETM message header from the given connection,
1005  *		return the allocated ETM message header
1006  *		(which is guaranteed to be large enough to reuse as a
1007  *		RESPONSE msg hdr) and its size
1008  *		or NULL and set errno on failure
1009  */
1010 
1011 static void *
etm_hdr_read(fmd_hdl_t * hdl,etm_xport_conn_t conn,size_t * szp)1012 etm_hdr_read(fmd_hdl_t *hdl, etm_xport_conn_t conn, size_t *szp)
1013 {
1014 	uint8_t			*hdrp;		/* ptr to header to return */
1015 	size_t			hdr_sz;		/* sizeof *hdrp */
1016 	etm_proto_v1_pp_t	pp;		/* protocol preamble */
1017 	etm_proto_v1_ev_hdr_t	*ev_hdrp;	/* for FMA_EVENT msg */
1018 	etm_proto_v1_ctl_hdr_t	*ctl_hdrp;	/* for CONTROL msg */
1019 	etm_proto_v1_resp_hdr_t *resp_hdrp;	/* for RESPONSE msg */
1020 	etm_proto_v3_sa_hdr_t	*sa_hdrp;	/* for ALERT msg */
1021 	uint32_t		*lenp;		/* ptr to FMA event length */
1022 	ssize_t			i, n;		/* gen use */
1023 	uint8_t	misc_buf[ETM_MISC_BUF_SZ];	/* for var sized hdrs */
1024 	int			dummy_int;	/* dummy var to appease lint */
1025 
1026 	hdrp = NULL; hdr_sz = 0;
1027 
1028 	/* read the magic number which starts the protocol preamble */
1029 
1030 	if ((n = etm_magic_read(hdl, conn, &pp.pp_magic_num)) < 0) {
1031 		errno = (-n);
1032 		etm_stats.etm_magic_bad.fmds_value.ui64++;
1033 		return (NULL);
1034 	}
1035 
1036 	/* read the rest of the protocol preamble all at once */
1037 
1038 	if ((n = etm_io_op(hdl, "bad io read on preamble",
1039 	    conn, &pp.pp_proto_ver, sizeof (pp) - sizeof (pp.pp_magic_num),
1040 	    ETM_IO_OP_RD)) < 0) {
1041 		errno = (-n);
1042 		return (NULL);
1043 	}
1044 
1045 	/*
1046 	 * Design_Note:	The magic number was already network decoded; but
1047 	 *		some other preamble fields also need to be decoded,
1048 	 *		specifically pp_xid and pp_timeout. The rest of the
1049 	 *		preamble fields are byte sized and hence need no
1050 	 *		decoding.
1051 	 */
1052 
1053 	pp.pp_xid = ntohl(pp.pp_xid);
1054 	pp.pp_timeout = ntohl(pp.pp_timeout);
1055 
1056 	/* sanity check the header as best we can */
1057 
1058 	if ((pp.pp_proto_ver < ETM_PROTO_V1) ||
1059 	    (pp.pp_proto_ver > ETM_PROTO_V3)) {
1060 		fmd_hdl_error(hdl, "error: bad proto ver %d\n",
1061 		    (int)pp.pp_proto_ver);
1062 		errno = EPROTO;
1063 		etm_stats.etm_ver_bad.fmds_value.ui64++;
1064 		return (NULL);
1065 	}
1066 
1067 	dummy_int = pp.pp_msg_type;
1068 	if ((dummy_int <= ETM_MSG_TYPE_TOO_LOW) ||
1069 	    (dummy_int >= ETM_MSG_TYPE_TOO_BIG)) {
1070 		fmd_hdl_error(hdl, "error: bad msg type %d", dummy_int);
1071 		errno = EBADMSG;
1072 		etm_stats.etm_msgtype_bad.fmds_value.ui64++;
1073 		return (NULL);
1074 	}
1075 
1076 	/* handle [var sized] hdrs for FMA_EVENT, CONTROL, RESPONSE msgs */
1077 
1078 	if (pp.pp_msg_type == ETM_MSG_TYPE_FMA_EVENT) {
1079 
1080 		ev_hdrp = (void*)&misc_buf[0];
1081 		hdr_sz = sizeof (*ev_hdrp);
1082 		(void) memcpy(&ev_hdrp->ev_pp, &pp, sizeof (pp));
1083 
1084 		/* sanity check the header's timeout */
1085 
1086 		if ((ev_hdrp->ev_pp.pp_proto_ver == ETM_PROTO_V1) &&
1087 		    (ev_hdrp->ev_pp.pp_timeout != ETM_PROTO_V1_TIMEOUT_NONE)) {
1088 			errno = ETIME;
1089 			etm_stats.etm_timeout_bad.fmds_value.ui64++;
1090 			return (NULL);
1091 		}
1092 
1093 		/* get all FMA event lengths from the header */
1094 
1095 		lenp = (uint32_t *)&ev_hdrp->ev_lens[0]; lenp--;
1096 		i = -1;	/* cnt of length entries preceding 0 */
1097 		do {
1098 			i++; lenp++;
1099 			if ((sizeof (*ev_hdrp) + (i * sizeof (*lenp))) >=
1100 			    ETM_MISC_BUF_SZ) {
1101 				errno = E2BIG;	/* ridiculous size */
1102 				etm_stats.etm_evlens_bad.fmds_value.ui64++;
1103 				return (NULL);
1104 			}
1105 			if ((n = etm_io_op(hdl, "bad io read on event len",
1106 			    conn, lenp, sizeof (*lenp), ETM_IO_OP_RD)) < 0) {
1107 				errno = (-n);
1108 				return (NULL);
1109 			}
1110 			*lenp = ntohl(*lenp);
1111 
1112 		} while (*lenp != 0);
1113 		i += 0; /* first len already counted by sizeof(ev_hdr) */
1114 		hdr_sz += (i * sizeof (*lenp));
1115 
1116 		etm_stats.etm_rd_hdr_fmaevent.fmds_value.ui64++;
1117 
1118 	} else if (pp.pp_msg_type == ETM_MSG_TYPE_CONTROL) {
1119 
1120 		ctl_hdrp = (void*)&misc_buf[0];
1121 		hdr_sz = sizeof (*ctl_hdrp);
1122 		(void) memcpy(&ctl_hdrp->ctl_pp, &pp, sizeof (pp));
1123 
1124 		/* sanity check the header's sub type (control selector) */
1125 
1126 		if ((ctl_hdrp->ctl_pp.pp_sub_type <= ETM_CTL_SEL_TOO_LOW) ||
1127 		    (ctl_hdrp->ctl_pp.pp_sub_type >= ETM_CTL_SEL_TOO_BIG)) {
1128 			fmd_hdl_error(hdl, "error: bad ctl sub type %d\n",
1129 			    (int)ctl_hdrp->ctl_pp.pp_sub_type);
1130 			errno = EBADMSG;
1131 			etm_stats.etm_subtype_bad.fmds_value.ui64++;
1132 			return (NULL);
1133 		}
1134 
1135 		/* get the control length */
1136 
1137 		if ((n = etm_io_op(hdl, "bad io read on ctl len",
1138 		    conn, &ctl_hdrp->ctl_len, sizeof (ctl_hdrp->ctl_len),
1139 		    ETM_IO_OP_RD)) < 0) {
1140 			errno = (-n);
1141 			return (NULL);
1142 		}
1143 
1144 		ctl_hdrp->ctl_len = ntohl(ctl_hdrp->ctl_len);
1145 
1146 		etm_stats.etm_rd_hdr_control.fmds_value.ui64++;
1147 
1148 	} else if (pp.pp_msg_type == ETM_MSG_TYPE_RESPONSE) {
1149 
1150 		resp_hdrp = (void*)&misc_buf[0];
1151 		hdr_sz = sizeof (*resp_hdrp);
1152 		(void) memcpy(&resp_hdrp->resp_pp, &pp, sizeof (pp));
1153 
1154 		/* sanity check the header's timeout */
1155 
1156 		if (resp_hdrp->resp_pp.pp_timeout !=
1157 		    ETM_PROTO_V1_TIMEOUT_NONE) {
1158 			errno = ETIME;
1159 			etm_stats.etm_timeout_bad.fmds_value.ui64++;
1160 			return (NULL);
1161 		}
1162 
1163 		/* get the response code and length */
1164 
1165 		if ((n = etm_io_op(hdl, "bad io read on resp code+len",
1166 		    conn, &resp_hdrp->resp_code,
1167 		    sizeof (resp_hdrp->resp_code)
1168 		    + sizeof (resp_hdrp->resp_len),
1169 		    ETM_IO_OP_RD)) < 0) {
1170 			errno = (-n);
1171 			return (NULL);
1172 		}
1173 
1174 		resp_hdrp->resp_code = ntohl(resp_hdrp->resp_code);
1175 		resp_hdrp->resp_len = ntohl(resp_hdrp->resp_len);
1176 
1177 		etm_stats.etm_rd_hdr_response.fmds_value.ui64++;
1178 
1179 	} else if (pp.pp_msg_type == ETM_MSG_TYPE_ALERT) {
1180 
1181 		sa_hdrp = (void*)&misc_buf[0];
1182 		hdr_sz = sizeof (*sa_hdrp);
1183 		(void) memcpy(&sa_hdrp->sa_pp, &pp, sizeof (pp));
1184 
1185 		/* sanity check the header's protocol version */
1186 
1187 		if (sa_hdrp->sa_pp.pp_proto_ver != ETM_PROTO_V3) {
1188 			errno = EPROTO;
1189 			etm_stats.etm_ver_bad.fmds_value.ui64++;
1190 			return (NULL);
1191 		}
1192 
1193 		/* get the priority and length */
1194 
1195 		if ((n = etm_io_op(hdl, "bad io read on sa priority+len",
1196 		    conn, &sa_hdrp->sa_priority,
1197 		    sizeof (sa_hdrp->sa_priority)
1198 		    + sizeof (sa_hdrp->sa_len),
1199 		    ETM_IO_OP_RD)) < 0) {
1200 			errno = (-n);
1201 			return (NULL);
1202 		}
1203 
1204 		sa_hdrp->sa_priority = ntohl(sa_hdrp->sa_priority);
1205 		sa_hdrp->sa_len = ntohl(sa_hdrp->sa_len);
1206 
1207 		etm_stats.etm_rd_hdr_alert.fmds_value.ui64++;
1208 
1209 	} /* whether we have FMA_EVENT, ALERT, CONTROL, or RESPONSE msg */
1210 
1211 	/*
1212 	 * choose a header size that allows hdr reuse for RESPONSE msgs,
1213 	 * allocate and populate the message header, and
1214 	 * return alloc size to caller for later free of hdrp
1215 	 */
1216 
1217 	hdr_sz = MAX(hdr_sz, sizeof (*resp_hdrp));
1218 	hdrp = fmd_hdl_zalloc(hdl, hdr_sz, FMD_SLEEP);
1219 	(void) memcpy(hdrp, misc_buf, hdr_sz);
1220 
1221 	if (etm_debug_lvl >= 3) {
1222 		fmd_hdl_debug(hdl, "info: msg hdr hexdump %d bytes:\n", hdr_sz);
1223 		etm_hexdump(hdl, hdrp, hdr_sz);
1224 	}
1225 	*szp = hdr_sz;
1226 	return (hdrp);
1227 
1228 } /* etm_hdr_read() */
1229 
1230 /*
1231  * etm_hdr_write - create and write a [variable sized] ETM message header
1232  *		to the given connection appropriate for the given FMA event
1233  *		and type of nvlist encoding,
1234  *		return the allocated ETM message header and its size
1235  *		or NULL and set errno on failure
1236  */
1237 
1238 static void*
etm_hdr_write(fmd_hdl_t * hdl,etm_xport_conn_t conn,nvlist_t * evp,int encoding,size_t * szp)1239 etm_hdr_write(fmd_hdl_t *hdl, etm_xport_conn_t conn, nvlist_t *evp,
1240     int encoding, size_t *szp)
1241 {
1242 	etm_proto_v1_ev_hdr_t	*hdrp;		/* for FMA_EVENT msg */
1243 	size_t			hdr_sz;		/* sizeof *hdrp */
1244 	uint32_t		*lenp;		/* ptr to FMA event length */
1245 	size_t			evsz;		/* packed FMA event size */
1246 	ssize_t			n;		/* gen use */
1247 
1248 	/* allocate and populate the message header for 1 FMA event */
1249 
1250 	hdr_sz = sizeof (*hdrp) + (1 * sizeof (hdrp->ev_lens[0]));
1251 
1252 	hdrp = fmd_hdl_zalloc(hdl, hdr_sz, FMD_SLEEP);
1253 
1254 	/*
1255 	 * Design_Note: Although the ETM protocol supports it, we do not (yet)
1256 	 *		want responses/ACKs on FMA events that we send. All
1257 	 *		such messages are sent with ETM_PROTO_V1_TIMEOUT_NONE.
1258 	 */
1259 
1260 	hdrp->ev_pp.pp_magic_num = ETM_PROTO_MAGIC_NUM;
1261 	hdrp->ev_pp.pp_magic_num = htonl(hdrp->ev_pp.pp_magic_num);
1262 	hdrp->ev_pp.pp_proto_ver = ETM_PROTO_V1;
1263 	hdrp->ev_pp.pp_msg_type = ETM_MSG_TYPE_FMA_EVENT;
1264 	hdrp->ev_pp.pp_sub_type = 0;
1265 	hdrp->ev_pp.pp_rsvd_pad = 0;
1266 	hdrp->ev_pp.pp_xid = etm_xid_cur;
1267 	hdrp->ev_pp.pp_xid = htonl(hdrp->ev_pp.pp_xid);
1268 	etm_xid_cur += ETM_XID_INC;
1269 	hdrp->ev_pp.pp_timeout = ETM_PROTO_V1_TIMEOUT_NONE;
1270 	hdrp->ev_pp.pp_timeout = htonl(hdrp->ev_pp.pp_timeout);
1271 
1272 	lenp = &hdrp->ev_lens[0];
1273 
1274 	if ((n = nvlist_size(evp, &evsz, encoding)) != 0) {
1275 		errno = n;
1276 		fmd_hdl_free(hdl, hdrp, hdr_sz);
1277 		etm_stats.etm_os_nvlist_size_fail.fmds_value.ui64++;
1278 		return (NULL);
1279 	}
1280 
1281 	/* indicate 1 FMA event, network encode its length, and 0-terminate */
1282 
1283 	etm_stats.etm_wr_max_ev_per_msg.fmds_value.ui64 = 1;
1284 
1285 	*lenp = evsz; *lenp = htonl(*lenp); lenp++;
1286 	*lenp = 0; *lenp = htonl(*lenp); lenp++;
1287 
1288 	/*
1289 	 * write the network encoded header to the transport, and
1290 	 * return alloc size to caller for later free
1291 	 */
1292 
1293 	if ((n = etm_io_op(hdl, "bad io write on event hdr",
1294 	    conn, hdrp, hdr_sz, ETM_IO_OP_WR)) < 0) {
1295 		errno = (-n);
1296 		fmd_hdl_free(hdl, hdrp, hdr_sz);
1297 		return (NULL);
1298 	}
1299 
1300 	*szp = hdr_sz;
1301 	return (hdrp);
1302 
1303 } /* etm_hdr_write() */
1304 
1305 /*
1306  * etm_post_to_fmd - post the given FMA event to FMD
1307  *			via a FMD transport API call,
1308  *			return 0 or -errno value
1309  *
1310  * caveats:	the FMA event (evp) is freed by FMD,
1311  *		thus callers of this function should
1312  *		immediately discard any ptr they have to the
1313  *		nvlist without freeing or dereferencing it
1314  */
1315 
1316 static int
etm_post_to_fmd(fmd_hdl_t * hdl,fmd_xprt_t * fmd_xprt,nvlist_t * evp)1317 etm_post_to_fmd(fmd_hdl_t *hdl, fmd_xprt_t *fmd_xprt, nvlist_t *evp)
1318 {
1319 	ssize_t			ev_sz;		/* sizeof *evp */
1320 
1321 	(void) nvlist_size(evp, (size_t *)&ev_sz, NV_ENCODE_XDR);
1322 
1323 	if (etm_debug_lvl >= 2) {
1324 		etm_show_time(hdl, "ante ev post");
1325 	}
1326 	fmd_xprt_post(hdl, fmd_xprt, evp, 0);
1327 	etm_stats.etm_wr_fmd_fmaevent.fmds_value.ui64++;
1328 	etm_stats.etm_wr_fmd_bytes.fmds_value.ui64 += ev_sz;
1329 	if (etm_debug_lvl >= 1) {
1330 		fmd_hdl_debug(hdl, "info: event %p post ok to FMD\n", evp);
1331 	}
1332 	if (etm_debug_lvl >= 2) {
1333 		etm_show_time(hdl, "post ev post");
1334 	}
1335 	return (0);
1336 
1337 } /* etm_post_to_fmd() */
1338 
1339 /*
1340  * Ideally we would just use syslog(3C) for outputting our messages.
1341  * Unfortunately, as this module is running within the FMA daemon context,
1342  * that would create the situation where this module's openlog() would
1343  * have the monopoly on syslog(3C) for the daemon and all its modules.
1344  * To avoid that situation, this module uses the same logic as the
1345  * syslog-msgs FM module to directly call into the log(4D) and sysmsg(4D)
1346  * devices for syslog and console.
1347  */
1348 
1349 static int
etm_post_to_syslog(fmd_hdl_t * hdl,uint32_t priority,uint32_t body_sz,uint8_t * body_buf)1350 etm_post_to_syslog(fmd_hdl_t *hdl, uint32_t priority, uint32_t body_sz,
1351     uint8_t *body_buf)
1352 {
1353 	char		*sysmessage;	/* Formatted message */
1354 	size_t		formatlen;	/* maximum length of sysmessage */
1355 	struct strbuf	ctl, dat;	/* structs pushed to the logfd */
1356 	uint32_t	msgid;		/* syslog message ID number */
1357 
1358 	if ((syslog_file == 0) && (syslog_cons == 0)) {
1359 		return (0);
1360 	}
1361 
1362 	if (etm_debug_lvl >= 2) {
1363 		etm_show_time(hdl, "ante syslog post");
1364 	}
1365 
1366 	formatlen = body_sz + 64; /* +64 for prefix strings added below */
1367 	sysmessage = fmd_hdl_zalloc(hdl, formatlen, FMD_SLEEP);
1368 
1369 	if (syslog_file) {
1370 		STRLOG_MAKE_MSGID(body_buf, msgid);
1371 		(void) snprintf(sysmessage, formatlen,
1372 		    "SC Alert: [ID %u FACILITY_AND_PRIORITY] %s", msgid,
1373 		    body_buf);
1374 
1375 		syslog_ctl.pri = syslog_facility | priority;
1376 
1377 		ctl.buf = (void *)&syslog_ctl;
1378 		ctl.len = sizeof (syslog_ctl);
1379 
1380 		dat.buf = sysmessage;
1381 		dat.len = strlen(sysmessage) + 1;
1382 
1383 		if (putmsg(syslog_logfd, &ctl, &dat, 0) != 0) {
1384 			fmd_hdl_debug(hdl, "putmsg failed: %s\n",
1385 			    strerror(errno));
1386 			etm_stats.etm_log_err.fmds_value.ui64++;
1387 		}
1388 	}
1389 
1390 	if (syslog_cons) {
1391 		(void) snprintf(sysmessage, formatlen,
1392 		    "SC Alert: %s\r\n", body_buf);
1393 
1394 		dat.buf = sysmessage;
1395 		dat.len = strlen(sysmessage) + 1;
1396 
1397 		if (write(syslog_msgfd, dat.buf, dat.len) != dat.len) {
1398 			fmd_hdl_debug(hdl, "write failed: %s\n",
1399 			    strerror(errno));
1400 			etm_stats.etm_msg_err.fmds_value.ui64++;
1401 		}
1402 	}
1403 
1404 	fmd_hdl_free(hdl, sysmessage, formatlen);
1405 
1406 	if (etm_debug_lvl >= 2) {
1407 		etm_show_time(hdl, "post syslog post");
1408 	}
1409 
1410 	return (0);
1411 }
1412 
1413 
1414 /*
1415  * etm_req_ver_negot - send an ETM control message to the other end requesting
1416  *			that the ETM protocol version be negotiated/set
1417  */
1418 
1419 static void
etm_req_ver_negot(fmd_hdl_t * hdl)1420 etm_req_ver_negot(fmd_hdl_t *hdl)
1421 {
1422 	etm_xport_addr_t	*addrv;		/* default dst addr(s) */
1423 	etm_xport_conn_t	conn;		/* connection to other end */
1424 	etm_proto_v1_ctl_hdr_t	*ctl_hdrp;	/* for CONTROL msg */
1425 	size_t			hdr_sz;		/* sizeof header */
1426 	uint8_t			*body_buf;	/* msg body buffer */
1427 	uint32_t		body_sz;	/* sizeof *body_buf */
1428 	ssize_t			i;		/* gen use */
1429 
1430 	/* populate an ETM control msg to send */
1431 
1432 	hdr_sz = sizeof (*ctl_hdrp);
1433 	body_sz = (3 + 1);		/* version bytes plus null byte */
1434 
1435 	ctl_hdrp = fmd_hdl_zalloc(hdl, hdr_sz + body_sz, FMD_SLEEP);
1436 
1437 	ctl_hdrp->ctl_pp.pp_magic_num = htonl(ETM_PROTO_MAGIC_NUM);
1438 	ctl_hdrp->ctl_pp.pp_proto_ver = ETM_PROTO_V1;
1439 	ctl_hdrp->ctl_pp.pp_msg_type = ETM_MSG_TYPE_CONTROL;
1440 	ctl_hdrp->ctl_pp.pp_sub_type = ETM_CTL_SEL_VER_NEGOT_REQ;
1441 	ctl_hdrp->ctl_pp.pp_rsvd_pad = 0;
1442 	etm_xid_ver_negot = etm_xid_cur;
1443 	etm_xid_cur += ETM_XID_INC;
1444 	ctl_hdrp->ctl_pp.pp_xid = htonl(etm_xid_ver_negot);
1445 	ctl_hdrp->ctl_pp.pp_timeout = htonl(ETM_PROTO_V1_TIMEOUT_FOREVER);
1446 	ctl_hdrp->ctl_len = htonl(body_sz);
1447 
1448 	body_buf = (void*)&ctl_hdrp->ctl_len;
1449 	body_buf += sizeof (ctl_hdrp->ctl_len);
1450 	*body_buf++ = ETM_PROTO_V3;
1451 	*body_buf++ = ETM_PROTO_V2;
1452 	*body_buf++ = ETM_PROTO_V1;
1453 	*body_buf++ = '\0';
1454 
1455 	/*
1456 	 * open and close a connection to send the ETM control msg
1457 	 * to any/all of the default dst addrs
1458 	 */
1459 
1460 	if ((addrv = etm_xport_get_ev_addrv(hdl, NULL)) == NULL) {
1461 		fmd_hdl_error(hdl,
1462 		    "error: bad ctl dst addrs errno %d\n", errno);
1463 		etm_stats.etm_xport_get_ev_addrv_fail.fmds_value.ui64++;
1464 		goto func_ret;
1465 	}
1466 
1467 	for (i = 0; addrv[i] != NULL; i++) {
1468 
1469 		if (etm_conn_open(hdl, "bad conn open during ver negot",
1470 		    addrv[i], &conn) < 0) {
1471 			continue;
1472 		}
1473 		if (etm_io_op(hdl, "bad io write on ctl hdr+body",
1474 		    conn, ctl_hdrp, hdr_sz + body_sz, ETM_IO_OP_WR) >= 0) {
1475 			etm_stats.etm_wr_hdr_control.fmds_value.ui64++;
1476 			etm_stats.etm_wr_body_control.fmds_value.ui64++;
1477 		}
1478 		(void) etm_conn_close(hdl, "bad conn close during ver negot",
1479 		    conn);
1480 
1481 	} /* foreach dst addr */
1482 
1483 func_ret:
1484 
1485 	if (addrv != NULL) {
1486 		etm_xport_free_addrv(hdl, addrv);
1487 	}
1488 	fmd_hdl_free(hdl, ctl_hdrp, hdr_sz + body_sz);
1489 
1490 } /* etm_req_ver_negot() */
1491 
1492 
1493 
1494 /*
1495  * etm_iosvc_msg_enq - add element to tail of ETM iosvc msg queue
1496  * etm_iosvc_msg_deq - del element from head of ETM iosvc msg  queue
1497  * need to grab the mutex lock before calling this routine
1498  * return >0 for success, or -errno value
1499  */
1500 static int
etm_iosvc_msg_enq(fmd_hdl_t * hdl,etm_iosvc_t * iosvc,etm_iosvc_q_ele_t * msgp)1501 etm_iosvc_msg_enq(fmd_hdl_t *hdl, etm_iosvc_t *iosvc, etm_iosvc_q_ele_t *msgp)
1502 {
1503 	etm_iosvc_q_ele_t		*newp;	/* ptr to new msg q ele */
1504 
1505 	if (iosvc->msg_q_cur_len >= iosvc->msg_q_max_len) {
1506 		fmd_hdl_debug(hdl, "warning: enq to full msg queue\n");
1507 		return (-E2BIG);
1508 	}
1509 
1510 	newp = fmd_hdl_zalloc(hdl, sizeof (*newp), FMD_SLEEP);
1511 	(void) memcpy(newp, msgp, sizeof (*newp));
1512 	newp->msg_nextp = NULL;
1513 
1514 	if (iosvc->msg_q_cur_len == 0) {
1515 		iosvc->msg_q_head = newp;
1516 	} else {
1517 		iosvc->msg_q_tail->msg_nextp = newp;
1518 	}
1519 
1520 	iosvc->msg_q_tail = newp;
1521 	iosvc->msg_q_cur_len++;
1522 	fmd_hdl_debug(hdl, "info: current msg queue length %d\n",
1523 	    iosvc->msg_q_cur_len);
1524 
1525 	return (1);
1526 
1527 } /* etm_iosvc_msg_enq() */
1528 
1529 static int
etm_iosvc_msg_deq(fmd_hdl_t * hdl,etm_iosvc_t * iosvc,etm_iosvc_q_ele_t * msgp)1530 etm_iosvc_msg_deq(fmd_hdl_t *hdl, etm_iosvc_t *iosvc, etm_iosvc_q_ele_t *msgp)
1531 {
1532 	etm_iosvc_q_ele_t	*oldp;	/* ptr to old msg q ele */
1533 
1534 	if (iosvc->msg_q_cur_len == 0) {
1535 		fmd_hdl_debug(hdl, "warning: deq from empty responder queue\n");
1536 		return (-ENOENT);
1537 	}
1538 
1539 	(void) memcpy(msgp, iosvc->msg_q_head, sizeof (*msgp));
1540 	msgp->msg_nextp = NULL;
1541 
1542 	oldp = iosvc->msg_q_head;
1543 	iosvc->msg_q_head = iosvc->msg_q_head->msg_nextp;
1544 
1545 	/*
1546 	 * free the mem alloc-ed in etm_iosvc_msg_enq()
1547 	 */
1548 	fmd_hdl_free(hdl, oldp, sizeof (*oldp));
1549 
1550 	iosvc->msg_q_cur_len--;
1551 	if (iosvc->msg_q_cur_len == 0) {
1552 		iosvc->msg_q_tail = NULL;
1553 	}
1554 
1555 	return (1);
1556 
1557 } /* etm_iosvc_msg_deq() */
1558 
1559 
1560 /*
1561  * etm_msg_enq_head():
1562  * enq the msg to the head of the Q.
1563  * If the Q is full, drop the msg at the tail then enq the msg at head.
1564  * need to grab mutex lock iosvc->msg_q_lock before calling this routine.
1565  */
1566 static void
etm_msg_enq_head(fmd_hdl_t * fmd_hdl,etm_iosvc_t * iosvc,etm_iosvc_q_ele_t * msg_ele)1567 etm_msg_enq_head(fmd_hdl_t *fmd_hdl, etm_iosvc_t *iosvc,
1568     etm_iosvc_q_ele_t *msg_ele)
1569 {
1570 
1571 	etm_iosvc_q_ele_t	*newp;	/* iosvc msg ele ptr */
1572 
1573 	if (iosvc->msg_q_cur_len >= iosvc->msg_q_max_len) {
1574 		fmd_hdl_debug(fmd_hdl,
1575 		    "warning: add to head of a full msg queue."
1576 		    " Drop the msg at the tail\n");
1577 		/*
1578 		 * drop the msg at the tail
1579 		 */
1580 		newp = iosvc->msg_q_head;
1581 		while (newp->msg_nextp != iosvc->msg_q_tail) {
1582 			newp = newp->msg_nextp;
1583 		}
1584 
1585 		/*
1586 		 * free the msg in iosvc->msg_q_tail->msg
1587 		 * free the mem pointed to by iosvc->msg_q_tail
1588 		 */
1589 		fmd_hdl_free(fmd_hdl, iosvc->msg_q_tail->msg,
1590 		    iosvc->msg_q_tail->msg_size);
1591 		fmd_hdl_free(fmd_hdl, iosvc->msg_q_tail, sizeof (*newp));
1592 		iosvc->msg_q_tail = newp;
1593 		iosvc->msg_q_tail->msg_nextp = NULL;
1594 		iosvc->msg_q_cur_len--;
1595 	}
1596 
1597 	/*
1598 	 * enq the msg to the head
1599 	 */
1600 	newp = fmd_hdl_zalloc(fmd_hdl, sizeof (*newp), FMD_SLEEP);
1601 	(void) memcpy(newp, msg_ele, sizeof (*newp));
1602 	if (iosvc->msg_q_cur_len == 0) {
1603 		newp->msg_nextp = NULL;
1604 		iosvc->msg_q_tail = newp;
1605 	} else {
1606 		newp->msg_nextp = iosvc->msg_q_head;
1607 	}
1608 	iosvc->msg_q_head = newp;
1609 	iosvc->msg_q_cur_len++;
1610 } /* etm_msg_enq_head() */
1611 
1612 /*
1613  * etm_iosvc_cleanup():
1614  * Clean up an iosvc structure
1615  * 1) close the fmd_xprt if it has not been closed
1616  * 2) Terminate the send/revc threads
1617  * 3) If the clean_msg_q flag is set, free all fma events in the queue. In
1618  *    addition, if the chpt_remove flag is set, delete the checkpoint so that
1619  *    the events are not persisted.
1620  */
1621 static void
etm_iosvc_cleanup(fmd_hdl_t * fmd_hdl,etm_iosvc_t * iosvc,boolean_t clean_msg_q,boolean_t ckpt_remove)1622 etm_iosvc_cleanup(fmd_hdl_t *fmd_hdl, etm_iosvc_t *iosvc, boolean_t clean_msg_q,
1623     boolean_t ckpt_remove)
1624 {
1625 
1626 	etm_iosvc_q_ele_t	msg_ele;	/* io svc msg Q ele */
1627 
1628 	iosvc->thr_is_dying = 1;
1629 
1630 	iosvc->ds_hdl = DS_INVALID_HDL;
1631 	if (iosvc->fmd_xprt != NULL) {
1632 		fmd_xprt_close(fmd_hdl, iosvc->fmd_xprt);
1633 		iosvc->fmd_xprt = NULL;
1634 	} /* if fmd-xprt has been opened */
1635 
1636 	if (iosvc->send_tid != 0) {
1637 		fmd_thr_signal(fmd_hdl, iosvc->send_tid);
1638 		fmd_thr_destroy(fmd_hdl, iosvc->send_tid);
1639 		iosvc->send_tid = 0;
1640 	} /* if io svc send thread was created ok */
1641 
1642 	if (iosvc->recv_tid != 0) {
1643 		fmd_thr_signal(fmd_hdl, iosvc->recv_tid);
1644 		fmd_thr_destroy(fmd_hdl, iosvc->recv_tid);
1645 		iosvc->recv_tid = 0;
1646 	} /* if root domain recv thread was created */
1647 
1648 
1649 	if (clean_msg_q) {
1650 		iosvc->ldom_name[0] = '\0';
1651 
1652 		(void) pthread_mutex_lock(&iosvc->msg_q_lock);
1653 		while (iosvc->msg_q_cur_len > 0) {
1654 			(void) etm_iosvc_msg_deq(fmd_hdl, iosvc, &msg_ele);
1655 			if (ckpt_remove == B_TRUE &&
1656 			    msg_ele.ckpt_flag != ETM_CKPT_NOOP) {
1657 				etm_ckpt_remove(fmd_hdl, &msg_ele);
1658 			}
1659 			fmd_hdl_free(fmd_hdl, msg_ele.msg, msg_ele.msg_size);
1660 		}
1661 		(void) pthread_mutex_unlock(&iosvc->msg_q_lock);
1662 	}
1663 
1664 	return;
1665 
1666 } /* etm_iosvc_cleanup() */
1667 
1668 /*
1669  * etm_iosvc_lookup(using ldom_name or ds_hdl when ldom_name is empty)
1670  * not found, create one, add to iosvc_list
1671  */
1672 etm_iosvc_t *
etm_iosvc_lookup(fmd_hdl_t * fmd_hdl,char * ldom_name,ds_hdl_t ds_hdl,boolean_t iosvc_create)1673 etm_iosvc_lookup(fmd_hdl_t *fmd_hdl, char *ldom_name, ds_hdl_t ds_hdl,
1674     boolean_t iosvc_create)
1675 {
1676 	uint32_t		i;			/* for loop var */
1677 	int32_t			first_empty_slot = -1;	/* remember that */
1678 
1679 	for (i = 0; i < NUM_OF_ROOT_DOMAINS; i++) {
1680 		if (ldom_name[0] == '\0') {
1681 			/*
1682 			 * search by hdl passed in
1683 			 * the only time this is used is at ds_unreg_cb time.
1684 			 * there is no ldom name, only the valid ds_hdl.
1685 			 * find an iosvc with the matching ds_hdl.
1686 			 * ignore the iosvc_create flag, should never need to
1687 			 * create an iosvc for ds_unreg_cb
1688 			 */
1689 			if (ds_hdl == iosvc_list[i].ds_hdl) {
1690 				if (etm_debug_lvl >= 2) {
1691 				fmd_hdl_debug(fmd_hdl,
1692 			    "info: found an iosvc at slot %d w/ ds_hdl %d \n",
1693 				    i, iosvc_list[i].ds_hdl);
1694 				}
1695 				if (iosvc_list[i].ldom_name[0] != '\0')
1696 					if (etm_debug_lvl >= 2) {
1697 						fmd_hdl_debug(fmd_hdl,
1698 				    "info: found an iosvc w/ ldom_name %s \n",
1699 						    iosvc_list[i].ldom_name);
1700 				}
1701 				return (&iosvc_list[i]);
1702 			} else {
1703 				continue;
1704 			}
1705 		} else if (iosvc_list[i].ldom_name[0] != '\0') {
1706 			/*
1707 			 * this is  an non-empty iosvc structure slot
1708 			 */
1709 			if (strcmp(ldom_name, iosvc_list[i].ldom_name) == 0) {
1710 				/*
1711 				 * found an iosvc structure that matches the
1712 				 * passed in ldom_name, return the ptr
1713 				 */
1714 				if (etm_debug_lvl >= 2) {
1715 					fmd_hdl_debug(fmd_hdl, "info: found an "
1716 					    "iosvc at slot %d w/ ds_hdl %d \n",
1717 					    i, iosvc_list[i].ds_hdl);
1718 					fmd_hdl_debug(fmd_hdl, "info: found an "
1719 					    "iosvc w/ ldom_name %s \n",
1720 					    iosvc_list[i].ldom_name);
1721 				}
1722 				return (&iosvc_list[i]);
1723 			} else {
1724 				/*
1725 				 * non-empty slot with no-matching name,
1726 				 * move on to next slot.
1727 				 */
1728 				continue;
1729 			}
1730 		} else {
1731 			/*
1732 			 * found the 1st slot with ldom name being empty
1733 			 * remember the slot #, will be used for creating one
1734 			 */
1735 			if (first_empty_slot == -1) {
1736 				first_empty_slot = i;
1737 			}
1738 		}
1739 	}
1740 	if (iosvc_create == B_TRUE && first_empty_slot >= 0) {
1741 		/*
1742 		 * this is the case we need to add an iosvc at first_empty_slot
1743 		 * for the ldom_name at iosvc_list[first_empty_slot]
1744 		 */
1745 		fmd_hdl_debug(fmd_hdl,
1746 		    "info: create an iosvc with ldom name %s\n",
1747 		    ldom_name);
1748 		i = first_empty_slot;
1749 		(void) memcpy(&iosvc_list[i], &io_svc, sizeof (etm_iosvc_t));
1750 		(void) strcpy(iosvc_list[i].ldom_name, ldom_name);
1751 		fmd_hdl_debug(fmd_hdl, "info: iosvc #%d has ldom name %s\n",
1752 		    i, iosvc_list[i].ldom_name);
1753 		return (&iosvc_list[i]);
1754 	} else {
1755 		return (NULL);
1756 	}
1757 
1758 } /* etm_iosvc_lookup() */
1759 
1760 
1761 /*
1762  * etm_ckpt_remove:
1763  * remove the ckpt for the iosvc element
1764  */
1765 static void
etm_ckpt_remove(fmd_hdl_t * hdl,etm_iosvc_q_ele_t * ele)1766 etm_ckpt_remove(fmd_hdl_t *hdl, etm_iosvc_q_ele_t *ele)
1767 {
1768 	int		err;			/* temp error */
1769 	nvlist_t	*evp = NULL;		/* event pointer */
1770 	etm_proto_v1_ev_hdr_t	*hdrp;		/* hdr for FMA_EVENT */
1771 	char		*buf;			/* packed event pointer */
1772 
1773 	if ((ele->ckpt_flag == ETM_CKPT_NOOP) ||
1774 	    (etm_ldom_type != LDOM_TYPE_CONTROL)) {
1775 		return;
1776 	}
1777 
1778 	/* the pointer to the packed event in the etm message */
1779 	hdrp = (etm_proto_v1_ev_hdr_t *)((ptrdiff_t)ele->msg);
1780 	buf = (char *)((ptrdiff_t)hdrp + sizeof (*hdrp)
1781 	    + (1 * sizeof (hdrp->ev_lens[0])));
1782 
1783 	/* unpack it, then uncheckpoited it */
1784 	if ((err = nvlist_unpack(buf, hdrp->ev_lens[0], &evp, 0)) != 0) {
1785 		fmd_hdl_debug(hdl, "failed to unpack event(rc=%d)\n", err);
1786 		return;
1787 	}
1788 	(void) etm_ckpt_delete(hdl, evp);
1789 	nvlist_free(evp);
1790 }
1791 
1792 /*
1793  * etm_send_ds_msg()
1794  * call ds_send_msg() to send the msg passed in.
1795  * timedcond_wait for the ACK to come back.
1796  * if the ACK doesn't come in the specified time, retrun -EAGAIN.
1797  * other wise, return 1.
1798  */
1799 int
etm_send_ds_msg(fmd_hdl_t * fmd_hdl,boolean_t ckpt_remove,etm_iosvc_t * iosvc,etm_iosvc_q_ele_t * msg_ele,etm_proto_v1_ev_hdr_t * evhdrp)1800 etm_send_ds_msg(fmd_hdl_t *fmd_hdl, boolean_t ckpt_remove, etm_iosvc_t *iosvc,
1801     etm_iosvc_q_ele_t *msg_ele, etm_proto_v1_ev_hdr_t *evhdrp)
1802 {
1803 	uint32_t		rc;		/* for return code  */
1804 
1805 	struct timeval		tv;
1806 	struct timespec		timeout;
1807 
1808 
1809 	/*
1810 	 * call ds_send_msg(). Return (-EAGAIN) if not successful
1811 	 */
1812 	if ((rc = (*etm_ds_send_msg)(iosvc->ds_hdl, msg_ele->msg,
1813 	    msg_ele->msg_size)) != 0) {
1814 		fmd_hdl_debug(fmd_hdl, "info: ds_send_msg rc %d xid %d\n",
1815 		    rc, evhdrp->ev_pp.pp_xid);
1816 			return (-EAGAIN);
1817 	}
1818 
1819 	/*
1820 	 * wait on the cv for resp msg for cur_send_xid
1821 	 */
1822 	(void *) pthread_mutex_lock(&iosvc->msg_ack_lock);
1823 
1824 	(void) gettimeofday(&tv, 0);
1825 	timeout.tv_sec = tv.tv_sec + etm_fma_resp_wait_time;
1826 	timeout.tv_nsec = 0;
1827 
1828 	fmd_hdl_debug(fmd_hdl, "info: waiting on msg_ack_cv for ldom %s\n",
1829 	    iosvc->ldom_name);
1830 	rc = pthread_cond_timedwait(&iosvc->msg_ack_cv, &iosvc->msg_ack_lock,
1831 	    &timeout);
1832 	(void *) pthread_mutex_unlock(&iosvc->msg_ack_lock);
1833 	fmd_hdl_debug(fmd_hdl,  "info: msg_ack_cv returns with rc %d\n", rc);
1834 
1835 	/*
1836 	 * check to see if ack_ok is non-zero
1837 	 * if non-zero, resp msg has been received
1838 	 */
1839 	if (iosvc->ack_ok != 0) {
1840 		/*
1841 		 * ACK came ok,  this send is successful,
1842 		 * tell the caller ready to send next.
1843 		 * free mem alloc-ed in
1844 		 * etm_pack_ds_msg
1845 		 */
1846 		if (ckpt_remove == B_TRUE &&
1847 		    etm_ldom_type == LDOM_TYPE_CONTROL) {
1848 			etm_ckpt_remove(fmd_hdl, msg_ele);
1849 		}
1850 		fmd_hdl_free(fmd_hdl, msg_ele->msg, msg_ele->msg_size);
1851 		iosvc->cur_send_xid++;
1852 		return (1);
1853 	} else {
1854 		/*
1855 		 * the ACK did not come on time
1856 		 * tell the caller to resend cur_send_xid
1857 		 */
1858 		return (-EAGAIN);
1859 	} /* iosvc->ack_ok != 0 */
1860 } /* etm_send_ds_msg() */
1861 
1862 /*
1863  * both events from fmdo_send entry point and from SP are using the
1864  * etm_proto_v1_ev_hdr_t as its header and it will be the same header for all
1865  * ds send/recv msgs.
1866  * Idealy, we should use the hdr coming with the SP FMA event. Since fmdo_send
1867  * entry point can be called before FMA events from SP, we can't rely on
1868  * the SP FMA event hdr. Use the static hdr for packing ds msgs for fmdo_send
1869  * events.
1870  * return >0 for success, or -errno value
1871  * Design assumption: there is one FMA event per ds msg
1872  */
1873 int
etm_pack_ds_msg(fmd_hdl_t * fmd_hdl,etm_iosvc_t * iosvc,etm_proto_v1_ev_hdr_t * ev_hdrp,size_t hdr_sz,nvlist_t * evp,etm_pack_msg_type_t msg_type,uint_t ckpt_opt)1874 etm_pack_ds_msg(fmd_hdl_t *fmd_hdl, etm_iosvc_t *iosvc,
1875     etm_proto_v1_ev_hdr_t *ev_hdrp, size_t hdr_sz, nvlist_t *evp,
1876     etm_pack_msg_type_t msg_type, uint_t ckpt_opt)
1877 {
1878 	etm_proto_v1_ev_hdr_t	*hdrp;		/* for FMA_EVENT msg */
1879 	uint32_t		*lenp;		/* ptr to FMA event length */
1880 	size_t			evsz;		/* packed FMA event size */
1881 	char			*buf;
1882 	uint32_t		rc;		/* for return code  */
1883 	char			*msg;		/* body of msg to be Qed */
1884 
1885 	etm_iosvc_q_ele_t	msg_ele;	/* io svc msg Q ele */
1886 	etm_proto_v1_ev_hdr_t	*evhdrp;
1887 
1888 
1889 	if (ev_hdrp == NULL) {
1890 		hdrp = &iosvc_hdr;
1891 	} else {
1892 		hdrp = ev_hdrp;
1893 	}
1894 
1895 	/*
1896 	 * determine hdr_sz if 0, otherwise use the one passed in hdr_sz
1897 	 */
1898 
1899 	if (hdr_sz == 0) {
1900 		hdr_sz = sizeof (*hdrp) + (1 * sizeof (hdrp->ev_lens[0]));
1901 	}
1902 
1903 	/*
1904 	 * determine evp size
1905 	 */
1906 	(void) nvlist_size(evp, &evsz, NV_ENCODE_XDR);
1907 
1908 	/* indicate 1 FMA event, no network encoding, and 0-terminate */
1909 	lenp = &hdrp->ev_lens[0];
1910 	*lenp = evsz;
1911 
1912 	/*
1913 	 * now the total of mem needs to be alloc-ed/ds msg size is
1914 	 * hdr_sz + evsz
1915 	 * msg will be freed in etm_send_to_remote_root() after ds_send_msg()
1916 	 */
1917 	msg = fmd_hdl_zalloc(fmd_hdl, hdr_sz + evsz, FMD_SLEEP);
1918 
1919 
1920 	/*
1921 	 * copy hdr, 0 terminate the length vector,  and then evp
1922 	 */
1923 	(void) memcpy(msg, hdrp, sizeof (*hdrp));
1924 	hdrp = (etm_proto_v1_ev_hdr_t *)((ptrdiff_t)msg);
1925 	lenp = &hdrp->ev_lens[0];
1926 	lenp++;
1927 	*lenp = 0;
1928 
1929 	buf = fmd_hdl_zalloc(fmd_hdl, evsz, FMD_SLEEP);
1930 	(void) nvlist_pack(evp, (char **)&buf, &evsz, NV_ENCODE_XDR, 0);
1931 	(void) memcpy(msg + hdr_sz, buf, evsz);
1932 	fmd_hdl_free(fmd_hdl, buf, evsz);
1933 
1934 	fmd_hdl_debug(fmd_hdl, "info: hdr_sz= %d evsz= %d in etm_pack_ds_msg"
1935 	    "for ldom %s\n", hdr_sz, evsz, iosvc->ldom_name);
1936 	msg_ele.msg = msg;
1937 	msg_ele.msg_size = hdr_sz + evsz;
1938 	msg_ele.ckpt_flag = ckpt_opt;
1939 
1940 	/*
1941 	 * decide what to do with the msg:
1942 	 * if SP ereports (msg_type == SP_MSG), always enq the msg
1943 	 * if not SP ereports, ie, fmd xprt control msgs, enq it _only_ after
1944 	 * resource.fm.xprt.run has been sent (which sets start_sending_Q to 1)
1945 	 */
1946 	if ((msg_type == SP_MSG) ||
1947 	    (msg_type != SP_MSG) && (iosvc->start_sending_Q == 1)) {
1948 		/*
1949 		 * this is the case when the msg needs to be enq-ed
1950 		 */
1951 		(void) pthread_mutex_lock(&iosvc->msg_q_lock);
1952 		rc = etm_iosvc_msg_enq(fmd_hdl, iosvc, &msg_ele);
1953 		if ((rc > 0) && (ckpt_opt & ETM_CKPT_SAVE) &&
1954 		    (etm_ldom_type == LDOM_TYPE_CONTROL)) {
1955 			(void) etm_ckpt_add(fmd_hdl, evp);
1956 		}
1957 		if (iosvc->msg_q_cur_len == 1)
1958 			(void) pthread_cond_signal(&iosvc->msg_q_cv);
1959 		(void) pthread_mutex_unlock(&iosvc->msg_q_lock);
1960 	} else {
1961 		/*
1962 		 * fmd RDWR xprt procotol startup msgs, send it now!
1963 		 */
1964 		iosvc->ack_ok = 0;
1965 		evhdrp = (etm_proto_v1_ev_hdr_t *)((ptrdiff_t)msg_ele.msg);
1966 		evhdrp->ev_pp.pp_xid = iosvc->cur_send_xid + 1;
1967 		while (!iosvc->ack_ok && iosvc->ds_hdl != DS_INVALID_HDL &&
1968 		    !etm_is_dying) {
1969 			if (etm_send_ds_msg(fmd_hdl, B_FALSE, iosvc, &msg_ele,
1970 			    evhdrp) < 0) {
1971 				continue;
1972 			}
1973 		}
1974 		if (msg_type == FMD_XPRT_RUN_MSG)
1975 			iosvc->start_sending_Q = 1;
1976 	}
1977 
1978 	return (rc);
1979 
1980 } /* etm_pack_ds_msg() */
1981 
1982 /*
1983  * Design_Note:	For all etm_resp_q_*() functions and etm_resp_q_* globals,
1984  *		the mutex etm_resp_q_lock must be held by the caller.
1985  */
1986 
1987 /*
1988  * etm_resp_q_enq - add element to tail of ETM responder queue
1989  * etm_resp_q_deq - del element from head of ETM responder queue
1990  *
1991  * return >0 for success, or -errno value
1992  */
1993 
1994 static int
etm_resp_q_enq(fmd_hdl_t * hdl,etm_resp_q_ele_t * rqep)1995 etm_resp_q_enq(fmd_hdl_t *hdl, etm_resp_q_ele_t *rqep)
1996 {
1997 	etm_resp_q_ele_t	*newp;	/* ptr to new resp q ele */
1998 
1999 	if (etm_resp_q_cur_len >= etm_resp_q_max_len) {
2000 		fmd_hdl_debug(hdl, "warning: enq to full responder queue\n");
2001 		etm_stats.etm_enq_drop_resp_q.fmds_value.ui64++;
2002 		return (-E2BIG);
2003 	}
2004 
2005 	newp = fmd_hdl_zalloc(hdl, sizeof (*newp), FMD_SLEEP);
2006 	(void) memcpy(newp, rqep, sizeof (*newp));
2007 	newp->rqe_nextp = NULL;
2008 
2009 	if (etm_resp_q_cur_len == 0) {
2010 		etm_resp_q_head = newp;
2011 	} else {
2012 		etm_resp_q_tail->rqe_nextp = newp;
2013 	}
2014 	etm_resp_q_tail = newp;
2015 	etm_resp_q_cur_len++;
2016 	etm_stats.etm_resp_q_cur_len.fmds_value.ui64 = etm_resp_q_cur_len;
2017 
2018 	return (1);
2019 
2020 } /* etm_resp_q_enq() */
2021 
2022 static int
etm_resp_q_deq(fmd_hdl_t * hdl,etm_resp_q_ele_t * rqep)2023 etm_resp_q_deq(fmd_hdl_t *hdl, etm_resp_q_ele_t *rqep)
2024 {
2025 	etm_resp_q_ele_t	*oldp;	/* ptr to old resp q ele */
2026 
2027 	if (etm_resp_q_cur_len == 0) {
2028 		fmd_hdl_debug(hdl, "warning: deq from empty responder queue\n");
2029 		etm_stats.etm_deq_drop_resp_q.fmds_value.ui64++;
2030 		return (-ENOENT);
2031 	}
2032 
2033 	(void) memcpy(rqep, etm_resp_q_head, sizeof (*rqep));
2034 	rqep->rqe_nextp = NULL;
2035 
2036 	oldp = etm_resp_q_head;
2037 	etm_resp_q_head = etm_resp_q_head->rqe_nextp;
2038 	fmd_hdl_free(hdl, oldp, sizeof (*oldp));
2039 
2040 	etm_resp_q_cur_len--;
2041 	etm_stats.etm_resp_q_cur_len.fmds_value.ui64 = etm_resp_q_cur_len;
2042 	if (etm_resp_q_cur_len == 0) {
2043 		etm_resp_q_tail = NULL;
2044 	}
2045 
2046 	return (1);
2047 
2048 } /* etm_resp_q_deq() */
2049 
2050 /*
2051  * etm_maybe_enq_response - check the given message header to see
2052  *				whether a response has been requested,
2053  *				if so then enqueue the given connection
2054  *				and header for later transport by the
2055  *				responder thread as an ETM response msg,
2056  *				return 0 for nop, >0 success, or -errno value
2057  */
2058 
2059 static ssize_t
etm_maybe_enq_response(fmd_hdl_t * hdl,etm_xport_conn_t conn,void * hdrp,uint32_t hdr_sz,int32_t resp_code)2060 etm_maybe_enq_response(fmd_hdl_t *hdl, etm_xport_conn_t conn,
2061     void *hdrp, uint32_t hdr_sz, int32_t resp_code)
2062 {
2063 	ssize_t			rv;		/* ret val */
2064 	etm_proto_v1_pp_t	*ppp;		/* protocol preamble ptr */
2065 	uint8_t			orig_msg_type;	/* orig hdr's message type */
2066 	uint32_t		orig_timeout;	/* orig hdr's timeout */
2067 	etm_resp_q_ele_t	rqe;		/* responder queue ele */
2068 
2069 	ppp = hdrp;
2070 	orig_msg_type = ppp->pp_msg_type;
2071 	orig_timeout = ppp->pp_timeout;
2072 
2073 	/* bail out now if no response is to be sent */
2074 
2075 	if (orig_timeout == ETM_PROTO_V1_TIMEOUT_NONE) {
2076 		return (0);
2077 	} /* if a nop */
2078 
2079 	if ((orig_msg_type != ETM_MSG_TYPE_FMA_EVENT) &&
2080 	    (orig_msg_type != ETM_MSG_TYPE_ALERT) &&
2081 	    (orig_msg_type != ETM_MSG_TYPE_CONTROL)) {
2082 		fmd_hdl_debug(hdl, "warning: bad msg type 0x%x\n",
2083 		    orig_msg_type);
2084 		return (-EINVAL);
2085 	} /* if inappropriate hdr for a response msg */
2086 
2087 	/*
2088 	 * enqueue the msg hdr and nudge the responder thread
2089 	 * if the responder queue was previously empty
2090 	 */
2091 
2092 	rqe.rqe_conn = conn;
2093 	rqe.rqe_hdrp = hdrp;
2094 	rqe.rqe_hdr_sz = hdr_sz;
2095 	rqe.rqe_resp_code = resp_code;
2096 
2097 	(void) pthread_mutex_lock(&etm_resp_q_lock);
2098 
2099 	if (etm_resp_q_cur_len == etm_resp_q_max_len)
2100 		(void) pthread_cond_wait(&etm_resp_q_cv, &etm_resp_q_lock);
2101 
2102 	rv = etm_resp_q_enq(hdl, &rqe);
2103 	if (etm_resp_q_cur_len == 1)
2104 		(void) pthread_cond_signal(&etm_resp_q_cv);
2105 	(void) pthread_mutex_unlock(&etm_resp_q_lock);
2106 
2107 	return (rv);
2108 
2109 } /* etm_maybe_enq_response() */
2110 
2111 /*
2112  * Design_Note:	We rely on the fact that all message types have
2113  *		a common protocol preamble; if this fact should
2114  *		ever change it may break the code below. We also
2115  *		rely on the fact that FMA_EVENT and CONTROL headers
2116  *		returned by etm_hdr_read() will be sized large enough
2117  *		to reuse them as RESPONSE headers if the remote endpt
2118  *		asked for a response via the pp_timeout field.
2119  */
2120 
2121 /*
2122  * etm_send_response - use the given message header and response code
2123  *			to construct an appropriate response message,
2124  *			and send it back on the given connection,
2125  *			return >0 for success, or -errno value
2126  */
2127 
2128 static ssize_t
etm_send_response(fmd_hdl_t * hdl,etm_xport_conn_t conn,void * hdrp,int32_t resp_code)2129 etm_send_response(fmd_hdl_t *hdl, etm_xport_conn_t conn,
2130     void *hdrp, int32_t resp_code)
2131 {
2132 	ssize_t			rv;		/* ret val */
2133 	etm_proto_v1_pp_t	*ppp;		/* protocol preamble ptr */
2134 	etm_proto_v1_resp_hdr_t *resp_hdrp;	/* for RESPONSE msg */
2135 	uint8_t			resp_body[4];	/* response body if needed */
2136 	uint8_t			*resp_msg;	/* response hdr+body */
2137 	size_t			hdr_sz;		/* sizeof response hdr */
2138 	uint8_t			orig_msg_type;	/* orig hdr's message type */
2139 
2140 	ppp = hdrp;
2141 	orig_msg_type = ppp->pp_msg_type;
2142 
2143 	if (etm_debug_lvl >= 2) {
2144 		etm_show_time(hdl, "ante resp send");
2145 	}
2146 
2147 	/* reuse the given header as a response header */
2148 
2149 	resp_hdrp = hdrp;
2150 	resp_hdrp->resp_code = resp_code;
2151 	resp_hdrp->resp_len = 0;		/* default is empty body */
2152 
2153 	if ((orig_msg_type == ETM_MSG_TYPE_CONTROL) &&
2154 	    (ppp->pp_sub_type == ETM_CTL_SEL_VER_NEGOT_REQ)) {
2155 		resp_body[0] = ETM_PROTO_V2;
2156 		resp_body[1] = ETM_PROTO_V3;
2157 		resp_body[2] = 0;
2158 		resp_hdrp->resp_len = 3;
2159 	} /* if should send our/negotiated proto ver in resp body */
2160 
2161 	/* respond with the proto ver that was negotiated */
2162 
2163 	resp_hdrp->resp_pp.pp_proto_ver = etm_resp_ver;
2164 	resp_hdrp->resp_pp.pp_msg_type = ETM_MSG_TYPE_RESPONSE;
2165 	resp_hdrp->resp_pp.pp_timeout = ETM_PROTO_V1_TIMEOUT_NONE;
2166 
2167 	/*
2168 	 * send the whole response msg in one write, header and body;
2169 	 * avoid the alloc-and-copy if we can reuse the hdr as the msg,
2170 	 * ie, if the body is empty. update the response stats.
2171 	 */
2172 
2173 	hdr_sz = sizeof (etm_proto_v1_resp_hdr_t);
2174 
2175 	resp_msg = hdrp;
2176 	if (resp_hdrp->resp_len > 0) {
2177 		resp_msg = fmd_hdl_zalloc(hdl, hdr_sz + resp_hdrp->resp_len,
2178 		    FMD_SLEEP);
2179 		(void) memcpy(resp_msg, resp_hdrp, hdr_sz);
2180 		(void) memcpy(resp_msg + hdr_sz, resp_body,
2181 		    resp_hdrp->resp_len);
2182 	}
2183 
2184 	(void) pthread_mutex_lock(&etm_write_lock);
2185 	rv = etm_io_op(hdl, "bad io write on resp msg", conn,
2186 	    resp_msg, hdr_sz + resp_hdrp->resp_len, ETM_IO_OP_WR);
2187 	(void) pthread_mutex_unlock(&etm_write_lock);
2188 	if (rv < 0) {
2189 		goto func_ret;
2190 	}
2191 
2192 	etm_stats.etm_wr_hdr_response.fmds_value.ui64++;
2193 	etm_stats.etm_wr_body_response.fmds_value.ui64++;
2194 
2195 	fmd_hdl_debug(hdl, "info: sent V%u RESPONSE msg to xport "
2196 	    "xid 0x%x code %d len %u\n",
2197 	    (unsigned int)resp_hdrp->resp_pp.pp_proto_ver,
2198 	    resp_hdrp->resp_pp.pp_xid, resp_hdrp->resp_code,
2199 	    resp_hdrp->resp_len);
2200 func_ret:
2201 
2202 	if (resp_hdrp->resp_len > 0) {
2203 		fmd_hdl_free(hdl, resp_msg, hdr_sz + resp_hdrp->resp_len);
2204 	}
2205 	if (etm_debug_lvl >= 2) {
2206 		etm_show_time(hdl, "post resp send");
2207 	}
2208 	return (rv);
2209 
2210 } /* etm_send_response() */
2211 
2212 /*
2213  * etm_reset_xport - reset the transport layer (via fini;init)
2214  *			presumably for an error condition we cannot
2215  *			otherwise recover from (ex: hung LDC channel)
2216  *
2217  * caveats - no checking/locking is done to ensure an existing connection
2218  *		is idle during an xport reset; we don't want to deadlock
2219  *		and presumably the transport is stuck/unusable anyway
2220  */
2221 
2222 static void
etm_reset_xport(fmd_hdl_t * hdl)2223 etm_reset_xport(fmd_hdl_t *hdl)
2224 {
2225 	(void) etm_xport_fini(hdl);
2226 	(void) etm_xport_init(hdl);
2227 	etm_stats.etm_reset_xport.fmds_value.ui64++;
2228 
2229 } /* etm_reset_xport() */
2230 
2231 /*
2232  * etm_handle_new_conn - receive an ETM message sent from the other end via
2233  *			the given open connection, pull out any FMA events
2234  *			and post them to the local FMD (or handle any ETM
2235  *			control or response msg); when done, close the
2236  *			connection
2237  */
2238 
2239 static void
etm_handle_new_conn(fmd_hdl_t * hdl,etm_xport_conn_t conn)2240 etm_handle_new_conn(fmd_hdl_t *hdl, etm_xport_conn_t conn)
2241 {
2242 	etm_proto_v1_ev_hdr_t	*ev_hdrp;	/* for FMA_EVENT msg */
2243 	etm_proto_v1_ctl_hdr_t	*ctl_hdrp;	/* for CONTROL msg */
2244 	etm_proto_v1_resp_hdr_t *resp_hdrp;	/* for RESPONSE msg */
2245 	etm_proto_v3_sa_hdr_t	*sa_hdrp;	/* for ALERT msg */
2246 	etm_iosvc_t		*iosvc;		/* iosvc data structure */
2247 	int32_t			resp_code;	/* response code */
2248 	ssize_t			enq_rv;		/* resp_q enqueue status */
2249 	size_t			hdr_sz;		/* sizeof header */
2250 	size_t			evsz;		/* FMA event size */
2251 	uint8_t			*body_buf;	/* msg body buffer */
2252 	uint32_t		body_sz;	/* sizeof body_buf */
2253 	uint32_t		ev_cnt;		/* count of FMA events */
2254 	uint8_t			*bp;		/* byte ptr within body_buf */
2255 	nvlist_t		*evp;		/* ptr to unpacked FMA event */
2256 	char			*class;		/* FMA event class */
2257 	ssize_t			i, n;		/* gen use */
2258 	int			should_reset_xport; /* bool to reset xport */
2259 	char			ldom_name[MAX_LDOM_NAME]; /* ldom name */
2260 	int			rc;		/* return code */
2261 	uint64_t		did;		/* domain id */
2262 
2263 
2264 	if (etm_debug_lvl >= 2) {
2265 		etm_show_time(hdl, "ante conn handle");
2266 	}
2267 	fmd_hdl_debug(hdl, "info: handling new conn %p\n", conn);
2268 
2269 	should_reset_xport = 0;
2270 	ev_hdrp = NULL;
2271 	ctl_hdrp = NULL;
2272 	resp_hdrp = NULL;
2273 	sa_hdrp = NULL;
2274 	body_buf = NULL;
2275 	class = NULL;
2276 	evp = NULL;
2277 	resp_code = 0;	/* default is success */
2278 	enq_rv = 0;	/* default is nop, ie, did not enqueue */
2279 
2280 	/* read a network decoded message header from the connection */
2281 
2282 	if ((ev_hdrp = etm_hdr_read(hdl, conn, &hdr_sz)) == NULL) {
2283 		/* errno assumed set by above call */
2284 		should_reset_xport = (errno == ENOTACTIVE);
2285 		fmd_hdl_debug(hdl, "error: FMA event dropped: "
2286 		    "bad hdr read errno %d\n", errno);
2287 		etm_stats.etm_rd_drop_fmaevent.fmds_value.ui64++;
2288 		goto func_ret;
2289 	}
2290 
2291 	/*
2292 	 * handle the message based on its preamble pp_msg_type
2293 	 * which is known to be valid from etm_hdr_read() checks
2294 	 */
2295 
2296 	if (ev_hdrp->ev_pp.pp_msg_type == ETM_MSG_TYPE_FMA_EVENT) {
2297 
2298 		fmd_hdl_debug(hdl, "info: rcvd FMA_EVENT msg from xport\n");
2299 
2300 		/* allocate buf large enough for whole body / all FMA events */
2301 
2302 		body_sz = 0;
2303 		for (i = 0; ev_hdrp->ev_lens[i] != 0; i++) {
2304 			body_sz += ev_hdrp->ev_lens[i];
2305 		} /* for summing sizes of all FMA events */
2306 		if (i > etm_stats.etm_rd_max_ev_per_msg.fmds_value.ui64)
2307 			etm_stats.etm_rd_max_ev_per_msg.fmds_value.ui64 = i;
2308 		ev_cnt = i;
2309 
2310 		if (etm_debug_lvl >= 1) {
2311 			fmd_hdl_debug(hdl, "info: event lengths %u sum %u\n",
2312 			    ev_cnt, body_sz);
2313 		}
2314 
2315 		body_buf = fmd_hdl_zalloc(hdl, body_sz, FMD_SLEEP);
2316 
2317 		/* read all the FMA events at once */
2318 
2319 		if ((n = etm_io_op(hdl, "FMA event dropped: "
2320 		    "bad io read on event bodies", conn, body_buf, body_sz,
2321 		    ETM_IO_OP_RD)) < 0) {
2322 			should_reset_xport = (n == -ENOTACTIVE);
2323 			etm_stats.etm_rd_drop_fmaevent.fmds_value.ui64++;
2324 			goto func_ret;
2325 		}
2326 
2327 		etm_stats.etm_rd_xport_bytes.fmds_value.ui64 += body_sz;
2328 		etm_stats.etm_rd_body_fmaevent.fmds_value.ui64 += ev_cnt;
2329 
2330 		/*
2331 		 * now that we've read the entire ETM msg from the conn,
2332 		 * which avoids later ETM protocol framing errors if we didn't,
2333 		 * check for dup msg/xid against last good FMD posting,
2334 		 * if a dup then resend response but skip repost to FMD
2335 		 */
2336 
2337 		if (ev_hdrp->ev_pp.pp_xid == etm_xid_posted_logged_ev) {
2338 			enq_rv = etm_maybe_enq_response(hdl, conn,
2339 			    ev_hdrp, hdr_sz, 0);
2340 			fmd_hdl_debug(hdl, "info: skipping dup FMA event post "
2341 			    "xid 0x%x\n", etm_xid_posted_logged_ev);
2342 			etm_stats.etm_rd_dup_fmaevent.fmds_value.ui64++;
2343 			goto func_ret;
2344 		}
2345 
2346 		/* unpack each FMA event and post it to FMD */
2347 
2348 		bp = body_buf;
2349 		for (i = 0; i < ev_cnt; i++) {
2350 			if ((n = nvlist_unpack((char *)bp,
2351 			    ev_hdrp->ev_lens[i], &evp, 0)) != 0) {
2352 				resp_code = (-n);
2353 				enq_rv = etm_maybe_enq_response(hdl, conn,
2354 				    ev_hdrp, hdr_sz, resp_code);
2355 				fmd_hdl_error(hdl, "error: FMA event dropped: "
2356 				    "bad event body unpack errno %d\n", n);
2357 				if (etm_debug_lvl >= 2) {
2358 					fmd_hdl_debug(hdl, "info: FMA event "
2359 					    "hexdump %d bytes:\n",
2360 					    ev_hdrp->ev_lens[i]);
2361 					etm_hexdump(hdl, bp,
2362 					    ev_hdrp->ev_lens[i]);
2363 				}
2364 				etm_stats.etm_os_nvlist_unpack_fail.fmds_value.
2365 				    ui64++;
2366 				etm_stats.etm_rd_drop_fmaevent.fmds_value.
2367 				    ui64++;
2368 				bp += ev_hdrp->ev_lens[i];
2369 				continue;
2370 			}
2371 
2372 			if (etm_debug_lvl >= 1) {
2373 				(void) nvlist_lookup_string(evp, FM_CLASS,
2374 				    &class);
2375 				if (class == NULL) {
2376 					class = "NULL";
2377 				}
2378 				fmd_hdl_debug(hdl, "info: FMA event %p "
2379 				    "class %s\n", evp, class);
2380 			}
2381 
2382 			rc = nvlist_size(evp, &evsz, NV_ENCODE_XDR);
2383 			fmd_hdl_debug(hdl,
2384 			    "info: evp size before pack ds msg %d\n", evsz);
2385 			ldom_name[0] = '\0';
2386 			rc = etm_filter_find_ldom_id(hdl, evp, ldom_name,
2387 			    MAX_LDOM_NAME, &did);
2388 
2389 			/*
2390 			 * if rc is zero and the ldom_name is not "primary",
2391 			 * the evp belongs to a root domain, put the evp in an
2392 			 * outgoing etm queue,
2393 			 * in all other cases, whether ldom_name is primary or
2394 			 * can't find a ldom name, call etm_post_to_fmd
2395 			 */
2396 			if ((rc == 0) && strcmp(ldom_name, "primary") &&
2397 			    strcmp(ldom_name, "")) {
2398 				/*
2399 				 * use the ldom_name, guaranteered at this point
2400 				 * to be a valid ldom name/non-NULL, to find the
2401 				 * iosvc data.
2402 				 * add an iosvc struct if can not find one
2403 				 */
2404 				(void) pthread_mutex_unlock(&iosvc_list_lock);
2405 				iosvc = etm_iosvc_lookup(hdl, ldom_name,
2406 				    DS_INVALID_HDL, B_TRUE);
2407 				(void) pthread_mutex_unlock(&iosvc_list_lock);
2408 				if (iosvc == NULL) {
2409 					fmd_hdl_debug(hdl,
2410 					    "error: can't find iosvc for ldom "
2411 					    "name %s\n", ldom_name);
2412 				} else {
2413 					resp_code = 0;
2414 					(void) etm_pack_ds_msg(hdl, iosvc,
2415 					    ev_hdrp, hdr_sz, evp,
2416 					    SP_MSG, ETM_CKPT_SAVE);
2417 					/*
2418 					 * call the new fmd_xprt_log()
2419 					 */
2420 					fmd_xprt_log(hdl, etm_fmd_xprt, evp, 0);
2421 					etm_xid_posted_logged_ev =
2422 					    ev_hdrp->ev_pp.pp_xid;
2423 				}
2424 			} else {
2425 				/*
2426 				 * post the fma event to the control fmd
2427 				 */
2428 				resp_code = etm_post_to_fmd(hdl, etm_fmd_xprt,
2429 				    evp);
2430 				if (resp_code >= 0) {
2431 					etm_xid_posted_logged_ev =
2432 					    ev_hdrp->ev_pp.pp_xid;
2433 				}
2434 			}
2435 
2436 			evp = NULL;
2437 			enq_rv = etm_maybe_enq_response(hdl, conn,
2438 			    ev_hdrp, hdr_sz, resp_code);
2439 			bp += ev_hdrp->ev_lens[i];
2440 		} /* foreach FMA event in the body buffer */
2441 
2442 	} else if (ev_hdrp->ev_pp.pp_msg_type == ETM_MSG_TYPE_CONTROL) {
2443 
2444 		ctl_hdrp = (void*)ev_hdrp;
2445 
2446 		fmd_hdl_debug(hdl, "info: rcvd CONTROL msg from xport\n");
2447 		if (etm_debug_lvl >= 1) {
2448 			fmd_hdl_debug(hdl, "info: ctl sel %d xid 0x%x\n",
2449 			    (int)ctl_hdrp->ctl_pp.pp_sub_type,
2450 			    ctl_hdrp->ctl_pp.pp_xid);
2451 		}
2452 
2453 		/*
2454 		 * if we have a VER_NEGOT_REQ read the body and validate
2455 		 * the protocol version set contained therein,
2456 		 * otherwise we have a PING_REQ (which has no body)
2457 		 * and we [also] fall thru to the code which sends a
2458 		 * response msg if the pp_timeout field requested one
2459 		 */
2460 
2461 		if (ctl_hdrp->ctl_pp.pp_sub_type == ETM_CTL_SEL_VER_NEGOT_REQ) {
2462 
2463 			body_sz = ctl_hdrp->ctl_len;
2464 			body_buf = fmd_hdl_zalloc(hdl, body_sz, FMD_SLEEP);
2465 
2466 			if ((n = etm_io_op(hdl, "bad io read on ctl body",
2467 			    conn, body_buf, body_sz, ETM_IO_OP_RD)) < 0) {
2468 				should_reset_xport = (n == -ENOTACTIVE);
2469 				goto func_ret;
2470 			}
2471 
2472 			/* complain if version set completely incompatible */
2473 
2474 			for (i = 0; i < body_sz; i++) {
2475 				if ((body_buf[i] == ETM_PROTO_V1) ||
2476 				    (body_buf[i] == ETM_PROTO_V2) ||
2477 				    (body_buf[i] == ETM_PROTO_V3)) {
2478 					break;
2479 				}
2480 			}
2481 			if (i >= body_sz) {
2482 				etm_stats.etm_ver_bad.fmds_value.ui64++;
2483 				resp_code = (-EPROTO);
2484 			}
2485 
2486 		} /* if got version set request */
2487 
2488 		etm_stats.etm_rd_body_control.fmds_value.ui64++;
2489 
2490 		enq_rv = etm_maybe_enq_response(hdl, conn,
2491 		    ctl_hdrp, hdr_sz, resp_code);
2492 
2493 	} else if (ev_hdrp->ev_pp.pp_msg_type == ETM_MSG_TYPE_RESPONSE) {
2494 
2495 		resp_hdrp = (void*)ev_hdrp;
2496 
2497 		fmd_hdl_debug(hdl, "info: rcvd RESPONSE msg from xport\n");
2498 		if (etm_debug_lvl >= 1) {
2499 			fmd_hdl_debug(hdl, "info: resp xid 0x%x\n",
2500 			    (int)resp_hdrp->resp_pp.pp_xid);
2501 		}
2502 
2503 		body_sz = resp_hdrp->resp_len;
2504 		body_buf = fmd_hdl_zalloc(hdl, body_sz, FMD_SLEEP);
2505 
2506 		if ((n = etm_io_op(hdl, "bad io read on resp len",
2507 		    conn, body_buf, body_sz, ETM_IO_OP_RD)) < 0) {
2508 			should_reset_xport = (n == -ENOTACTIVE);
2509 			goto func_ret;
2510 		}
2511 
2512 		etm_stats.etm_rd_body_response.fmds_value.ui64++;
2513 
2514 		/*
2515 		 * look up the xid to interpret the response body
2516 		 *
2517 		 * ping is a nop; for ver negot confirm that a supported
2518 		 * protocol version was negotiated and remember which one
2519 		 */
2520 
2521 		if ((resp_hdrp->resp_pp.pp_xid != etm_xid_ping) &&
2522 		    (resp_hdrp->resp_pp.pp_xid != etm_xid_ver_negot)) {
2523 			etm_stats.etm_xid_bad.fmds_value.ui64++;
2524 			goto func_ret;
2525 		}
2526 
2527 		if (resp_hdrp->resp_pp.pp_xid == etm_xid_ver_negot) {
2528 			if ((body_buf[0] < ETM_PROTO_V1) ||
2529 			    (body_buf[0] > ETM_PROTO_V3)) {
2530 				etm_stats.etm_ver_bad.fmds_value.ui64++;
2531 				goto func_ret;
2532 			}
2533 			etm_resp_ver = body_buf[0];
2534 		} /* if have resp to last req to negotiate proto ver */
2535 
2536 	} else if (ev_hdrp->ev_pp.pp_msg_type == ETM_MSG_TYPE_ALERT) {
2537 
2538 		sa_hdrp = (void*)ev_hdrp;
2539 
2540 		fmd_hdl_debug(hdl, "info: rcvd ALERT msg from xport\n");
2541 		if (etm_debug_lvl >= 1) {
2542 			fmd_hdl_debug(hdl, "info: sa sel %d xid 0x%x\n",
2543 			    (int)sa_hdrp->sa_pp.pp_sub_type,
2544 			    sa_hdrp->sa_pp.pp_xid);
2545 		}
2546 
2547 		body_sz = sa_hdrp->sa_len;
2548 		body_buf = fmd_hdl_zalloc(hdl, body_sz, FMD_SLEEP);
2549 
2550 		if ((n = etm_io_op(hdl, "bad io read on sa body",
2551 		    conn, body_buf, body_sz, ETM_IO_OP_RD)) < 0) {
2552 			should_reset_xport = (n == -ENOTACTIVE);
2553 			goto func_ret;
2554 		}
2555 
2556 		etm_stats.etm_rd_body_alert.fmds_value.ui64++;
2557 
2558 		/*
2559 		 * now that we've read the entire ETM msg from the conn,
2560 		 * which avoids later ETM protocol framing errors if we didn't,
2561 		 * check for dup msg/xid against last good syslog posting,
2562 		 * if a dup then resend response but skip repost to syslog
2563 		 */
2564 
2565 		if (sa_hdrp->sa_pp.pp_xid == etm_xid_posted_sa) {
2566 			enq_rv = etm_maybe_enq_response(hdl, conn,
2567 			    sa_hdrp, hdr_sz, 0);
2568 			fmd_hdl_debug(hdl, "info: skipping dup ALERT post "
2569 			    "xid 0x%x\n", etm_xid_posted_sa);
2570 			etm_stats.etm_rd_dup_alert.fmds_value.ui64++;
2571 			goto func_ret;
2572 		}
2573 
2574 		resp_code = etm_post_to_syslog(hdl, sa_hdrp->sa_priority,
2575 		    body_sz, body_buf);
2576 		if (resp_code >= 0) {
2577 			etm_xid_posted_sa = sa_hdrp->sa_pp.pp_xid;
2578 		}
2579 		enq_rv = etm_maybe_enq_response(hdl, conn,
2580 		    sa_hdrp, hdr_sz, resp_code);
2581 	} /* whether we have a FMA_EVENT, CONTROL, RESPONSE or ALERT msg */
2582 
2583 func_ret:
2584 
2585 	if (etm_debug_lvl >= 2) {
2586 		etm_show_time(hdl, "post conn handle");
2587 	}
2588 
2589 	/*
2590 	 * if no responder ele was enqueued, close the conn now
2591 	 * and free the ETM msg hdr; the ETM msg body is not needed
2592 	 * by the responder thread and should always be freed here
2593 	 */
2594 
2595 	if (enq_rv <= 0) {
2596 		(void) etm_conn_close(hdl, "bad conn close after msg recv",
2597 		    conn);
2598 		if (ev_hdrp != NULL) {
2599 			fmd_hdl_free(hdl, ev_hdrp, hdr_sz);
2600 		}
2601 	}
2602 	if (body_buf != NULL) {
2603 		fmd_hdl_free(hdl, body_buf, body_sz);
2604 	}
2605 	if (should_reset_xport) {
2606 		etm_reset_xport(hdl);
2607 	}
2608 } /* etm_handle_new_conn() */
2609 
2610 /*
2611  * etm_handle_bad_accept - recover from a failed connection acceptance
2612  */
2613 
2614 static void
etm_handle_bad_accept(fmd_hdl_t * hdl,int nev)2615 etm_handle_bad_accept(fmd_hdl_t *hdl, int nev)
2616 {
2617 	int	should_reset_xport; /* bool to reset xport */
2618 
2619 	should_reset_xport = (nev == -ENOTACTIVE);
2620 	fmd_hdl_debug(hdl, "error: bad conn accept errno %d\n", (-nev));
2621 	etm_stats.etm_xport_accept_fail.fmds_value.ui64++;
2622 	(void) etm_sleep(etm_bad_acc_to_sec); /* avoid spinning CPU */
2623 	if (should_reset_xport) {
2624 		etm_reset_xport(hdl);
2625 	}
2626 } /* etm_handle_bad_accept() */
2627 
2628 /*
2629  * etm_server - loop forever accepting new connections
2630  *		using the given FMD handle,
2631  *		handling any ETM msgs sent from the other side
2632  *		via each such connection
2633  */
2634 
2635 static void
etm_server(void * arg)2636 etm_server(void *arg)
2637 {
2638 	etm_xport_conn_t	conn;		/* connection handle */
2639 	int			nev;		/* -errno val */
2640 	fmd_hdl_t		*hdl;		/* FMD handle */
2641 
2642 	hdl = arg;
2643 
2644 	fmd_hdl_debug(hdl, "info: connection server starting\n");
2645 
2646 	/*
2647 	 * Restore the checkpointed events and dispatch them before starting to
2648 	 * receive more events from the sp.
2649 	 */
2650 	etm_ckpt_recover(hdl);
2651 
2652 	while (!etm_is_dying) {
2653 
2654 		if ((conn = etm_xport_accept(hdl, NULL)) == NULL) {
2655 			/* errno assumed set by above call */
2656 			nev = (-errno);
2657 			if (etm_is_dying) {
2658 				break;
2659 			}
2660 			etm_handle_bad_accept(hdl, nev);
2661 			continue;
2662 		}
2663 
2664 		/* handle the new message/connection, closing it when done */
2665 
2666 		etm_handle_new_conn(hdl, conn);
2667 
2668 	} /* while accepting new connections until ETM dies */
2669 
2670 	/* ETM is dying (probably due to "fmadm unload etm") */
2671 
2672 	fmd_hdl_debug(hdl, "info: connection server is dying\n");
2673 
2674 } /* etm_server() */
2675 
2676 /*
2677  * etm_responder - loop forever waiting for new responder queue elements
2678  *		to be enqueued, for each one constructing and sending
2679  *		an ETM response msg to the other side, and closing its
2680  *		associated connection when appropriate
2681  *
2682  *	this thread exists to ensure that the etm_server() thread
2683  *	never pends indefinitely waiting on the xport write lock, and is
2684  *	hence always available to accept new connections and handle
2685  *	incoming messages
2686  *
2687  *	this design relies on the fact that each connection accepted and
2688  *	returned by the ETM xport layer is unique, and each can be closed
2689  *	independently of the others while multiple connections are
2690  *	outstanding
2691  */
2692 
2693 static void
etm_responder(void * arg)2694 etm_responder(void *arg)
2695 {
2696 	ssize_t			n;		/* gen use */
2697 	fmd_hdl_t		*hdl;		/* FMD handle */
2698 	etm_resp_q_ele_t	rqe;		/* responder queue ele */
2699 
2700 	hdl = arg;
2701 
2702 	fmd_hdl_debug(hdl, "info: responder server starting\n");
2703 
2704 	while (!etm_is_dying) {
2705 
2706 		(void) pthread_mutex_lock(&etm_resp_q_lock);
2707 
2708 		while (etm_resp_q_cur_len == 0) {
2709 			(void) pthread_cond_wait(&etm_resp_q_cv,
2710 			    &etm_resp_q_lock);
2711 			if (etm_is_dying) {
2712 				(void) pthread_mutex_unlock(&etm_resp_q_lock);
2713 				goto func_ret;
2714 			}
2715 		} /* while the responder queue is empty, wait to be nudged */
2716 
2717 		/*
2718 		 * for every responder ele that has been enqueued,
2719 		 * dequeue and send it as an ETM response msg,
2720 		 * closing its associated conn and freeing its hdr
2721 		 *
2722 		 * enter the queue draining loop holding the responder
2723 		 * queue lock, but do not hold the lock indefinitely
2724 		 * (the actual send may pend us indefinitely),
2725 		 * so that other threads will never pend for long
2726 		 * trying to enqueue a new element
2727 		 */
2728 
2729 		while (etm_resp_q_cur_len > 0) {
2730 
2731 			(void) etm_resp_q_deq(hdl, &rqe);
2732 
2733 			if ((etm_resp_q_cur_len + 1) == etm_resp_q_max_len)
2734 				(void) pthread_cond_signal(&etm_resp_q_cv);
2735 
2736 			(void) pthread_mutex_unlock(&etm_resp_q_lock);
2737 
2738 			if ((n = etm_send_response(hdl, rqe.rqe_conn,
2739 			    rqe.rqe_hdrp, rqe.rqe_resp_code)) < 0) {
2740 				fmd_hdl_error(hdl, "error: bad resp send "
2741 				    "errno %d\n", (-n));
2742 			}
2743 
2744 			(void) etm_conn_close(hdl, "bad conn close after resp",
2745 			    rqe.rqe_conn);
2746 			fmd_hdl_free(hdl, rqe.rqe_hdrp, rqe.rqe_hdr_sz);
2747 
2748 			if (etm_is_dying) {
2749 				goto func_ret;
2750 			}
2751 			(void) pthread_mutex_lock(&etm_resp_q_lock);
2752 
2753 		} /* while draining the responder queue */
2754 
2755 		(void) pthread_mutex_unlock(&etm_resp_q_lock);
2756 
2757 	} /* while awaiting and sending resp msgs until ETM dies */
2758 
2759 func_ret:
2760 
2761 	/* ETM is dying (probably due to "fmadm unload etm") */
2762 
2763 	fmd_hdl_debug(hdl, "info: responder server is dying\n");
2764 
2765 	(void) pthread_mutex_lock(&etm_resp_q_lock);
2766 	if (etm_resp_q_cur_len > 0) {
2767 		fmd_hdl_error(hdl, "warning: %d response msgs dropped\n",
2768 		    (int)etm_resp_q_cur_len);
2769 		while (etm_resp_q_cur_len > 0) {
2770 			(void) etm_resp_q_deq(hdl, &rqe);
2771 			(void) etm_conn_close(hdl, "bad conn close after deq",
2772 			    rqe.rqe_conn);
2773 			fmd_hdl_free(hdl, rqe.rqe_hdrp, rqe.rqe_hdr_sz);
2774 		}
2775 	}
2776 	(void) pthread_mutex_unlock(&etm_resp_q_lock);
2777 
2778 } /* etm_responder() */
2779 
2780 static void *
etm_init_alloc(size_t size)2781 etm_init_alloc(size_t size)
2782 {
2783 	return (fmd_hdl_alloc(init_hdl, size, FMD_SLEEP));
2784 }
2785 
2786 static void
etm_init_free(void * addr,size_t size)2787 etm_init_free(void *addr, size_t size)
2788 {
2789 	fmd_hdl_free(init_hdl, addr, size);
2790 }
2791 
2792 /*
2793  * ---------------------root ldom support functions -----------------------
2794  */
2795 
2796 /*
2797  * use a static array async_event_q instead of dynamicaly allocated mem  queue
2798  * for etm_async_q_enq and etm_async_q_deq.
2799  * This is not running in an fmd aux thread, can't use the fmd_hdl_* funcs.
2800  * caller needs to grab the mutex lock before calling this func.
2801  * return >0 for success, or -errno value
2802  */
2803 static int
etm_async_q_enq(etm_async_event_ele_t * async_e)2804 etm_async_q_enq(etm_async_event_ele_t *async_e)
2805 {
2806 
2807 	if (etm_async_q_cur_len >= etm_async_q_max_len) {
2808 		/* etm_stats.etm_enq_drop_async_q.fmds_value.ui64++; */
2809 		return (-E2BIG);
2810 	}
2811 
2812 	(void) memcpy(&async_event_q[etm_async_q_tail], async_e,
2813 	    sizeof (*async_e));
2814 
2815 	etm_async_q_tail++;
2816 	if (etm_async_q_tail == etm_async_q_max_len) {
2817 		etm_async_q_tail = 0;
2818 	}
2819 	etm_async_q_cur_len++;
2820 
2821 /* etm_stats.etm_async_q_cur_len.fmds_value.ui64 = etm_async_q_cur_len; */
2822 
2823 	return (1);
2824 
2825 } /* etm_async_q_enq() */
2826 
2827 
2828 static int
etm_async_q_deq(etm_async_event_ele_t * async_e)2829 etm_async_q_deq(etm_async_event_ele_t *async_e)
2830 {
2831 
2832 	if (etm_async_q_cur_len == 0) {
2833 		/* etm_stats.etm_deq_drop_async_q.fmds_value.ui64++; */
2834 		return (-ENOENT);
2835 	}
2836 
2837 	(void) memcpy(async_e, &async_event_q[etm_async_q_head],
2838 	    sizeof (*async_e));
2839 
2840 	etm_async_q_head++;
2841 	if (etm_async_q_head == etm_async_q_max_len) {
2842 		etm_async_q_head = 0;
2843 	}
2844 	etm_async_q_cur_len--;
2845 
2846 	return (1);
2847 } /* etm_async_q_deq */
2848 
2849 
2850 /*
2851  * setting up the fields in iosvc at DS_REG_CB time
2852  */
2853 void
etm_iosvc_setup(fmd_hdl_t * fmd_hdl,etm_iosvc_t * iosvc,etm_async_event_ele_t * async_e)2854 etm_iosvc_setup(fmd_hdl_t *fmd_hdl, etm_iosvc_t *iosvc,
2855     etm_async_event_ele_t *async_e)
2856 {
2857 	iosvc->ds_hdl = async_e->ds_hdl;
2858 	iosvc->cur_send_xid = 0;
2859 	iosvc->xid_posted_ev = 0;
2860 	iosvc->start_sending_Q = 0;
2861 
2862 	/*
2863 	 * open the fmd xprt if it
2864 	 * hasn't been previously opened
2865 	 */
2866 	fmd_hdl_debug(fmd_hdl,  "info: before fmd_xprt_open ldom_name is %s\n",
2867 	    async_e->ldom_name);
2868 
2869 	if (iosvc->fmd_xprt == NULL) {
2870 		iosvc->fmd_xprt = fmd_xprt_open(fmd_hdl, flags, NULL, iosvc);
2871 	}
2872 
2873 	iosvc->thr_is_dying = 0;
2874 	if (iosvc->recv_tid == 0) {
2875 		iosvc->recv_tid = fmd_thr_create(fmd_hdl,
2876 		    etm_recv_from_remote_root, iosvc);
2877 	}
2878 	if (iosvc->send_tid == 0) {
2879 		iosvc->send_tid = fmd_thr_create(fmd_hdl,
2880 		    etm_send_to_remote_root, iosvc);
2881 	}
2882 } /* etm_iosvc_setup() */
2883 
2884 
2885 /*
2886  * ds userland interface ds_reg_cb  callback func
2887  */
2888 
2889 /* ARGSUSED */
2890 static void
etm_iosvc_reg_handler(ds_hdl_t ds_hdl,ds_cb_arg_t arg,ds_ver_t * ver,ds_domain_hdl_t dhdl)2891 etm_iosvc_reg_handler(ds_hdl_t ds_hdl, ds_cb_arg_t arg, ds_ver_t *ver,
2892     ds_domain_hdl_t dhdl)
2893 {
2894 	etm_async_event_ele_t	async_ele;
2895 
2896 
2897 	/*
2898 	 * do version check here.
2899 	 * checked the ver received here against etm_iosvc_vers here
2900 	 */
2901 	if (etm_iosvc_vers[0].major != ver->major ||
2902 	    etm_iosvc_vers[0].minor != ver->minor) {
2903 		/*
2904 		 * can't log an fmd debug msg,
2905 		 * not running in an fmd aux thread
2906 		 */
2907 		return;
2908 	}
2909 
2910 	/*
2911 	 * the callback should have a valid ldom_name
2912 	 * can't log fmd debugging msg here since this is not in an fmd aux
2913 	 * thread. log fmd debug msg in etm_async_event_handle()
2914 	 */
2915 	async_ele.ds_hdl = ds_hdl;
2916 	async_ele.dhdl = dhdl;
2917 	async_ele.ldom_name[0] = '\0';
2918 	async_ele.event_type = ETM_ASYNC_EVENT_DS_REG_CB;
2919 	(void) pthread_mutex_lock(&etm_async_event_q_lock);
2920 	(void) etm_async_q_enq(&async_ele);
2921 	if (etm_async_q_cur_len == 1)
2922 		(void) pthread_cond_signal(&etm_async_event_q_cv);
2923 	(void) pthread_mutex_unlock(&etm_async_event_q_lock);
2924 
2925 } /* etm_iosvc_reg_handler */
2926 
2927 
2928 /*
2929  * ds userland interface ds_unreg_cb  callback func
2930  */
2931 
2932 /*ARGSUSED*/
2933 static void
etm_iosvc_unreg_handler(ds_hdl_t hdl,ds_cb_arg_t arg)2934 etm_iosvc_unreg_handler(ds_hdl_t hdl, ds_cb_arg_t arg)
2935 {
2936 	etm_async_event_ele_t	async_ele;
2937 
2938 	/*
2939 	 * fill in async_ele and enqueue async_ele
2940 	 */
2941 	async_ele.ldom_name[0] = '\0';
2942 	async_ele.ds_hdl = hdl;
2943 	async_ele.event_type = ETM_ASYNC_EVENT_DS_UNREG_CB;
2944 	(void) pthread_mutex_lock(&etm_async_event_q_lock);
2945 	(void) etm_async_q_enq(&async_ele);
2946 	if (etm_async_q_cur_len == 1)
2947 		(void) pthread_cond_signal(&etm_async_event_q_cv);
2948 	(void) pthread_mutex_unlock(&etm_async_event_q_lock);
2949 } /* etm_iosvc_unreg_handler */
2950 
2951 /*
2952  * ldom event registration callback func
2953  */
2954 
2955 /* ARGSUSED */
2956 static void
ldom_event_handler(char * ldom_name,ldom_event_t event,ldom_cb_arg_t data)2957 ldom_event_handler(char *ldom_name, ldom_event_t event, ldom_cb_arg_t data)
2958 {
2959 	etm_async_event_ele_t	async_ele;
2960 
2961 	/*
2962 	 * the callback will have a valid ldom_name
2963 	 */
2964 	async_ele.ldom_name[0] = '\0';
2965 	if (ldom_name)
2966 		(void) strcpy(async_ele.ldom_name, ldom_name);
2967 	async_ele.ds_hdl = DS_INVALID_HDL;
2968 
2969 	/*
2970 	 * fill in async_ele and enq async_ele
2971 	 */
2972 	switch (event) {
2973 	case LDOM_EVENT_BIND:
2974 		async_ele.event_type = ETM_ASYNC_EVENT_LDOM_BIND;
2975 		break;
2976 	case LDOM_EVENT_UNBIND:
2977 		async_ele.event_type = ETM_ASYNC_EVENT_LDOM_UNBIND;
2978 		break;
2979 	case LDOM_EVENT_ADD:
2980 		async_ele.event_type = ETM_ASYNC_EVENT_LDOM_ADD;
2981 		break;
2982 	case LDOM_EVENT_REMOVE:
2983 		async_ele.event_type = ETM_ASYNC_EVENT_LDOM_REMOVE;
2984 		break;
2985 	default:
2986 		/*
2987 		 * for all other ldom events, do nothing
2988 		 */
2989 		return;
2990 	} /* switch (event) */
2991 
2992 	(void) pthread_mutex_lock(&etm_async_event_q_lock);
2993 	(void) etm_async_q_enq(&async_ele);
2994 	if (etm_async_q_cur_len == 1)
2995 		(void) pthread_cond_signal(&etm_async_event_q_cv);
2996 	(void) pthread_mutex_unlock(&etm_async_event_q_lock);
2997 
2998 } /* ldom_event_handler */
2999 
3000 
3001 /*
3002  * This is running as an fmd aux thread.
3003  * This is the func that actually handle the events, which include:
3004  * 1. ldom events. ldom events are  on Control Domain only
3005  * 2. any DS userland callback funcs
3006  * these events are already Q-ed in the async_event_ele_q
3007  * deQ and process the events accordingly
3008  */
3009 static void
etm_async_event_handler(void * arg)3010 etm_async_event_handler(void *arg)
3011 {
3012 
3013 	fmd_hdl_t		*fmd_hdl = (fmd_hdl_t *)arg;
3014 	etm_iosvc_t		*iosvc;		/* ptr 2 iosvc struct */
3015 	etm_async_event_ele_t	async_e;
3016 
3017 	fmd_hdl_debug(fmd_hdl, "info: etm_async_event_handler starting\n");
3018 	/*
3019 	 *  handle etm is not dying and Q len > 0
3020 	 */
3021 	while (!etm_is_dying) {
3022 		/*
3023 		 * grab the lock to check the Q len
3024 		 */
3025 		(void) pthread_mutex_lock(&etm_async_event_q_lock);
3026 		fmd_hdl_debug(fmd_hdl, "info: etm_async_q_cur_len %d\n",
3027 		    etm_async_q_cur_len);
3028 
3029 		while (etm_async_q_cur_len > 0) {
3030 			(void) etm_async_q_deq(&async_e);
3031 			(void) pthread_mutex_unlock(&etm_async_event_q_lock);
3032 			fmd_hdl_debug(fmd_hdl,
3033 			    "info: processing an async event type %d ds_hdl"
3034 			    " %d\n", async_e.event_type, async_e.ds_hdl);
3035 			if (async_e.ldom_name[0] != '\0') {
3036 				fmd_hdl_debug(fmd_hdl,
3037 				    "info: procssing async evt ldom_name %s\n",
3038 				    async_e.ldom_name);
3039 			}
3040 
3041 			/*
3042 			 * at this point, if async_e.ldom_name is not NULL,
3043 			 * we have a valid iosvc strcut ptr.
3044 			 * the only time async_e.ldom_name is NULL is  at
3045 			 * ds_unreg_cb()
3046 			 */
3047 			switch (async_e.event_type)  {
3048 			case ETM_ASYNC_EVENT_LDOM_UNBIND:
3049 			case ETM_ASYNC_EVENT_LDOM_REMOVE:
3050 				/*
3051 				 * we have a valid ldom_name,
3052 				 * etm_lookup_struct(ldom_name)
3053 				 * do nothing if can't find an iosvc
3054 				 * no iosvc clean up to do
3055 				 */
3056 				(void) pthread_mutex_lock(
3057 				    &iosvc_list_lock);
3058 				iosvc = etm_iosvc_lookup(fmd_hdl,
3059 				    async_e.ldom_name,
3060 				    async_e.ds_hdl, B_FALSE);
3061 				if (iosvc == NULL) {
3062 					fmd_hdl_debug(fmd_hdl,
3063 					    "error: can't find iosvc for ldom "
3064 					    "name %s\n",
3065 					    async_e.ldom_name);
3066 					(void) pthread_mutex_unlock(
3067 					    &iosvc_list_lock);
3068 					break;
3069 				}
3070 				/*
3071 				 * Clean up the queue, delete all messages and
3072 				 * do not persist checkpointed fma events.
3073 				 */
3074 				etm_iosvc_cleanup(fmd_hdl, iosvc, B_TRUE,
3075 				    B_TRUE);
3076 				(void) pthread_mutex_unlock(
3077 				    &iosvc_list_lock);
3078 				break;
3079 
3080 			case ETM_ASYNC_EVENT_LDOM_BIND:
3081 
3082 				/*
3083 				 * create iosvc if it has not been
3084 				 * created
3085 				 * async_e.ds_hdl is invalid
3086 				 * async_e.ldom_name is valid ldom_name
3087 				 */
3088 				(void) pthread_mutex_lock(
3089 				    &iosvc_list_lock);
3090 				iosvc = etm_iosvc_lookup(fmd_hdl,
3091 				    async_e.ldom_name,
3092 				    async_e.ds_hdl, B_TRUE);
3093 				if (iosvc == NULL) {
3094 					fmd_hdl_debug(fmd_hdl,
3095 					    "error: can't create iosvc for "
3096 					    "async evnt %d\n",
3097 					    async_e.event_type);
3098 					(void) pthread_mutex_unlock(
3099 					    &iosvc_list_lock);
3100 					break;
3101 				}
3102 				(void) strcpy(iosvc->ldom_name,
3103 				    async_e.ldom_name);
3104 				iosvc->ds_hdl = async_e.ds_hdl;
3105 				(void) pthread_mutex_unlock(
3106 				    &iosvc_list_lock);
3107 				break;
3108 
3109 			case ETM_ASYNC_EVENT_DS_REG_CB:
3110 				if (etm_ldom_type == LDOM_TYPE_CONTROL) {
3111 					/*
3112 					 * find the root ldom name from
3113 					 * ldom domain hdl/id
3114 					 */
3115 					if (etm_filter_find_ldom_name(
3116 					    fmd_hdl, async_e.dhdl,
3117 					    async_e.ldom_name,
3118 					    MAX_LDOM_NAME) != 0) {
3119 						fmd_hdl_debug(fmd_hdl,
3120 						    "error: can't find root "
3121 						    "domain name from did %d\n",
3122 						    async_e.dhdl);
3123 						break;
3124 					} else {
3125 						fmd_hdl_debug(fmd_hdl,
3126 						    "info: etm_filter_find_"
3127 						    "ldom_name returned %s\n",
3128 						    async_e.ldom_name);
3129 					}
3130 					/*
3131 					 * now we should have a valid
3132 					 * root domain name.
3133 					 * lookup the iosvc struct
3134 					 * associated with the ldom_name
3135 					 * and init the iosvc struct
3136 					 */
3137 					(void) pthread_mutex_lock(
3138 					    &iosvc_list_lock);
3139 					iosvc = etm_iosvc_lookup(
3140 					    fmd_hdl, async_e.ldom_name,
3141 					    async_e.ds_hdl, B_TRUE);
3142 					if (iosvc == NULL) {
3143 						fmd_hdl_debug(fmd_hdl,
3144 						    "error: can't create iosvc "
3145 						    "for async evnt %d\n",
3146 						    async_e.event_type);
3147 						(void) pthread_mutex_unlock(
3148 						    &iosvc_list_lock);
3149 						break;
3150 					}
3151 
3152 					etm_iosvc_setup(fmd_hdl, iosvc,
3153 					    &async_e);
3154 					(void) pthread_mutex_unlock(
3155 					    &iosvc_list_lock);
3156 				} else {
3157 					iosvc = &io_svc;
3158 					(void) strcpy(iosvc->ldom_name,
3159 					    async_e.ldom_name);
3160 
3161 					etm_iosvc_setup(fmd_hdl, iosvc,
3162 					    &async_e);
3163 				}
3164 				break;
3165 
3166 			case ETM_ASYNC_EVENT_DS_UNREG_CB:
3167 				/*
3168 				 * decide which iosvc struct to perform
3169 				 * this UNREG callback on.
3170 				 */
3171 				if (etm_ldom_type == LDOM_TYPE_CONTROL) {
3172 					(void) pthread_mutex_lock(
3173 					    &iosvc_list_lock);
3174 					/*
3175 					 * lookup the iosvc struct w/
3176 					 * ds_hdl
3177 					 */
3178 					iosvc = etm_iosvc_lookup(
3179 					    fmd_hdl, async_e.ldom_name,
3180 					    async_e.ds_hdl, B_FALSE);
3181 					if (iosvc == NULL) {
3182 						fmd_hdl_debug(fmd_hdl,
3183 						    "error: can't find iosvc "
3184 						    "for async evnt %d\n",
3185 						    async_e.event_type);
3186 						(void) pthread_mutex_unlock(
3187 						    &iosvc_list_lock);
3188 						break;
3189 					}
3190 
3191 					/*
3192 					 * ds_hdl and fmd_xprt_open
3193 					 * go hand to hand together
3194 					 * after unreg_cb,
3195 					 * ds_hdl is INVALID and
3196 					 * fmd_xprt is closed.
3197 					 * the ldom name and the msg Q
3198 					 * remains in iosvc_list
3199 					 */
3200 					if (*iosvc->ldom_name != '\0')
3201 						fmd_hdl_debug(fmd_hdl,
3202 						    "info: iosvc  w/ ldom_name "
3203 						    "%s \n", iosvc->ldom_name);
3204 
3205 					/*
3206 					 * destroy send/recv threads and
3207 					 * other clean up on Control side.
3208 					 */
3209 					etm_iosvc_cleanup(fmd_hdl, iosvc,
3210 					    B_FALSE, B_FALSE);
3211 					(void) pthread_mutex_unlock(
3212 					    &iosvc_list_lock);
3213 				} else {
3214 					iosvc = &io_svc;
3215 					/*
3216 					 * destroy send/recv threads and
3217 					 * then clean up on Root side.
3218 					 */
3219 					etm_iosvc_cleanup(fmd_hdl, iosvc,
3220 					    B_FALSE, B_FALSE);
3221 				}
3222 				break;
3223 
3224 			default:
3225 				/*
3226 				 * for all other events, etm doesn't care.
3227 				 * already logged an fmd info msg w/
3228 				 * the event type. Do nothing here.
3229 				 */
3230 				break;
3231 			} /* switch (async_e.event_type) */
3232 
3233 			if (etm_ldom_type == LDOM_TYPE_CONTROL) {
3234 				etm_filter_handle_ldom_event(fmd_hdl,
3235 				    async_e.event_type, async_e.ldom_name);
3236 			}
3237 
3238 			/*
3239 			 * grab the lock to check the q length again
3240 			 */
3241 			(void) pthread_mutex_lock(&etm_async_event_q_lock);
3242 
3243 			if (etm_is_dying) {
3244 				break;
3245 			}
3246 		}	/* etm_async_q_cur_len */
3247 
3248 		/*
3249 		 * we have the mutex lock at this point, whether
3250 		 * . etm_is_dying  and/or
3251 		 * . q_len == 0
3252 		 */
3253 		if (!etm_is_dying && etm_async_q_cur_len == 0) {
3254 			fmd_hdl_debug(fmd_hdl,
3255 			    "info: cond wait on async_event_q_cv\n");
3256 			(void) pthread_cond_wait(&etm_async_event_q_cv,
3257 			    &etm_async_event_q_lock);
3258 			fmd_hdl_debug(fmd_hdl,
3259 			    "info: cond wait on async_event_q_cv rtns\n");
3260 		}
3261 		(void) pthread_mutex_unlock(&etm_async_event_q_lock);
3262 	} /* etm_is_dying */
3263 
3264 	fmd_hdl_debug(fmd_hdl,
3265 	    "info: etm async event handler thread exiting\n");
3266 
3267 } /* etm_async_event_handler */
3268 
3269 /*
3270  * deQ what's in iosvc msg Q
3271  * send iosvc_msgp to the remote io svc ldom by calling ds_send_msg()
3272  * the iosvc_msgp already has the packed msg, which is hdr + 1 fma event
3273  */
3274 static void
etm_send_to_remote_root(void * arg)3275 etm_send_to_remote_root(void *arg)
3276 {
3277 
3278 	etm_iosvc_t		*iosvc = (etm_iosvc_t *)arg;	/* iosvc ptr */
3279 	etm_iosvc_q_ele_t	msg_ele;	/* iosvc msg ele */
3280 	etm_proto_v1_ev_hdr_t	*ev_hdrp;	/* hdr for FMA_EVENT */
3281 	fmd_hdl_t		*fmd_hdl = init_hdl;	/* fmd handle */
3282 
3283 
3284 	fmd_hdl_debug(fmd_hdl,
3285 	    "info: send to remote iosvc starting w/ ldom_name %s\n",
3286 	    iosvc->ldom_name);
3287 
3288 	/*
3289 	 *  loop forever until etm_is_dying or thr_is_dying
3290 	 */
3291 	while (!etm_is_dying && !iosvc->thr_is_dying) {
3292 		if (iosvc->ds_hdl != DS_INVALID_HDL &&
3293 		    iosvc->start_sending_Q > 0) {
3294 			(void) pthread_mutex_lock(&iosvc->msg_q_lock);
3295 			while (iosvc->msg_q_cur_len > 0 &&
3296 			    iosvc->ds_hdl != DS_INVALID_HDL)  {
3297 				(void) etm_iosvc_msg_deq(fmd_hdl, iosvc,
3298 				    &msg_ele);
3299 				if (etm_debug_lvl >= 3) {
3300 					fmd_hdl_debug(fmd_hdl, "info: valid "
3301 					    "ds_hdl before ds_send_msg \n");
3302 				}
3303 				(void) pthread_mutex_unlock(&iosvc->msg_q_lock);
3304 
3305 				iosvc->ack_ok = 0;
3306 				ev_hdrp = (etm_proto_v1_ev_hdr_t *)
3307 				    ((ptrdiff_t)msg_ele.msg);
3308 				ev_hdrp->ev_pp.pp_xid = iosvc->cur_send_xid + 1;
3309 				while (!iosvc->ack_ok &&
3310 				    iosvc->ds_hdl != DS_INVALID_HDL &&
3311 				    !etm_is_dying) {
3312 					/*
3313 					 * call ds_send_msg() to send the msg,
3314 					 * wait for the recv end to send the
3315 					 * resp msg back.
3316 					 * If resp msg is recv-ed, ack_ok
3317 					 * will be set to 1.
3318 					 * otherwise, retry.
3319 					 */
3320 					if (etm_send_ds_msg(fmd_hdl, B_TRUE,
3321 					    iosvc, &msg_ele, ev_hdrp) < 0) {
3322 						continue;
3323 					}
3324 
3325 					if (etm_is_dying || iosvc->thr_is_dying)
3326 						break;
3327 				}
3328 
3329 				/*
3330 				 * if out of the while loop but !ack_ok, ie,
3331 				 * ds_hdl becomes invalid at some point
3332 				 * while waiting the resp msg, we need to put
3333 				 * the msg back to the head of the Q.
3334 				 */
3335 				if (!iosvc->ack_ok) {
3336 					(void) pthread_mutex_lock(
3337 					    &iosvc->msg_q_lock);
3338 					/*
3339 					 * put the msg back to the head of Q.
3340 					 * If the Q is full at this point,
3341 					 * drop the msg at the tail, enq this
3342 					 * msg to the head.
3343 					 */
3344 					etm_msg_enq_head(fmd_hdl, iosvc,
3345 					    &msg_ele);
3346 					(void) pthread_mutex_unlock(
3347 					    &iosvc->msg_q_lock);
3348 				}
3349 
3350 				/*
3351 				 *
3352 				 * grab the lock to check the Q len again
3353 				 */
3354 				(void) pthread_mutex_lock(&iosvc->msg_q_lock);
3355 				if (etm_is_dying || iosvc->thr_is_dying) {
3356 					break;
3357 				}
3358 			} /* while dequeing iosvc msgs to send */
3359 
3360 			/*
3361 			 * we have the mutex lock for msg_q_lock at this point
3362 			 * we are here because
3363 			 * 1) q_len == 0: then wait on the cv for Q to be filled
3364 			 * 2) etm_is_dying
3365 			 */
3366 			if (!etm_is_dying && !iosvc->thr_is_dying &&
3367 			    iosvc->msg_q_cur_len == 0) {
3368 				fmd_hdl_debug(fmd_hdl,
3369 				    "info: waiting on msg_q_cv\n");
3370 				(void) pthread_cond_wait(&iosvc->msg_q_cv,
3371 				    &iosvc->msg_q_lock);
3372 			}
3373 			(void) pthread_mutex_unlock(&iosvc->msg_q_lock);
3374 			if (etm_is_dying || iosvc->thr_is_dying)  {
3375 				break;
3376 			}
3377 		} else {
3378 			(void) etm_sleep(1);
3379 		} /* wait for the start_sendingQ > 0 */
3380 	} /* etm_is_dying or thr_is_dying */
3381 	fmd_hdl_debug(fmd_hdl, "info; etm send thread exiting \n");
3382 } /* etm_send_to_remote_root */
3383 
3384 
3385 /*
3386  * receive etm msgs from the remote root ldom by calling ds_recv_msg()
3387  * if FMA events/ereports, call fmd_xprt_post() to post to fmd
3388  * send ACK back by calling ds_send_msg()
3389  */
3390 static void
etm_recv_from_remote_root(void * arg)3391 etm_recv_from_remote_root(void *arg)
3392 {
3393 	etm_iosvc_t		*iosvc = (etm_iosvc_t *)arg;	/* iosvc ptr */
3394 	etm_proto_v1_pp_t	*pp;		/* protocol preamble */
3395 	etm_proto_v1_ev_hdr_t	*ev_hdrp;	/* for FMA_EVENT msg */
3396 	etm_proto_v1_resp_hdr_t	*resp_hdrp;	/* for RESPONSE msg */
3397 	int32_t			resp_code = 0;	/* default is success */
3398 	int32_t			rc;		/* return value */
3399 	size_t			maxlen = MAXLEN;
3400 						/* max msg len */
3401 	char			msgbuf[MAXLEN];	/* recv msg buf */
3402 	size_t			msg_size;	/* recv msg size */
3403 	size_t			hdr_sz;		/* sizeof *hdrp */
3404 	size_t			evsz;		/* sizeof *evp */
3405 	size_t			fma_event_size;	/* sizeof FMA event  */
3406 	nvlist_t		*evp;		/* ptr to the nvlist */
3407 	char			*buf;		/* ptr to the nvlist */
3408 	static uint32_t		mem_alloc = 0;	/* indicate if alloc mem */
3409 	char			*msg;		/* ptr to alloc mem */
3410 	fmd_hdl_t		*fmd_hdl = init_hdl;
3411 
3412 
3413 
3414 	fmd_hdl_debug(fmd_hdl,
3415 	    "info: recv from remote iosvc starting with ldom name %s \n",
3416 	    iosvc->ldom_name);
3417 
3418 	/*
3419 	 * loop forever until etm_is_dying or the thread is dying
3420 	 */
3421 
3422 	msg = msgbuf;
3423 	while (!etm_is_dying && !iosvc->thr_is_dying) {
3424 		if (iosvc->ds_hdl == DS_INVALID_HDL) {
3425 			fmd_hdl_debug(fmd_hdl,
3426 			    "info: ds_hdl is invalid in recv thr\n");
3427 			(void) etm_sleep(1);
3428 			continue;
3429 		}
3430 
3431 		/*
3432 		 * for now, there are FMA_EVENT and ACK msg type.
3433 		 * use FMA_EVENT buf as the maxlen, hdr+1 fma event.
3434 		 * FMA_EVENT is big enough to hold an ACK msg.
3435 		 * the actual msg size received is in msg_size.
3436 		 */
3437 		rc = (*etm_ds_recv_msg)(iosvc->ds_hdl, msg, maxlen, &msg_size);
3438 		if (rc == EFBIG) {
3439 			fmd_hdl_debug(fmd_hdl,
3440 			    "info: ds_recv_msg needs mem the size of %d\n",
3441 			    msg_size);
3442 			msg = fmd_hdl_zalloc(fmd_hdl, msg_size, FMD_SLEEP);
3443 			mem_alloc = 1;
3444 		} else if (rc == 0) {
3445 			fmd_hdl_debug(fmd_hdl,
3446 			    "info: ds_recv_msg received a msg ok\n");
3447 			/*
3448 			 * check the magic # in  msg.hdr
3449 			 */
3450 			pp = (etm_proto_v1_pp_t *)((ptrdiff_t)msg);
3451 			if (pp->pp_magic_num != ETM_PROTO_MAGIC_NUM) {
3452 				fmd_hdl_debug(fmd_hdl,
3453 				    "info: bad ds recv on magic\n");
3454 				continue;
3455 			}
3456 
3457 			/*
3458 			 * check the msg type against msg_size to be sure
3459 			 * that received msg is not a truncated msg
3460 			 */
3461 			if (pp->pp_msg_type == ETM_MSG_TYPE_FMA_EVENT) {
3462 
3463 				ev_hdrp = (etm_proto_v1_ev_hdr_t *)
3464 				    ((ptrdiff_t)msg);
3465 				fmd_hdl_debug(fmd_hdl, "info: ds received "
3466 				    "FMA EVENT xid=%d msg_size=%d\n",
3467 				    ev_hdrp->ev_pp.pp_xid, msg_size);
3468 				hdr_sz = sizeof (*ev_hdrp) +
3469 				    1*(sizeof (ev_hdrp->ev_lens[0]));
3470 				fma_event_size = hdr_sz + ev_hdrp->ev_lens[0];
3471 				if (fma_event_size != msg_size) {
3472 					fmd_hdl_debug(fmd_hdl, "info: wrong "
3473 					    "ev msg size received\n");
3474 					continue;
3475 					/*
3476 					 * Simply  do nothing. The send side
3477 					 * will timedcond_wait waiting on the
3478 					 * resp msg will timeout and
3479 					 * re-send the same msg.
3480 					 */
3481 				}
3482 				if (etm_debug_lvl >= 3) {
3483 					fmd_hdl_debug(fmd_hdl,  "info: recv msg"
3484 					    " size %d hdrsz %d evp size %d\n",
3485 					    msg_size, hdr_sz,
3486 					    ev_hdrp->ev_lens[0]);
3487 				}
3488 
3489 				if (ev_hdrp->ev_pp.pp_xid !=
3490 				    iosvc->xid_posted_ev) {
3491 					/*
3492 					 * different from last xid posted to
3493 					 * fmd, post to fmd now.
3494 					 */
3495 					buf = msg + hdr_sz;
3496 					rc = nvlist_unpack(buf,
3497 					    ev_hdrp->ev_lens[0], &evp, 0);
3498 					rc = nvlist_size(evp, &evsz,
3499 					    NV_ENCODE_XDR);
3500 					fmd_hdl_debug(fmd_hdl,
3501 					    "info: evp size %d before fmd"
3502 					    "post\n", evsz);
3503 
3504 					if ((rc = etm_post_to_fmd(fmd_hdl,
3505 					    iosvc->fmd_xprt, evp)) >= 0) {
3506 						fmd_hdl_debug(fmd_hdl,
3507 						    "info: xid posted to fmd %d"
3508 						    "\n",
3509 						    ev_hdrp->ev_pp.pp_xid);
3510 						iosvc->xid_posted_ev =
3511 						    ev_hdrp->ev_pp.pp_xid;
3512 					}
3513 				}
3514 
3515 				/*
3516 				 * ready to  send the RESPONSE msg back
3517 				 * reuse the msg buffer as the response buffer
3518 				 */
3519 				resp_hdrp = (etm_proto_v1_resp_hdr_t *)
3520 				    ((ptrdiff_t)msg);
3521 				resp_hdrp->resp_pp.pp_msg_type =
3522 				    ETM_MSG_TYPE_RESPONSE;
3523 
3524 				resp_hdrp->resp_code = resp_code;
3525 				resp_hdrp->resp_len = sizeof (*resp_hdrp);
3526 
3527 				/*
3528 				 * send the whole response msg in one send
3529 				 */
3530 				if ((*etm_ds_send_msg)(iosvc->ds_hdl, msg,
3531 				    sizeof (*resp_hdrp)) != 0) {
3532 					fmd_hdl_debug(fmd_hdl,
3533 					    "info: send response msg failed\n");
3534 				} else {
3535 					fmd_hdl_debug(fmd_hdl,
3536 					    "info: ds send resp msg ok"
3537 					    "size %d\n", sizeof (*resp_hdrp));
3538 				}
3539 			} else if (pp->pp_msg_type == ETM_MSG_TYPE_RESPONSE) {
3540 				fmd_hdl_debug(fmd_hdl,
3541 				    "info: ds received respond msg xid=%d"
3542 				    "msg_size=%d for ldom %s\n", pp->pp_xid,
3543 				    msg_size, iosvc->ldom_name);
3544 				if (sizeof (*resp_hdrp) != msg_size) {
3545 					fmd_hdl_debug(fmd_hdl,
3546 					    "info: wrong resp msg size"
3547 					    "received\n");
3548 					fmd_hdl_debug(fmd_hdl,
3549 					    "info: resp msg size %d recv resp"
3550 					    "msg size %d\n",
3551 					    sizeof (*resp_hdrp), msg_size);
3552 					continue;
3553 				}
3554 				/*
3555 				 * is the pp.pp_xid == iosvc->cur_send_xid+1,
3556 				 * if so, nudge the send routine to send next
3557 				 */
3558 				if (pp->pp_xid != iosvc->cur_send_xid+1) {
3559 					fmd_hdl_debug(fmd_hdl,
3560 					    "info: ds received resp msg xid=%d "
3561 					    "doesn't match cur_send_id=%d\n",
3562 					    pp->pp_xid, iosvc->cur_send_xid+1);
3563 					continue;
3564 				}
3565 				(void) pthread_mutex_lock(&iosvc->msg_ack_lock);
3566 				iosvc->ack_ok = 1;
3567 				(void) pthread_cond_signal(&iosvc->msg_ack_cv);
3568 				(void) pthread_mutex_unlock(
3569 				    &iosvc->msg_ack_lock);
3570 				fmd_hdl_debug(fmd_hdl,
3571 				    "info: signaling msg_ack_cv\n");
3572 			} else {
3573 				/*
3574 				 * place holder for future msg types
3575 				 */
3576 				fmd_hdl_debug(fmd_hdl,
3577 				    "info: ds received unrecognized msg\n");
3578 			}
3579 			if (mem_alloc) {
3580 				fmd_hdl_free(fmd_hdl, msg, msg_size);
3581 				mem_alloc = 0;
3582 				msg = msgbuf;
3583 			}
3584 		} else {
3585 			if (etm_debug_lvl >= 3) {
3586 				fmd_hdl_debug(fmd_hdl,
3587 				    "info: ds_recv_msg() failed\n");
3588 			}
3589 		} /* ds_recv_msg() returns */
3590 	} /* etm_is_dying */
3591 
3592 	/*
3593 	 * need to free the mem allocated in msg upon exiting the thread
3594 	 */
3595 	if (mem_alloc) {
3596 		fmd_hdl_free(fmd_hdl, msg, msg_size);
3597 		mem_alloc = 0;
3598 		msg = msgbuf;
3599 	}
3600 	fmd_hdl_debug(fmd_hdl, "info; etm recv thread exiting \n");
3601 } /* etm_recv_from_remote_root */
3602 
3603 
3604 
3605 /*
3606  * etm_ds_init
3607  *		initialize DS services function pointers by calling
3608  *		dlopen() followed by  dlsym() for each ds func.
3609  *		if any dlopen() or dlsym() call fails, return -ENOENT
3610  *		return >0 for successs, -ENOENT for failure
3611  */
3612 static int
etm_ds_init(fmd_hdl_t * hdl)3613 etm_ds_init(fmd_hdl_t *hdl)
3614 {
3615 	int rc = 0;
3616 
3617 	if ((etm_dl_hdl = dlopen(etm_dl_path, etm_dl_mode)) == NULL) {
3618 		fmd_hdl_debug(hdl, "error: failed to dlopen %s\n", etm_dl_path);
3619 		return (-ENOENT);
3620 	}
3621 
3622 	etm_ds_svc_reg = (int (*)(ds_capability_t *cap, ds_ops_t *ops))
3623 	    dlsym(etm_dl_hdl, "ds_svc_reg");
3624 	if (etm_ds_svc_reg == NULL) {
3625 		fmd_hdl_debug(hdl,
3626 		    "error: failed to dlsym ds_svc_reg() w/ error %s\n",
3627 		    dlerror());
3628 		rc = -ENOENT;
3629 	}
3630 
3631 
3632 	etm_ds_clnt_reg = (int (*)(ds_capability_t *cap, ds_ops_t *ops))
3633 	    dlsym(etm_dl_hdl, "ds_clnt_reg");
3634 	if (etm_ds_clnt_reg == NULL) {
3635 		fmd_hdl_debug(hdl,
3636 		    "error: dlsym(ds_clnt_reg) failed w/ errno %d\n", errno);
3637 		rc = -ENOENT;
3638 	}
3639 
3640 	etm_ds_send_msg = (int (*)(ds_hdl_t hdl, void *buf, size_t buflen))
3641 	    dlsym(etm_dl_hdl, "ds_send_msg");
3642 	if (etm_ds_send_msg == NULL) {
3643 		fmd_hdl_debug(hdl, "error: dlsym(ds_send_msg) failed\n");
3644 		rc = -ENOENT;
3645 	}
3646 
3647 	etm_ds_recv_msg = (int (*)(ds_hdl_t hdl, void *buf, size_t buflen,
3648 	    size_t *msglen))dlsym(etm_dl_hdl, "ds_recv_msg");
3649 	if (etm_ds_recv_msg == NULL) {
3650 		fmd_hdl_debug(hdl, "error: dlsym(ds_recv_msg) failed\n");
3651 		rc = -ENOENT;
3652 	}
3653 
3654 	etm_ds_fini = (int (*)(void))dlsym(etm_dl_hdl, "ds_fini");
3655 	if (etm_ds_fini == NULL) {
3656 		fmd_hdl_debug(hdl, "error: dlsym(ds_fini) failed\n");
3657 		rc = -ENOENT;
3658 	}
3659 
3660 	if (rc == -ENOENT) {
3661 		(void) dlclose(etm_dl_hdl);
3662 	}
3663 	return (rc);
3664 
3665 } /* etm_ds_init() */
3666 
3667 
3668 /*
3669  * -------------------------- FMD entry points -------------------------------
3670  */
3671 
3672 /*
3673  * _fmd_init - initialize the transport for use by ETM and start the
3674  *		server daemon to accept new connections to us
3675  *
3676  *		FMD will read our *.conf and subscribe us to FMA events
3677  */
3678 
3679 void
_fmd_init(fmd_hdl_t * hdl)3680 _fmd_init(fmd_hdl_t *hdl)
3681 {
3682 	struct timeval		tmv;		/* timeval */
3683 	ssize_t			n;		/* gen use */
3684 	const struct facility	*fp;		/* syslog facility matching */
3685 	char			*facname;	/* syslog facility property */
3686 	uint32_t		type_mask;	/* type of the local host */
3687 	int			rc;		/* funcs return code */
3688 
3689 
3690 	if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
3691 		return; /* invalid data in configuration file */
3692 	}
3693 
3694 	fmd_hdl_debug(hdl, "info: module initializing\n");
3695 
3696 	init_hdl = hdl;
3697 	etm_lhp = ldom_init(etm_init_alloc, etm_init_free);
3698 
3699 	/*
3700 	 * decide the ldom type, do initialization accordingly
3701 	 */
3702 	if ((rc = ldom_get_type(etm_lhp, &type_mask)) != 0) {
3703 		fmd_hdl_debug(hdl, "error: can't decide ldom type\n");
3704 		fmd_hdl_debug(hdl, "info: module unregistering\n");
3705 		ldom_fini(etm_lhp);
3706 		fmd_hdl_unregister(hdl);
3707 		return;
3708 	}
3709 
3710 	if ((type_mask & LDOM_TYPE_LEGACY) || (type_mask & LDOM_TYPE_CONTROL)) {
3711 		if (type_mask & LDOM_TYPE_LEGACY) {
3712 			/*
3713 			 * running on a legacy sun4v domain,
3714 			 * act as the the old sun4v
3715 			 */
3716 			etm_ldom_type = LDOM_TYPE_LEGACY;
3717 			fmd_hdl_debug(hdl, "info: running as the old sun4v\n");
3718 			ldom_fini(etm_lhp);
3719 		} else if (type_mask & LDOM_TYPE_CONTROL) {
3720 			etm_ldom_type = LDOM_TYPE_CONTROL;
3721 			fmd_hdl_debug(hdl, "info: running as control domain\n");
3722 
3723 			/*
3724 			 * looking for libds.so.1.
3725 			 * If not found, don't do DS registration. As a result,
3726 			 * there will be no DS callbacks or other DS services.
3727 			 */
3728 			if (etm_ds_init(hdl) >= 0) {
3729 				etm_filter_init(hdl);
3730 				etm_ckpt_init(hdl);
3731 
3732 				flags = FMD_XPRT_RDWR | FMD_XPRT_ACCEPT;
3733 
3734 				/*
3735 				 * ds client registration
3736 				 */
3737 				if ((rc = (*etm_ds_clnt_reg)(&iosvc_caps,
3738 				    &iosvc_ops))) {
3739 					fmd_hdl_debug(hdl,
3740 					"error: ds_clnt_reg(): errno %d\n", rc);
3741 				}
3742 			} else {
3743 				fmd_hdl_debug(hdl, "error: dlopen() libds "
3744 				    "failed, continue without the DS services");
3745 			}
3746 
3747 			/*
3748 			 * register for ldom status events
3749 			 */
3750 			if ((rc = ldom_register_event(etm_lhp,
3751 			    ldom_event_handler, hdl))) {
3752 				fmd_hdl_debug(hdl,
3753 				    "error: ldom_register_event():"
3754 				    " errno %d\n", rc);
3755 			}
3756 
3757 			/*
3758 			 * create the thread for handling both the ldom status
3759 			 * change and service events
3760 			 */
3761 			etm_async_e_tid = fmd_thr_create(hdl,
3762 			    etm_async_event_handler, hdl);
3763 		}
3764 
3765 		/* setup statistics and properties from FMD */
3766 
3767 		(void) fmd_stat_create(hdl, FMD_STAT_NOALLOC,
3768 		    sizeof (etm_stats) / sizeof (fmd_stat_t),
3769 		    (fmd_stat_t *)&etm_stats);
3770 
3771 		etm_fma_resp_wait_time = fmd_prop_get_int32(hdl,
3772 		    ETM_PROP_NM_FMA_RESP_WAIT_TIME);
3773 		etm_debug_lvl = fmd_prop_get_int32(hdl, ETM_PROP_NM_DEBUG_LVL);
3774 		etm_debug_max_ev_cnt = fmd_prop_get_int32(hdl,
3775 		    ETM_PROP_NM_DEBUG_MAX_EV_CNT);
3776 		fmd_hdl_debug(hdl, "info: etm_debug_lvl %d "
3777 		    "etm_debug_max_ev_cnt %d\n", etm_debug_lvl,
3778 		    etm_debug_max_ev_cnt);
3779 
3780 		etm_resp_q_max_len = fmd_prop_get_int32(hdl,
3781 		    ETM_PROP_NM_MAX_RESP_Q_LEN);
3782 		etm_stats.etm_resp_q_max_len.fmds_value.ui64 =
3783 		    etm_resp_q_max_len;
3784 		etm_bad_acc_to_sec = fmd_prop_get_int32(hdl,
3785 		    ETM_PROP_NM_BAD_ACC_TO_SEC);
3786 
3787 		/*
3788 		 * obtain an FMD transport handle so we can post
3789 		 * FMA events later
3790 		 */
3791 
3792 		etm_fmd_xprt = fmd_xprt_open(hdl, FMD_XPRT_RDONLY, NULL, NULL);
3793 
3794 		/*
3795 		 * encourage protocol transaction id to be unique per module
3796 		 * load
3797 		 */
3798 
3799 		(void) gettimeofday(&tmv, NULL);
3800 		etm_xid_cur = (uint32_t)((tmv.tv_sec << 10) |
3801 		    ((unsigned long)tmv.tv_usec >> 10));
3802 
3803 		/* init the ETM transport */
3804 
3805 		if ((n = etm_xport_init(hdl)) != 0) {
3806 			fmd_hdl_error(hdl, "error: bad xport init errno %d\n",
3807 			    (-n));
3808 			fmd_hdl_unregister(hdl);
3809 			return;
3810 		}
3811 
3812 		/*
3813 		 * Cache any properties we use every time we receive an alert.
3814 		 */
3815 		syslog_file = fmd_prop_get_int32(hdl, ETM_PROP_NM_SYSLOGD);
3816 		syslog_cons = fmd_prop_get_int32(hdl, ETM_PROP_NM_CONSOLE);
3817 
3818 		if (syslog_file && (syslog_logfd = open("/dev/conslog",
3819 		    O_WRONLY | O_NOCTTY)) == -1) {
3820 			fmd_hdl_error(hdl,
3821 			    "error: failed to open /dev/conslog");
3822 			syslog_file = 0;
3823 		}
3824 
3825 		if (syslog_cons && (syslog_msgfd = open("/dev/sysmsg",
3826 		    O_WRONLY | O_NOCTTY)) == -1) {
3827 			fmd_hdl_error(hdl, "error: failed to open /dev/sysmsg");
3828 			syslog_cons = 0;
3829 		}
3830 
3831 		if (syslog_file) {
3832 			/*
3833 			 * Look up the value of the "facility" property and
3834 			 * use it to determine * what syslog LOG_* facility
3835 			 * value we use to fill in our log_ctl_t.
3836 			 */
3837 			facname = fmd_prop_get_string(hdl,
3838 			    ETM_PROP_NM_FACILITY);
3839 
3840 			for (fp = syslog_facs; fp->fac_name != NULL; fp++) {
3841 				if (strcmp(fp->fac_name, facname) == 0)
3842 					break;
3843&