1733a535rb/*
2733a535rb * CDDL HEADER START
3733a535rb *
4733a535rb * The contents of this file are subject to the terms of the
531e37bbvn * Common Development and Distribution License (the "License").
631e37bbvn * You may not use this file except in compliance with the License.
7733a535rb *
8733a535rb * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9733a535rb * or http://www.opensolaris.org/os/licensing.
10733a535rb * See the License for the specific language governing permissions
11733a535rb * and limitations under the License.
12733a535rb *
13733a535rb * When distributing Covered Code, include this CDDL HEADER in each
14733a535rb * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15733a535rb * If applicable, add the following below this CDDL HEADER, with the
16733a535rb * fields enclosed by brackets "[]" replaced with your own identifying
17733a535rb * information: Portions Copyright [yyyy] [name of copyright owner]
18733a535rb *
19733a535rb * CDDL HEADER END
20733a535rb */
21733a535rb
22733a535rb/*
23c5fb5d3Karl Davis * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24733a535rb * Use is subject to license terms.
25733a535rb */
26733a535rb
27733a535rb/*
28733a535rb * etm.c	FMA Event Transport Module implementation, a plugin of FMD
29733a535rb *		for sun4v/Ontario
30733a535rb *
31733a535rb * plugin for sending/receiving FMA events to/from service processor
32733a535rb */
33733a535rb
34733a535rb/*
35733a535rb * --------------------------------- includes --------------------------------
36733a535rb */
37733a535rb
38733a535rb#include <sys/fm/protocol.h>
39733a535rb#include <sys/fm/util.h>
4031e37bbvn#include <sys/fm/ldom.h>
414b476eddarudy#include <sys/strlog.h>
424b476eddarudy#include <sys/syslog.h>
432535165Vuong Nguyen#include <sys/libds.h>
44b8677b7rb#include <netinet/in.h>
45b8677b7rb#include <fm/fmd_api.h>
46733a535rb
47733a535rb#include "etm_xport_api.h"
48733a535rb#include "etm_etm_proto.h"
49733a535rb#include "etm_impl.h"
502535165Vuong Nguyen#include "etm_iosvc.h"
512535165Vuong Nguyen#include "etm_filter.h"
522535165Vuong Nguyen#include "etm_ckpt.h"
53733a535rb
54733a535rb#include <pthread.h>
55733a535rb#include <signal.h>
56733a535rb#include <stropts.h>
57733a535rb#include <locale.h>
58733a535rb#include <strings.h>
59733a535rb#include <stdlib.h>
60733a535rb#include <unistd.h>
61733a535rb#include <limits.h>
62733a535rb#include <values.h>
63733a535rb#include <alloca.h>
64733a535rb#include <errno.h>
652535165Vuong Nguyen#include <dlfcn.h>
662535165Vuong Nguyen#include <link.h>
67733a535rb#include <fcntl.h>
68733a535rb#include <time.h>
69733a535rb
70733a535rb/*
71733a535rb * ----------------------------- forward decls -------------------------------
72733a535rb */
73733a535rb
74733a535rbstatic void
75733a535rbetm_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class);
76733a535rb
772535165Vuong Nguyenstatic int
782535165Vuong Nguyenetm_send(fmd_hdl_t *hdl, fmd_xprt_t *xp, fmd_event_t *event, nvlist_t *nvl);
792535165Vuong Nguyen
802535165Vuong Nguyenstatic void
812535165Vuong Nguyenetm_send_to_remote_root(void *arg);
822535165Vuong Nguyen
832535165Vuong Nguyenstatic void
842535165Vuong Nguyenetm_recv_from_remote_root(void *arg);
852535165Vuong Nguyen
86d279c7bVuong Nguyenstatic void
87d279c7bVuong Nguyenetm_ckpt_remove(fmd_hdl_t *hdl, etm_iosvc_q_ele_t *ele);
88d279c7bVuong Nguyen
89733a535rb/*
90733a535rb * ------------------------- data structs for FMD ----------------------------
91733a535rb */
92733a535rb
932c07a09Yanmin Sunstatic const fmd_hdl_ops_t fmd_ops = {
94733a535rb	etm_recv,	/* fmdo_recv */
95733a535rb	NULL,		/* fmdo_timeout */
96733a535rb	NULL,		/* fmdo_close */
97733a535rb	NULL,		/* fmdo_stats */
98733a535rb	NULL,		/* fmdo_gc */
992535165Vuong Nguyen	etm_send,	/* fmdo_send */
100733a535rb};
101733a535rb
102733a535rbstatic const fmd_prop_t fmd_props[] = {
1032535165Vuong Nguyen	{ ETM_PROP_NM_XPORT_ADDRS,		FMD_TYPE_STRING, "" },
1042535165Vuong Nguyen	{ ETM_PROP_NM_DEBUG_LVL,		FMD_TYPE_INT32, "0" },
1052535165Vuong Nguyen	{ ETM_PROP_NM_DEBUG_MAX_EV_CNT,		FMD_TYPE_INT32, "-1" },
1062535165Vuong Nguyen	{ ETM_PROP_NM_CONSOLE,			FMD_TYPE_BOOL, "false" },
1072535165Vuong Nguyen	{ ETM_PROP_NM_SYSLOGD,			FMD_TYPE_BOOL, "true" },
1082535165Vuong Nguyen	{ ETM_PROP_NM_FACILITY,			FMD_TYPE_STRING, "LOG_DAEMON" },
109c5fb5d3Karl Davis	{ ETM_PROP_NM_MAX_RESP_Q_LEN,		FMD_TYPE_UINT32, "32" },
1102535165Vuong Nguyen	{ ETM_PROP_NM_BAD_ACC_TO_SEC,		FMD_TYPE_UINT32, "1" },
1112535165Vuong Nguyen	{ ETM_PROP_NM_FMA_RESP_WAIT_TIME,	FMD_TYPE_INT32, "240" },
112733a535rb	{ NULL, 0, NULL }
113733a535rb};
114733a535rb
1154b476eddarudy
116733a535rbstatic const fmd_hdl_info_t fmd_info = {
1172535165Vuong Nguyen	"FMA Event Transport Module", "1.2", &fmd_ops, fmd_props
118733a535rb};
119733a535rb
120733a535rb/*
121733a535rb * ----------------------- private consts and defns --------------------------
122733a535rb */
123733a535rb
124733a535rb/* misc buffer for variable sized protocol header fields */
125733a535rb
126733a535rb#define	ETM_MISC_BUF_SZ	(4 * 1024)
127733a535rb
1282535165Vuong Nguyenstatic uint32_t
1292535165Vuong Nguyenetm_ldom_type = LDOM_TYPE_LEGACY;
1302535165Vuong Nguyen
131733a535rb/* try limit for IO operations w/ capped exp backoff sleep on retry */
132733a535rb
133733a535rb/*
134733a535rb * Design_Note:	ETM will potentially retry forever IO operations that the
135733a535rb *		transport fails with EAGAIN (aka EWOULDBLOCK) rather than
136733a535rb *		giving up after some number of seconds. This avoids
137733a535rb *		dropping FMA events while the service processor is down,
138733a535rb *		but at the risk of pending fmdo_recv() forever and
139733a535rb *		overflowing FMD's event queue for ETM.
140733a535rb *		A future TBD enhancement would be to always recv
141733a535rb *		and send each ETM msg in a single read/write() to reduce
142733a535rb *		the risk of failure between ETM msg hdr and body,
143733a535rb *		assuming the MTU_SZ is large enough.
144733a535rb */
145733a535rb
146733a535rb#define	ETM_TRY_MAX_CNT		(MAXINT - 1)
147733a535rb#define	ETM_TRY_BACKOFF_RATE	(4)
148733a535rb#define	ETM_TRY_BACKOFF_CAP	(60)
149733a535rb
150e2ff4acrb/* amount to increment protocol transaction id on each new send */
151733a535rb
1522535165Vuong Nguyen#define	ETM_XID_INC		(2)
153733a535rb
154b8677b7rbtypedef struct etm_resp_q_ele {
155b8677b7rb
156b8677b7rb	etm_xport_conn_t	rqe_conn;	/* open connection to send on */
157b8677b7rb	etm_proto_v1_pp_t	*rqe_hdrp;	/* ptr to ETM msg hdr */
158b8677b7rb	size_t			rqe_hdr_sz;	/* sizeof ETM msg hdr */
159b8677b7rb	int32_t			rqe_resp_code;	/* response code to send */
160b8677b7rb
161b8677b7rb	struct etm_resp_q_ele	*rqe_nextp;	/* PRIVATE - next ele ptr */
162b8677b7rb
163b8677b7rb} etm_resp_q_ele_t;	/* responder queue element */
164b8677b7rb
165733a535rb/*
166733a535rb * ---------------------------- global data ----------------------------------
167733a535rb */
168733a535rb
16931e37bbvnstatic fmd_hdl_t
1702535165Vuong Nguyen*init_hdl = NULL;	/* used in mem allocator and several other places */
17131e37bbvn
172733a535rbstatic int
173733a535rbetm_debug_lvl = 0;	/* debug level: 0 is off, 1 is on, 2 is more, etc */
174733a535rb
175733a535rbstatic int
176733a535rbetm_debug_max_ev_cnt = -1; /* max allowed event count for debugging */
177733a535rb
17800ab125rbstatic fmd_xprt_t
17900ab125rb*etm_fmd_xprt = NULL;	/* FMD transport layer handle */
18000ab125rb
181733a535rbstatic pthread_t
182733a535rbetm_svr_tid = NULL;	/* thread id of connection acceptance server */
183733a535rb
184b8677b7rbstatic pthread_t
185b8677b7rbetm_resp_tid = NULL;	/* thread id of msg responder */
186b8677b7rb
187b8677b7rbstatic etm_resp_q_ele_t
188b8677b7rb*etm_resp_q_head = NULL; /* ptr to cur head of responder queue */
189b8677b7rb
190b8677b7rbstatic etm_resp_q_ele_t
191b8677b7rb*etm_resp_q_tail = NULL; /* ptr to cur tail of responder queue */
192b8677b7rb
193b8677b7rbstatic uint32_t
194b8677b7rbetm_resp_q_cur_len = 0;	/* cur length (ele cnt) of responder queue */
195b8677b7rb
196b8677b7rbstatic uint32_t
197b8677b7rbetm_resp_q_max_len = 0;	/* max length (ele cnt) of responder queue */
198b8677b7rb
1992ca9f23rbstatic uint32_t
2002ca9f23rbetm_bad_acc_to_sec = 0;	/* sleep timeout (in sec) after bad conn accept */
2012ca9f23rb
202b8677b7rbstatic pthread_mutex_t
203b8677b7rbetm_resp_q_lock = PTHREAD_MUTEX_INITIALIZER;	/* protects responder queue */
204b8677b7rb
205b8677b7rbstatic pthread_cond_t
206b8677b7rbetm_resp_q_cv = PTHREAD_COND_INITIALIZER;	/* nudges msg responder */
207b8677b7rb
208733a535rbstatic volatile int
209733a535rbetm_is_dying = 0;	/* bool for dying (killing self) */
210733a535rb
211733a535rbstatic uint32_t
212e2ff4acrbetm_xid_cur = 0;	/* current transaction id for sends */
213733a535rb
214733a535rbstatic uint32_t
215733a535rbetm_xid_ping = 0;	/* xid of last CONTROL msg sent requesting ping */
216733a535rb
217733a535rbstatic uint32_t
218e2ff4acrbetm_xid_ver_negot = 0;	/* xid of last CONTROL msg sent requesting ver negot */
219e2ff4acrb
220e2ff4acrbstatic uint32_t
2212535165Vuong Nguyenetm_xid_posted_logged_ev = 0;
2222535165Vuong Nguyen			/* xid of last FMA_EVENT msg/event posted OK to FMD */
223e2ff4acrb
224b8677b7rbstatic uint32_t
225b8677b7rbetm_xid_posted_sa = 0;	/* xid of last ALERT msg/event posted OK to syslog */
226b8677b7rb
227e2ff4acrbstatic uint8_t
228e2ff4acrbetm_resp_ver = ETM_PROTO_V1; /* proto ver [negotiated] for msg sends */
229733a535rb
2302535165Vuong Nguyenstatic uint32_t
2312535165Vuong Nguyenetm_fma_resp_wait_time = 30;	/*  time (sec) wait for fma event resp */
2322535165Vuong Nguyen
2332ae6665jruttstatic pthread_mutex_t
2342ae6665jruttetm_write_lock = PTHREAD_MUTEX_INITIALIZER;	/* for write operations */
2352ae6665jrutt
2364b476eddarudystatic log_ctl_t syslog_ctl;	/* log(7D) meta-data for each msg */
2374b476eddarudystatic int syslog_facility;	/* log(7D) facility (part of priority) */
2384b476eddarudystatic int syslog_logfd = -1;	/* log(7D) file descriptor */
2394b476eddarudystatic int syslog_msgfd = -1;	/* sysmsg(7D) file descriptor */
2404b476eddarudystatic int syslog_file = 0;	/* log to syslog_logfd */
2414b476eddarudystatic int syslog_cons = 0;	/* log to syslog_msgfd */
2424b476eddarudy
2434b476eddarudystatic const struct facility {
2444b476eddarudy	const char *fac_name;
2454b476eddarudy	int fac_value;
2464b476eddarudy} syslog_facs[] = {
2474b476eddarudy	{ "LOG_DAEMON", LOG_DAEMON },
2484b476eddarudy	{ "LOG_LOCAL0", LOG_LOCAL0 },
2494b476eddarudy	{ "LOG_LOCAL1", LOG_LOCAL1 },
2504b476eddarudy	{ "LOG_LOCAL2", LOG_LOCAL2 },
2514b476eddarudy	{ "LOG_LOCAL3", LOG_LOCAL3 },
2524b476eddarudy	{ "LOG_LOCAL4", LOG_LOCAL4 },
2534b476eddarudy	{ "LOG_LOCAL5", LOG_LOCAL5 },
2544b476eddarudy	{ "LOG_LOCAL6", LOG_LOCAL6 },
2554b476eddarudy	{ "LOG_LOCAL7", LOG_LOCAL7 },
2564b476eddarudy	{ NULL, 0 }
2574b476eddarudy};
2584b476eddarudy
259733a535rbstatic struct stats {
260733a535rb
261733a535rb	/* ETM msg counters */
262733a535rb
263733a535rb	fmd_stat_t etm_rd_hdr_fmaevent;
264733a535rb	fmd_stat_t etm_rd_hdr_control;
2654b476eddarudy	fmd_stat_t etm_rd_hdr_alert;
266733a535rb	fmd_stat_t etm_rd_hdr_response;
267733a535rb	fmd_stat_t etm_rd_body_fmaevent;
268733a535rb	fmd_stat_t etm_rd_body_control;
2694b476eddarudy	fmd_stat_t etm_rd_body_alert;
270733a535rb	fmd_stat_t etm_rd_body_response;
271733a535rb	fmd_stat_t etm_wr_hdr_fmaevent;
272733a535rb	fmd_stat_t etm_wr_hdr_control;
273733a535rb	fmd_stat_t etm_wr_hdr_response;
274733a535rb	fmd_stat_t etm_wr_body_fmaevent;
275733a535rb	fmd_stat_t etm_wr_body_control;
276733a535rb	fmd_stat_t etm_wr_body_response;
277733a535rb
278b8677b7rb	fmd_stat_t etm_rd_max_ev_per_msg;
279b8677b7rb	fmd_stat_t etm_wr_max_ev_per_msg;
280b8677b7rb
281b8677b7rb	fmd_stat_t etm_resp_q_cur_len;
282b8677b7rb	fmd_stat_t etm_resp_q_max_len;
283b8677b7rb
284733a535rb	/* ETM byte counters */
285733a535rb
286733a535rb	fmd_stat_t etm_wr_fmd_bytes;
287733a535rb	fmd_stat_t etm_rd_fmd_bytes;
288733a535rb	fmd_stat_t etm_wr_xport_bytes;
289733a535rb	fmd_stat_t etm_rd_xport_bytes;
290733a535rb
291733a535rb	fmd_stat_t etm_magic_drop_bytes;
292733a535rb
293733a535rb	/* ETM [dropped] FMA event counters */
294733a535rb
295733a535rb	fmd_stat_t etm_rd_fmd_fmaevent;
296733a535rb	fmd_stat_t etm_wr_fmd_fmaevent;
297733a535rb
298733a535rb	fmd_stat_t etm_rd_drop_fmaevent;
299733a535rb	fmd_stat_t etm_wr_drop_fmaevent;
300733a535rb
301e2ff4acrb	fmd_stat_t etm_rd_dup_fmaevent;
302e2ff4acrb	fmd_stat_t etm_wr_dup_fmaevent;
303e2ff4acrb
304b8677b7rb	fmd_stat_t etm_rd_dup_alert;
305b8677b7rb	fmd_stat_t etm_wr_dup_alert;
306b8677b7rb
307b8677b7rb	fmd_stat_t etm_enq_drop_resp_q;
308b8677b7rb	fmd_stat_t etm_deq_drop_resp_q;
309b8677b7rb
310733a535rb	/* ETM protocol failures */
311733a535rb
312733a535rb	fmd_stat_t etm_magic_bad;
313733a535rb	fmd_stat_t etm_ver_bad;
314733a535rb	fmd_stat_t etm_msgtype_bad;
315733a535rb	fmd_stat_t etm_subtype_bad;
316733a535rb	fmd_stat_t etm_xid_bad;
317733a535rb	fmd_stat_t etm_fmaeventlen_bad;
318733a535rb	fmd_stat_t etm_respcode_bad;
319733a535rb	fmd_stat_t etm_timeout_bad;
320733a535rb	fmd_stat_t etm_evlens_bad;
321733a535rb
322733a535rb	/* IO operation failures */
323733a535rb
324733a535rb	fmd_stat_t etm_xport_wr_fail;
325733a535rb	fmd_stat_t etm_xport_rd_fail;
326733a535rb	fmd_stat_t etm_xport_pk_fail;
327733a535rb
328733a535rb	/* IO operation retries */
329733a535rb
330733a535rb	fmd_stat_t etm_xport_wr_retry;
331733a535rb	fmd_stat_t etm_xport_rd_retry;
332733a535rb	fmd_stat_t etm_xport_pk_retry;
333733a535rb
334733a535rb	/* system and library failures */
335733a535rb
336733a535rb	fmd_stat_t etm_os_nvlist_pack_fail;
337733a535rb	fmd_stat_t etm_os_nvlist_unpack_fail;
338733a535rb	fmd_stat_t etm_os_nvlist_size_fail;
339733a535rb	fmd_stat_t etm_os_pthread_create_fail;
340733a535rb
341733a535rb	/* xport API failures */
342733a535rb
343733a535rb	fmd_stat_t etm_xport_get_ev_addrv_fail;
344733a535rb	fmd_stat_t etm_xport_open_fail;
345733a535rb	fmd_stat_t etm_xport_close_fail;
346733a535rb	fmd_stat_t etm_xport_accept_fail;
347733a535rb	fmd_stat_t etm_xport_open_retry;
348733a535rb
349733a535rb	/* FMD entry point bad arguments */
350733a535rb
351733a535rb	fmd_stat_t etm_fmd_init_badargs;
352733a535rb	fmd_stat_t etm_fmd_fini_badargs;
353733a535rb
3544b476eddarudy	/* Alert logging errors */
355b8677b7rb
3564b476eddarudy	fmd_stat_t etm_log_err;
3574b476eddarudy	fmd_stat_t etm_msg_err;
3584b476eddarudy
3592ca9f23rb	/* miscellaneous stats */
3602ca9f23rb
3612ca9f23rb	fmd_stat_t etm_reset_xport;
3622ca9f23rb
363733a535rb} etm_stats = {
364733a535rb
365733a535rb	/* ETM msg counters */
366733a535rb
367733a535rb	{ "etm_rd_hdr_fmaevent", FMD_TYPE_UINT64,
368733a535rb		"ETM fmaevent msg headers rcvd from xport" },
369733a535rb	{ "etm_rd_hdr_control", FMD_TYPE_UINT64,
370733a535rb		"ETM control msg headers rcvd from xport" },
3714b476eddarudy	{ "etm_rd_hdr_alert", FMD_TYPE_UINT64,
3724b476eddarudy		"ETM alert msg headers rcvd from xport" },
373733a535rb	{ "etm_rd_hdr_response", FMD_TYPE_UINT64,
374733a535rb		"ETM response msg headers rcvd from xport" },
375733a535rb	{ "etm_rd_body_fmaevent", FMD_TYPE_UINT64,
376733a535rb		"ETM fmaevent msg bodies rcvd from xport" },
377733a535rb	{ "etm_rd_body_control", FMD_TYPE_UINT64,
378733a535rb		"ETM control msg bodies rcvd from xport" },
3794b476eddarudy	{ "etm_rd_body_alert", FMD_TYPE_UINT64,
3804b476eddarudy		"ETM alert msg bodies rcvd from xport" },
381733a535rb	{ "etm_rd_body_response", FMD_TYPE_UINT64,
382733a535rb		"ETM response msg bodies rcvd from xport" },
383733a535rb	{ "etm_wr_hdr_fmaevent", FMD_TYPE_UINT64,
384733a535rb		"ETM fmaevent msg headers sent to xport" },
385733a535rb	{ "etm_wr_hdr_control", FMD_TYPE_UINT64,
386733a535rb		"ETM control msg headers sent to xport" },
387733a535rb	{ "etm_wr_hdr_response", FMD_TYPE_UINT64,
388733a535rb		"ETM response msg headers sent to xport" },
389733a535rb	{ "etm_wr_body_fmaevent", FMD_TYPE_UINT64,
390733a535rb		"ETM fmaevent msg bodies sent to xport" },
391733a535rb	{ "etm_wr_body_control", FMD_TYPE_UINT64,
392733a535rb		"ETM control msg bodies sent to xport" },
393733a535rb	{ "etm_wr_body_response", FMD_TYPE_UINT64,
394733a535rb		"ETM response msg bodies sent to xport" },
395733a535rb
396b8677b7rb	{ "etm_rd_max_ev_per_msg", FMD_TYPE_UINT64,
397b8677b7rb		"max FMA events per ETM msg from xport" },
398b8677b7rb	{ "etm_wr_max_ev_per_msg", FMD_TYPE_UINT64,
399b8677b7rb		"max FMA events per ETM msg to xport" },
400b8677b7rb
401b8677b7rb	{ "etm_resp_q_cur_len", FMD_TYPE_UINT64,
402b8677b7rb		"cur enqueued response msgs to xport" },
403b8677b7rb	{ "etm_resp_q_max_len", FMD_TYPE_UINT64,
404b8677b7rb		"max enqueable response msgs to xport" },
405b8677b7rb
406733a535rb	/* ETM byte counters */
407733a535rb
408733a535rb	{ "etm_wr_fmd_bytes", FMD_TYPE_UINT64,
409733a535rb		"bytes of FMA events sent to FMD" },
410733a535rb	{ "etm_rd_fmd_bytes", FMD_TYPE_UINT64,
411733a535rb		"bytes of FMA events rcvd from FMD" },
412733a535rb	{ "etm_wr_xport_bytes", FMD_TYPE_UINT64,
413733a535rb		"bytes of FMA events sent to xport" },
414733a535rb	{ "etm_rd_xport_bytes", FMD_TYPE_UINT64,
415733a535rb		"bytes of FMA events rcvd from xport" },
416733a535rb
417733a535rb	{ "etm_magic_drop_bytes", FMD_TYPE_UINT64,
418733a535rb		"bytes dropped from xport pre magic num" },
419733a535rb
420733a535rb	/* ETM [dropped] FMA event counters */
421733a535rb
422733a535rb	{ "etm_rd_fmd_fmaevent", FMD_TYPE_UINT64,
423733a535rb		"FMA events rcvd from FMD" },
424733a535rb	{ "etm_wr_fmd_fmaevent", FMD_TYPE_UINT64,
425733a535rb		"FMA events sent to FMD" },
426733a535rb
427733a535rb	{ "etm_rd_drop_fmaevent", FMD_TYPE_UINT64,
428733a535rb		"dropped FMA events from xport" },
429733a535rb	{ "etm_wr_drop_fmaevent", FMD_TYPE_UINT64,
430733a535rb		"dropped FMA events to xport" },
431733a535rb
432e2ff4acrb	{ "etm_rd_dup_fmaevent", FMD_TYPE_UINT64,
433b8677b7rb	    "duplicate FMA events rcvd from xport" },
434e2ff4acrb	{ "etm_wr_dup_fmaevent", FMD_TYPE_UINT64,
435b8677b7rb	    "duplicate FMA events sent to xport" },
436b8677b7rb
437b8677b7rb	{ "etm_rd_dup_alert", FMD_TYPE_UINT64,
438b8677b7rb	    "duplicate ALERTs rcvd from xport" },
439b8677b7rb	{ "etm_wr_dup_alert", FMD_TYPE_UINT64,
440b8677b7rb	    "duplicate ALERTs sent to xport" },
441b8677b7rb
442b8677b7rb	{ "etm_enq_drop_resp_q", FMD_TYPE_UINT64,
443b8677b7rb	    "dropped response msgs on enq" },
444b8677b7rb	{ "etm_deq_drop_resp_q", FMD_TYPE_UINT64,
445b8677b7rb	    "dropped response msgs on deq" },
446e2ff4acrb
447733a535rb	/* ETM protocol failures */
448733a535rb
449733a535rb	{ "etm_magic_bad", FMD_TYPE_UINT64,
450733a535rb		"ETM msgs w/ invalid magic num" },
451733a535rb	{ "etm_ver_bad", FMD_TYPE_UINT64,
452733a535rb		"ETM msgs w/ invalid protocol version" },
453733a535rb	{ "etm_msgtype_bad", FMD_TYPE_UINT64,
454733a535rb		"ETM msgs w/ invalid message type" },
455733a535rb	{ "etm_subtype_bad", FMD_TYPE_UINT64,
456733a535rb		"ETM msgs w/ invalid sub type" },
457733a535rb	{ "etm_xid_bad", FMD_TYPE_UINT64,
458733a535rb		"ETM msgs w/ unmatched xid" },
459733a535rb	{ "etm_fmaeventlen_bad", FMD_TYPE_UINT64,
460733a535rb		"ETM msgs w/ invalid FMA event length" },
461733a535rb	{ "etm_respcode_bad", FMD_TYPE_UINT64,
462733a535rb		"ETM msgs w/ invalid response code" },
463733a535rb	{ "etm_timeout_bad", FMD_TYPE_UINT64,
464733a535rb		"ETM msgs w/ invalid timeout value" },
465733a535rb	{ "etm_evlens_bad", FMD_TYPE_UINT64,
466733a535rb		"ETM msgs w/ too many event lengths" },
467733a535rb
468733a535rb	/* IO operation failures */
469733a535rb
470733a535rb	{ "etm_xport_wr_fail", FMD_TYPE_UINT64,
471733a535rb		"xport write failures" },
472733a535rb	{ "etm_xport_rd_fail", FMD_TYPE_UINT64,
473733a535rb		"xport read failures" },
474733a535rb	{ "etm_xport_pk_fail", FMD_TYPE_UINT64,
475733a535rb		"xport peek failures" },
476733a535rb
477733a535rb	/* IO operation retries */
478733a535rb
479733a535rb	{ "etm_xport_wr_retry", FMD_TYPE_UINT64,
480733a535rb		"xport write retries" },
481733a535rb	{ "etm_xport_rd_retry", FMD_TYPE_UINT64,
482733a535rb		"xport read retries" },
483733a535rb	{ "etm_xport_pk_retry", FMD_TYPE_UINT64,
484733a535rb		"xport peek retries" },
485733a535rb
486733a535rb	/* system and library failures */
487733a535rb
488733a535rb	{ "etm_os_nvlist_pack_fail", FMD_TYPE_UINT64,
489733a535rb		"nvlist_pack failures" },
490733a535rb	{ "etm_os_nvlist_unpack_fail", FMD_TYPE_UINT64,
491733a535rb		"nvlist_unpack failures" },
492733a535rb	{ "etm_os_nvlist_size_fail", FMD_TYPE_UINT64,
493733a535rb		"nvlist_size failures" },
494733a535rb	{ "etm_os_pthread_create_fail", FMD_TYPE_UINT64,
495733a535rb		"pthread_create failures" },
496733a535rb
497733a535rb	/* transport API failures */
498733a535rb
499733a535rb	{ "etm_xport_get_ev_addrv_fail", FMD_TYPE_UINT64,
500733a535rb		"xport get event addrv API failures" },
501733a535rb	{ "etm_xport_open_fail", FMD_TYPE_UINT64,
502733a535rb		"xport open API failures" },
503733a535rb	{ "etm_xport_close_fail", FMD_TYPE_UINT64,
504733a535rb		"xport close API failures" },
505733a535rb	{ "etm_xport_accept_fail", FMD_TYPE_UINT64,
506733a535rb		"xport accept API failures" },
507733a535rb	{ "etm_xport_open_retry", FMD_TYPE_UINT64,
508733a535rb		"xport open API retries" },
509733a535rb
510733a535rb	/* FMD entry point bad arguments */
511733a535rb
512733a535rb	{ "etm_fmd_init_badargs", FMD_TYPE_UINT64,
5132535165Vuong Nguyen	    "bad arguments from fmd_init entry point" },
514733a535rb	{ "etm_fmd_fini_badargs", FMD_TYPE_UINT64,
5152535165Vuong Nguyen	    "bad arguments from fmd_fini entry point" },
5164b476eddarudy
5174b476eddarudy	/* Alert logging errors */
518b8677b7rb
5194b476eddarudy	{ "etm_log_err", FMD_TYPE_UINT64,
5204b476eddarudy		"failed to log message to log(7D)" },
5214b476eddarudy	{ "etm_msg_err", FMD_TYPE_UINT64,
5222ca9f23rb		"failed to log message to sysmsg(7D)" },
5232ca9f23rb
5242ca9f23rb	/* miscellaneous stats */
5252ca9f23rb
5262ca9f23rb	{ "etm_reset_xport", FMD_TYPE_UINT64,
5272ca9f23rb		"xport resets after xport API failure" }
528733a535rb};
529733a535rb
5302535165Vuong Nguyen
5312535165Vuong Nguyen/*
5322535165Vuong Nguyen * -------------------- global data for Root ldom-------------------------
5332535165Vuong Nguyen */
5342535165Vuong Nguyen
5352535165Vuong Nguyenldom_hdl_t
5362535165Vuong Nguyen*etm_lhp = NULL;		/* ldom pointer */
5372535165Vuong Nguyen
5382535165Vuong Nguyenstatic void *etm_dl_hdl = (void *)NULL;
5392535165Vuong Nguyenstatic const char *etm_dl_path = "libds.so.1";
5402535165Vuong Nguyenstatic int etm_dl_mode = (RTLD_NOW | RTLD_LOCAL);
5412535165Vuong Nguyen
5422535165Vuong Nguyenstatic int(*etm_ds_svc_reg)(ds_capability_t *cap, ds_ops_t *ops) =
5432535165Vuong Nguyen	(int (*)(ds_capability_t *cap, ds_ops_t *ops))NULL;
5442535165Vuong Nguyenstatic int(*etm_ds_clnt_reg)(ds_capability_t *cap, ds_ops_t *ops) =
5452535165Vuong Nguyen	(int (*)(ds_capability_t *cap, ds_ops_t *ops))NULL;
5462535165Vuong Nguyenstatic int(*etm_ds_send_msg)(ds_hdl_t hdl, void *buf, size_t buflen) =
5472535165Vuong Nguyen	(int (*)(ds_hdl_t hdl, void *buf, size_t buflen))NULL;
5482535165Vuong Nguyenstatic int(*etm_ds_recv_msg)(ds_hdl_t hdl, void *buf, size_t buflen,
5492535165Vuong Nguyen    size_t *msglen) =
5502535165Vuong Nguyen	(int (*)(ds_hdl_t hdl, void *buf, size_t buflen, size_t *msglen))NULL;
5512535165Vuong Nguyenstatic int (*etm_ds_fini)(void) = (int (*)(void))NULL;
5522535165Vuong Nguyen
5532535165Vuong Nguyenstatic pthread_mutex_t
5542535165Vuong Nguyeniosvc_list_lock =  PTHREAD_MUTEX_INITIALIZER;
5552535165Vuong Nguyen
5562535165Vuong Nguyenstatic pthread_t
5572535165Vuong Nguyenetm_async_e_tid = NULL;	/* thread id of io svc async event handler */
5582535165Vuong Nguyen
5592535165Vuong Nguyenstatic etm_proto_v1_ev_hdr_t iosvc_hdr = {
5602535165Vuong Nguyen	ETM_PROTO_MAGIC_NUM,	/* magic number */
5612535165Vuong Nguyen	ETM_PROTO_V1,		/* default to V1, not checked */
5622535165Vuong Nguyen	ETM_MSG_TYPE_FMA_EVENT,	/* Root Domain inteoduces only FMA events */
5632535165Vuong Nguyen	0,			/* sub-type */
5642535165Vuong Nguyen	0,			/* pad */
5652535165Vuong Nguyen	0,			/* add the xid at the Q send time */
5662535165Vuong Nguyen	ETM_PROTO_V1_TIMEOUT_NONE,
5672535165Vuong Nguyen	0			/* ev_lens, 0-termed, after 1 FMA event */
5682535165Vuong Nguyen};
5692535165Vuong Nguyen
5702535165Vuong Nguyen/*
5712535165Vuong Nguyen * static iosvc_list
5722535165Vuong Nguyen */
5732535165Vuong Nguyenstatic etm_iosvc_t iosvc_list[NUM_OF_ROOT_DOMAINS] = {
5742535165Vuong Nguyen	{"", 0}, {"", 0}, {"", 0}, {"", 0}, {"", 0}, {"", 0},
5752535165Vuong Nguyen	{"", 0}, {"", 0}
5762535165Vuong Nguyen};
5772535165Vuong Nguyen
5782535165Vuong Nguyenstatic etm_iosvc_t io_svc = {
5792535165Vuong Nguyen	"\0",				/* ldom_name */
5802535165Vuong Nguyen	PTHREAD_COND_INITIALIZER,	/* nudges */
5812535165Vuong Nguyen	PTHREAD_MUTEX_INITIALIZER,	/* protects the iosvc msg Q */
5822535165Vuong Nguyen	NULL,				/* iosvc msg Q head */
5832535165Vuong Nguyen	NULL,				/* iosvc msg Q tail */
5842535165Vuong Nguyen	0,				/* msg Q current length */
5852535165Vuong Nguyen	100,				/* msg Q max length */
5862535165Vuong Nguyen	0,				/* current transaction id */
5872535165Vuong Nguyen	0,				/* xid of last event posted to FMD */
5882535165Vuong Nguyen	DS_INVALID_HDL,			/* DS handle */
5892535165Vuong Nguyen	NULL,				/* fmd xprt handle */
5902535165Vuong Nguyen	NULL,				/* tid 4 send to remote RootDomain */
5912535165Vuong Nguyen	NULL,				/* tid 4 recv from remote RootDomain */
5922535165Vuong Nguyen	PTHREAD_COND_INITIALIZER,	/* nudges etm_send_to_remote_root */
5932535165Vuong Nguyen	PTHREAD_MUTEX_INITIALIZER,	/* protects msg_ack_cv */
5942535165Vuong Nguyen	0,				/* send/recv threads are not dying */
5952535165Vuong Nguyen	0,				/* flag for start sending msg Q */
5962535165Vuong Nguyen	0				/* indicate if the ACK has come  */
5972535165Vuong Nguyen};
5982535165Vuong Nguyenetm_iosvc_t *io_svc_p = &io_svc;
5992535165Vuong Nguyen
6002535165Vuong Nguyen
6012535165Vuong Nguyenstatic uint32_t
6022535165Vuong Nguyenflags;					/* flags for fmd_xprt_open */
6032535165Vuong Nguyen
6042535165Vuong Nguyenstatic etm_async_event_ele_t
6052535165Vuong Nguyenasync_event_q[ASYNC_EVENT_Q_SIZE];	/* holds the async events */
6062535165Vuong Nguyen
6072535165Vuong Nguyenstatic uint32_t
6082535165Vuong Nguyenetm_async_q_head = 0;		/* ptr to cur head of async event queue */
6092535165Vuong Nguyen
6102535165Vuong Nguyenstatic uint32_t
6112535165Vuong Nguyenetm_async_q_tail = 0;		/* ptr to cur tail of async event queue */
6122535165Vuong Nguyen
6132535165Vuong Nguyenstatic uint32_t
6142535165Vuong Nguyenetm_async_q_cur_len = 0;	/* cur length (ele cnt) of async event queue */
6152535165Vuong Nguyen
6162535165Vuong Nguyenstatic uint32_t
6172535165Vuong Nguyenetm_async_q_max_len = ASYNC_EVENT_Q_SIZE;
6182535165Vuong Nguyen				/* max length (ele cnt) of async event queue */
6192535165Vuong Nguyen
6202535165Vuong Nguyenstatic pthread_cond_t
6212535165Vuong Nguyenetm_async_event_q_cv = PTHREAD_COND_INITIALIZER;
6222535165Vuong Nguyen				/* nudges  async event handler */
6232535165Vuong Nguyen
6242535165Vuong Nguyenstatic pthread_mutex_t
6252535165Vuong Nguyenetm_async_event_q_lock = PTHREAD_MUTEX_INITIALIZER;
6262535165Vuong Nguyen				/* protects async event q */
6272535165Vuong Nguyen
6282535165Vuong Nguyenstatic ds_ver_t
6292535165Vuong Nguyenetm_iosvc_vers[] = { { 1, 0} };
6302535165Vuong Nguyen
6312535165Vuong Nguyen#define	ETM_NVERS	(sizeof (etm_iosvc_vers) / sizeof (ds_ver_t))
6322535165Vuong Nguyen
6332535165Vuong Nguyenstatic ds_capability_t
6342535165Vuong Nguyeniosvc_caps = {
6352535165Vuong Nguyen	"ETM",				/* svc_id */
6362535165Vuong Nguyen	etm_iosvc_vers,			/* vers */
6372535165Vuong Nguyen	ETM_NVERS			/* number of vers */
6382535165Vuong Nguyen};
6392535165Vuong Nguyen
6402535165Vuong Nguyenstatic void
6412535165Vuong Nguyenetm_iosvc_reg_handler(ds_hdl_t hdl, ds_cb_arg_t arg, ds_ver_t *ver,
6422535165Vuong Nguyen    ds_domain_hdl_t did);
6432535165Vuong Nguyen
6442535165Vuong Nguyenstatic void
6452535165Vuong Nguyenetm_iosvc_unreg_handler(ds_hdl_t hdl, ds_cb_arg_t arg);
6462535165Vuong Nguyen
6472535165Vuong Nguyenstatic ds_ops_t
6482535165Vuong Nguyeniosvc_ops = {
6492535165Vuong Nguyen	etm_iosvc_reg_handler,		/* ds_reg_cb */
6502535165Vuong Nguyen	etm_iosvc_unreg_handler,	/* ds_unreg_cb */
6512535165Vuong Nguyen	NULL,				/* ds_data_cb */
6522535165Vuong Nguyen	NULL				/* cb_arg */
6532535165Vuong Nguyen};
6542535165Vuong Nguyen
6552535165Vuong Nguyen
656733a535rb/*
657733a535rb * -------------------------- support functions ------------------------------
658733a535rb */
659733a535rb
660733a535rb/*
661733a535rb * Design_Note:	Each failure worth reporting to FMD should be done using
662733a535rb *		a single call to fmd_hdl_error() as it logs an FMA event
663733a535rb *		for each call. Also be aware that all the fmd_hdl_*()
664733a535rb *		format strings currently use platform specific *printf()
665733a535rb *		routines; so "%p" under Solaris does not prepend "0x" to
666733a535rb *		the outputted hex digits, while Linux and VxWorks do.
667733a535rb */
668733a535rb
6692535165Vuong Nguyen
670733a535rb/*
671e2ff4acrb * etm_show_time - display the current time of day (for debugging) using
672e2ff4acrb *		the given FMD module handle and annotation string
673e2ff4acrb */
674e2ff4acrb
675e2ff4acrbstatic void
676e2ff4acrbetm_show_time(fmd_hdl_t *hdl, char *note_str)
677e2ff4acrb{
678e2ff4acrb	struct timeval		tmv;		/* timeval */
679e2ff4acrb
680e2ff4acrb	(void) gettimeofday(&tmv, NULL);
681e2ff4acrb	fmd_hdl_debug(hdl, "info: %s: cur Unix Epoch time %d.%06d\n",
682e2ff4acrb	    note_str, tmv.tv_sec, tmv.tv_usec);
683e2ff4acrb
684e2ff4acrb} /* etm_show_time() */
685e2ff4acrb
686e2ff4acrb/*
687733a535rb * etm_hexdump - hexdump the given buffer (for debugging) using
688733a535rb *		the given FMD module handle
689733a535rb */
690733a535rb
691733a535rbstatic void
692733a535rbetm_hexdump(fmd_hdl_t *hdl, void *buf, size_t byte_cnt)
693733a535rb{
694733a535rb	uint8_t		*bp;		/* byte ptr */
695733a535rb	int		i, j;		/* index */
696733a535rb	char		cb[80];		/* char buf */
697733a535rb	unsigned int	n;		/* a byte of data for sprintf() */
698733a535rb
699733a535rb	bp = buf;
700733a535rb	j = 0;
701733a535rb
702733a535rb	/*
703733a535rb	 * Design_Note:	fmd_hdl_debug() auto adds a newline if missing;
704733a535rb	 *		hence cb exists to accumulate a longer string.
705733a535rb	 */
706733a535rb
707733a535rb	for (i = 1; i <= byte_cnt; i++) {
708733a535rb		n = *bp++;
709733a535rb		(void) sprintf(&cb[j], "%2.2x ", n);
710733a535rb		j += 3;
711733a535rb		/* add a newline every 16 bytes or at the buffer's end */
712733a535rb		if (((i % 16) == 0) || (i >= byte_cnt)) {
713733a535rb			cb[j-1] = '\0';
714733a535rb			fmd_hdl_debug(hdl, "%s\n", cb);
715733a535rb			j = 0;
716733a535rb		}
717733a535rb	} /* for each byte in the buffer */
718733a535rb
719733a535rb} /* etm_hexdump() */
720733a535rb
721733a535rb/*
722733a535rb * etm_sleep - sleep the caller for the given number of seconds,
723733a535rb *		return 0 or -errno value
724733a535rb *
725733a535rb * Design_Note:	To avoid interfering with FMD's signal mask (SIGALRM)
726733a535rb *		do not use [Solaris] sleep(3C) and instead use
727733a535rb *		pthread_cond_wait() or nanosleep(), both of which
728733a535rb *		are POSIX spec-ed to leave signal masks alone.
729733a535rb *		This is needed for Solaris and Linux (domain and SP).
730733a535rb */
731733a535rb
732733a535rbstatic int
733733a535rbetm_sleep(unsigned sleep_sec)
734733a535rb{
735733a535rb	struct timespec	tms;	/* for nanosleep() */
736733a535rb
737733a535rb	tms.tv_sec = sleep_sec;
738733a535rb	tms.tv_nsec = 0;
739733a535rb
740733a535rb	if (nanosleep(&tms, NULL) < 0) {
741733a535rb		/* errno assumed set by above call */
742733a535rb		return (-errno);
743733a535rb	}
744733a535rb	return (0);
745733a535rb
746733a535rb} /* etm_sleep() */
747733a535rb
748733a535rb/*
749733a535rb * etm_conn_open - open a connection to the given transport address,
750733a535rb *		return 0 and the opened connection handle
751733a535rb *		or -errno value
752733a535rb *
753733a535rb * caveats:	the err_substr is used in failure cases for calling
754733a535rb *		fmd_hdl_error()
755733a535rb */
756733a535rb
757733a535rbstatic int
758733a535rbetm_conn_open(fmd_hdl_t *hdl, char *err_substr,
759733a535rb		etm_xport_addr_t addr, etm_xport_conn_t *connp)
760733a535rb{
761733a535rb	etm_xport_conn_t	conn;	/* connection to return */
762733a535rb	int			nev;	/* -errno value */
763733a535rb
764733a535rb	if ((conn = etm_xport_open(hdl, addr)) == NULL) {
765733a535rb		nev = (-errno);
766733a535rb		fmd_hdl_error(hdl, "error: %s: errno %d\n",
767b8677b7rb		    err_substr, errno);
768733a535rb		etm_stats.etm_xport_open_fail.fmds_value.ui64++;
769733a535rb		return (nev);
770733a535rb	} else {
771733a535rb		*connp = conn;
772733a535rb		return (0);
773733a535rb	}
774733a535rb} /* etm_conn_open() */
775733a535rb
776733a535rb/*
777733a535rb * etm_conn_close - close the given connection,
778733a535rb *		return 0 or -errno value
779733a535rb *
780733a535rb * caveats:	the err_substr is used in failure cases for calling
781733a535rb *		fmd_hdl_error()
782733a535rb */
783733a535rb
784733a535rbstatic int
785733a535rbetm_conn_close(fmd_hdl_t *hdl, char *err_substr, etm_xport_conn_t conn)
786733a535rb{
787733a535rb	int	nev;	/* -errno value */
788733a535rb
789733a535rb	if (etm_xport_close(hdl, conn) == NULL) {
790733a535rb		nev = (-errno);
791733a535rb		fmd_hdl_error(hdl, "warning: %s: errno %d\n",
792b8677b7rb		    err_substr, errno);
793733a535rb		etm_stats.etm_xport_close_fail.fmds_value.ui64++;
794733a535rb		return (nev);
795733a535rb	} else {
796733a535rb		return (0);
797733a535rb	}
798733a535rb} /* etm_conn_close() */
799733a535rb
800733a535rb/*
801733a535rb * etm_io_op - perform an IO operation on the given connection
802733a535rb *		with the given buffer,
803733a535rb *		accommodating MTU size and retrying op if needed,
804733a535rb *		return how many bytes actually done by the op
805733a535rb *		or -errno value
806733a535rb *
807733a535rb * caveats:	the err_substr is used in failure cases for calling
808733a535rb *		fmd_hdl_error()
809733a535rb */
810733a535rb
811733a535rbstatic ssize_t
812733a535rbetm_io_op(fmd_hdl_t *hdl, char *err_substr, etm_xport_conn_t conn,
813733a535rb				void *buf, size_t byte_cnt, int io_op)
814733a535rb{
815733a535rb	ssize_t		rv;		/* ret val / byte count */
816733a535rb	ssize_t		n;		/* gen use */
817733a535rb	uint8_t		*datap;		/* ptr to data */
818733a535rb	size_t		mtu_sz;		/* MTU size in bytes */
819733a535rb	int		(*io_func_ptr)(fmd_hdl_t *, etm_xport_conn_t,
820b8677b7rb	    void *, size_t);
821733a535rb	size_t		io_sz;		/* byte count for io_func_ptr */
822733a535rb	int		try_cnt;	/* number of tries done */
823733a535rb	int		sleep_sec;	/* exp backoff sleep period in sec */
824733a535rb	int		sleep_rv;	/* ret val from sleeping */
825733a535rb	fmd_stat_t	io_retry_stat;	/* IO retry stat to update */
826733a535rb	fmd_stat_t	io_fail_stat;	/* IO failure stat to update */
827733a535rb
828733a535rb	if ((conn == NULL) || (buf == NULL)) {
829733a535rb		return (-EINVAL);
830733a535rb	}
831733a535rb	switch (io_op) {
8322535165Vuong Nguyen	case ETM_IO_OP_RD:
8332535165Vuong Nguyen		io_func_ptr = etm_xport_read;
8342535165Vuong Nguyen		io_retry_stat = etm_stats.etm_xport_rd_retry;
8352535165Vuong Nguyen		io_fail_stat = etm_stats.etm_xport_rd_fail;
8362535165Vuong Nguyen		break;
8372535165Vuong Nguyen	case ETM_IO_OP_WR:
8382535165Vuong Nguyen		io_func_ptr = etm_xport_write;
8392535165Vuong Nguyen		io_retry_stat = etm_stats.etm_xport_wr_retry;
8402535165Vuong Nguyen		io_fail_stat = etm_stats.etm_xport_wr_fail;
8412535165Vuong Nguyen		break;
8422535165Vuong Nguyen	default:
8432535165Vuong Nguyen		return (-EINVAL);
844733a535rb	}
845733a535rb	if (byte_cnt == 0) {
846733a535rb		return (byte_cnt);	/* nop */
847733a535rb	}
848733a535rb
849733a535rb	/* obtain [current] MTU size */
850733a535rb
851733a535rb	if ((n = etm_xport_get_opt(hdl, conn, ETM_XPORT_OPT_MTU_SZ)) < 0) {
852733a535rb		mtu_sz = ETM_XPORT_MTU_SZ_DEF;
853733a535rb	} else {
854733a535rb		mtu_sz = n;
855733a535rb	}
856733a535rb
857733a535rb	/* loop until all IO done, try limit exceeded, or real failure */
858733a535rb
859733a535rb	rv = 0;
860733a535rb	datap = buf;
861733a535rb	while (rv < byte_cnt) {
862733a535rb		io_sz = MIN((byte_cnt - rv), mtu_sz);
863733a535rb		try_cnt = 0;
864733a535rb		sleep_sec = 0;
865733a535rb
866733a535rb		/* when give up, return -errno value even if partly done */
867733a535rb
868733a535rb		while ((n = (*io_func_ptr)(hdl, conn, datap, io_sz)) ==
869b8677b7rb		    (-EAGAIN)) {
870733a535rb			try_cnt++;
871733a535rb			if (try_cnt > ETM_TRY_MAX_CNT) {
872733a535rb				rv = n;
873733a535rb				goto func_ret;
874733a535rb			}
875733a535rb			if (etm_is_dying) {
876733a535rb				rv = (-EINTR);
877733a535rb				goto func_ret;
878733a535rb			}
879733a535rb			if ((sleep_rv = etm_sleep(sleep_sec)) < 0) {
880733a535rb				rv = sleep_rv;
881733a535rb				goto func_ret;
882733a535rb			}
883733a535rb			sleep_sec = ((sleep_sec == 0) ? 1 :
884b8677b7rb			    (sleep_sec * ETM_TRY_BACKOFF_RATE));
885733a535rb			sleep_sec = MIN(sleep_sec, ETM_TRY_BACKOFF_CAP);
886733a535rb			io_retry_stat.fmds_value.ui64++;
887733a535rb			if (etm_debug_lvl >= 1) {
888733a535rb				fmd_hdl_debug(hdl, "info: retrying io op %d "
889b8677b7rb				    "due to EAGAIN\n", io_op);
890733a535rb			}
891733a535rb		} /* while trying the io operation */
892733a535rb
893733a535rb		if (etm_is_dying) {
894733a535rb			rv = (-EINTR);
895733a535rb			goto func_ret;
896733a535rb		}
897733a535rb		if (n < 0) {
898733a535rb			rv = n;
899733a535rb			goto func_ret;
900733a535rb		}
901733a535rb		/* avoid spinning CPU when given 0 bytes but no error */
902733a535rb		if (n == 0) {
903733a535rb			if ((sleep_rv = etm_sleep(ETM_SLEEP_QUIK)) < 0) {
904733a535rb				rv = sleep_rv;
905733a535rb				goto func_ret;
906733a535rb			}
907733a535rb		}
908733a535rb		rv += n;
909733a535rb		datap += n;
910733a535rb	} /* while still have more data */
911733a535rb
912733a535rbfunc_ret:
913733a535rb
914733a535rb	if (rv < 0) {
915733a535rb		io_fail_stat.fmds_value.ui64++;
9162ae6665jrutt		fmd_hdl_debug(hdl, "error: %s: errno %d\n",
917b8677b7rb		    err_substr, (int)(-rv));
918733a535rb	}
919733a535rb	if (etm_debug_lvl >= 3) {
920733a535rb		fmd_hdl_debug(hdl, "info: io op %d ret %d of %d\n",
921b8677b7rb		    io_op, (int)rv, (int)byte_cnt);
922733a535rb	}
923733a535rb	return (rv);
924733a535rb
925733a535rb} /* etm_io_op() */
926733a535rb
927733a535rb/*
928733a535rb * etm_magic_read - read the magic number of an ETM message header
929733a535rb *		from the given connection into the given buffer,
930733a535rb *		return 0 or -errno value
931733a535rb *
932733a535rb * Design_Note:	This routine is intended to help protect ETM from protocol
933733a535rb *		framing errors as might be caused by an SP reset / crash in
934733a535rb *		the middle of an ETM message send; the connection will be
935733a535rb *		read from for as many bytes as needed until the magic number
936733a535rb *		is found using a sliding buffer for comparisons.
937733a535rb */
938733a535rb
939733a535rbstatic int
940733a535rbetm_magic_read(fmd_hdl_t *hdl, etm_xport_conn_t conn, uint32_t *magic_ptr)
941733a535rb{
942733a535rb	int		rv;		/* ret val */
943733a535rb	uint32_t	magic_num;	/* magic number */
944733a535rb	int		byte_cnt;	/* count of bytes read */
945733a535rb	uint8_t		buf5[4+1];	/* sliding input buffer */
946733a535rb	int		i, j;		/* indices into buf5 */
947733a535rb	ssize_t		n;		/* gen use */
948733a535rb	uint8_t		drop_buf[1024];	/* dropped bytes buffer */
949733a535rb
950733a535rb	rv = 0;		/* assume success */
951733a535rb	magic_num = 0;
952733a535rb	byte_cnt = 0;
953733a535rb	j = 0;
954733a535rb
955733a535rb	/* magic number bytes are sent in network (big endian) order */
956733a535rb
957733a535rb	while (magic_num != ETM_PROTO_MAGIC_NUM) {
958733a535rb		if ((n = etm_io_op(hdl, "bad io read on magic",
959b8677b7rb		    conn, &buf5[j], 1, ETM_IO_OP_RD)) < 0) {
960733a535rb			rv = n;
961733a535rb			goto func_ret;
962733a535rb		}
963733a535rb		byte_cnt++;
964733a535rb		j = MIN((j + 1), sizeof (magic_num));
965733a535rb		if (byte_cnt < sizeof (magic_num)) {
966733a535rb			continue;
967733a535rb		}
968733a535rb
969733a535rb		if (byte_cnt > sizeof (magic_num)) {
970733a535rb			etm_stats.etm_magic_drop_bytes.fmds_value.ui64++;
971733a535rb			i = MIN(byte_cnt - j - 1, sizeof (drop_buf) - 1);
972733a535rb			drop_buf[i] = buf5[0];
973733a535rb			for (i = 0; i < j; i++) {
974733a535rb				buf5[i] = buf5[i+1];
975733a535rb			} /* for sliding the buffer contents */
976733a535rb		}
977733a535rb		(void) memcpy(&magic_num, &buf5[0], sizeof (magic_num));
978733a535rb		magic_num = ntohl(magic_num);
979733a535rb	} /* for reading bytes until find magic number */
980733a535rb
981733a535rbfunc_ret:
982733a535rb
983733a535rb	if (byte_cnt != sizeof (magic_num)) {
9842ae6665jrutt		fmd_hdl_debug(hdl, "warning: bad proto frame "
985b8677b7rb		    "implies corrupt/lost msg(s)\n");
986733a535rb	}
987733a535rb	if ((byte_cnt > sizeof (magic_num)) && (etm_debug_lvl >= 2)) {
988733a535rb		i = MIN(byte_cnt - sizeof (magic_num), sizeof (drop_buf));
989733a535rb		fmd_hdl_debug(hdl, "info: magic drop hexdump "
990b8677b7rb		    "first %d of %d bytes:\n", i,
991b8677b7rb		    byte_cnt - sizeof (magic_num));
992733a535rb		etm_hexdump(hdl, drop_buf, i);
993733a535rb	}
994733a535rb
995733a535rb	if (rv == 0) {
996733a535rb		*magic_ptr = magic_num;
997733a535rb	}
998733a535rb	return (rv);
999733a535rb
1000733a535rb} /* etm_magic_read() */
1001733a535rb
1002733a535rb/*
1003733a535rb * etm_hdr_read - allocate, read, and validate a [variable sized]
1004733a535rb *		ETM message header from the given connection,
1005733a535rb *		return the allocated ETM message header
1006733a535rb *		(which is guaranteed to be large enough to reuse as a
1007733a535rb *		RESPONSE msg hdr) and its size
1008733a535rb *		or NULL and set errno on failure
1009733a535rb */
1010733a535rb
1011733a535rbstatic void *
1012733a535rbetm_hdr_read(fmd_hdl_t *hdl, etm_xport_conn_t conn, size_t *szp)
1013733a535rb{
1014733a535rb	uint8_t			*hdrp;		/* ptr to header to return */
1015733a535rb	size_t			hdr_sz;		/* sizeof *hdrp */
1016733a535rb	etm_proto_v1_pp_t	pp; 		/* protocol preamble */
1017733a535rb	etm_proto_v1_ev_hdr_t	*ev_hdrp;	/* for FMA_EVENT msg */
1018733a535rb	etm_proto_v1_ctl_hdr_t	*ctl_hdrp;	/* for CONTROL msg */
1019733a535rb	etm_proto_v1_resp_hdr_t *resp_hdrp;	/* for RESPONSE msg */
10204b476eddarudy	etm_proto_v3_sa_hdr_t	*sa_hdrp;	/* for ALERT msg */
1021733a535rb	uint32_t		*lenp;		/* ptr to FMA event length */
1022733a535rb	ssize_t			i, n;		/* gen use */
1023733a535rb	uint8_t	misc_buf[ETM_MISC_BUF_SZ];	/* for var sized hdrs */
1024733a535rb	int			dummy_int;	/* dummy var to appease lint */
1025733a535rb
1026733a535rb	hdrp = NULL; hdr_sz = 0;
1027733a535rb
1028733a535rb	/* read the magic number which starts the protocol preamble */
1029733a535rb
1030733a535rb	if ((n = etm_magic_read(hdl, conn, &pp.pp_magic_num)) < 0) {
1031733a535rb		errno = (-n);
1032733a535rb		etm_stats.etm_magic_bad.fmds_value.ui64++;
1033733a535rb		return (NULL);
1034733a535rb	}
1035733a535rb
1036733a535rb	/* read the rest of the protocol preamble all at once */
1037733a535rb
1038733a535rb	if ((n = etm_io_op(hdl, "bad io read on preamble",
1039b8677b7rb	    conn, &pp.pp_proto_ver, sizeof (pp) - sizeof (pp.pp_magic_num),
1040b8677b7rb	    ETM_IO_OP_RD)) < 0) {
1041733a535rb		errno = (-n);
1042733a535rb		return (NULL);
1043733a535rb	}
1044733a535rb
1045733a535rb	/*
1046733a535rb	 * Design_Note:	The magic number was already network decoded; but
1047733a535rb	 *		some other preamble fields also need to be decoded,
1048733a535rb	 *		specifically pp_xid and pp_timeout. The rest of the
1049733a535rb	 *		preamble fields are byte sized and hence need no
1050733a535rb	 *		decoding.
1051733a535rb	 */
1052733a535rb
1053733a535rb	pp.pp_xid = ntohl(pp.pp_xid);
1054733a535rb	pp.pp_timeout = ntohl(pp.pp_timeout);
1055733a535rb
1056733a535rb	/* sanity check the header as best we can */
1057733a535rb
1058e2ff4acrb	if ((pp.pp_proto_ver < ETM_PROTO_V1) ||
10594b476eddarudy	    (pp.pp_proto_ver > ETM_PROTO_V3)) {
1060733a535rb		fmd_hdl_error(hdl, "error: bad proto ver %d\n",
1061b8677b7rb		    (int)pp.pp_proto_ver);
1062733a535rb		errno = EPROTO;
1063733a535rb		etm_stats.etm_ver_bad.fmds_value.ui64++;
1064733a535rb		return (NULL);
1065733a535rb	}
1066733a535rb
1067733a535rb	dummy_int = pp.pp_msg_type;
1068733a535rb	if ((dummy_int <= ETM_MSG_TYPE_TOO_LOW) ||
1069733a535rb	    (dummy_int >= ETM_MSG_TYPE_TOO_BIG)) {
1070733a535rb		fmd_hdl_error(hdl, "error: bad msg type %d", dummy_int);
1071733a535rb		errno = EBADMSG;
1072733a535rb		etm_stats.etm_msgtype_bad.fmds_value.ui64++;
1073733a535rb		return (NULL);
1074733a535rb	}
1075733a535rb
1076733a535rb	/* handle [var sized] hdrs for FMA_EVENT, CONTROL, RESPONSE msgs */
1077733a535rb
1078733a535rb	if (pp.pp_msg_type == ETM_MSG_TYPE_FMA_EVENT) {
1079733a535rb
1080733a535rb		ev_hdrp = (void*)&misc_buf[0];
1081733a535rb		hdr_sz = sizeof (*ev_hdrp);
1082733a535rb		(void) memcpy(&ev_hdrp->ev_pp, &pp, sizeof (pp));
1083733a535rb
1084733a535rb		/* sanity check the header's timeout */
1085733a535rb
1086e2ff4acrb		if ((ev_hdrp->ev_pp.pp_proto_ver == ETM_PROTO_V1) &&
1087e2ff4acrb		    (ev_hdrp->ev_pp.pp_timeout != ETM_PROTO_V1_TIMEOUT_NONE)) {
1088733a535rb			errno = ETIME;
1089733a535rb			etm_stats.etm_timeout_bad.fmds_value.ui64++;
1090733a535rb			return (NULL);
1091733a535rb		}
1092733a535rb
1093733a535rb		/* get all FMA event lengths from the header */
1094733a535rb
1095733a535rb		lenp = (uint32_t *)&ev_hdrp->ev_lens[0]; lenp--;
1096733a535rb		i = -1;	/* cnt of length entries preceding 0 */
1097733a535rb		do {
1098733a535rb			i++; lenp++;
1099733a535rb			if ((sizeof (*ev_hdrp) + (i * sizeof (*lenp))) >=
1100b8677b7rb			    ETM_MISC_BUF_SZ) {
1101733a535rb				errno = E2BIG;	/* ridiculous size */
1102733a535rb				etm_stats.etm_evlens_bad.fmds_value.ui64++;
1103733a535rb				return (NULL);
1104733a535rb			}
1105733a535rb			if ((n = etm_io_op(hdl, "bad io read on event len",
1106b8677b7rb			    conn, lenp, sizeof (*lenp), ETM_IO_OP_RD)) < 0) {
1107733a535rb				errno = (-n);
1108733a535rb				return (NULL);
1109733a535rb			}
1110733a535rb			*lenp = ntohl(*lenp);
1111733a535rb
1112733a535rb		} while (*lenp != 0);
1113733a535rb		i += 0; /* first len already counted by sizeof(ev_hdr) */
1114733a535rb		hdr_sz += (i * sizeof (*lenp));
1115733a535rb
1116733a535rb		etm_stats.etm_rd_hdr_fmaevent.fmds_value.ui64++;
1117733a535rb
1118733a535rb	} else if (pp.pp_msg_type == ETM_MSG_TYPE_CONTROL) {
1119733a535rb
1120733a535rb		ctl_hdrp = (void*)&misc_buf[0];
1121733a535rb		hdr_sz = sizeof (*ctl_hdrp);
1122733a535rb		(void) memcpy(&ctl_hdrp->ctl_pp, &pp, sizeof (pp));
1123733a535rb
1124733a535rb		/* sanity check the header's sub type (control selector) */
1125733a535rb
1126733a535rb		if ((ctl_hdrp->ctl_pp.pp_sub_type <= ETM_CTL_SEL_TOO_LOW) ||
1127733a535rb		    (ctl_hdrp->ctl_pp.pp_sub_type >= ETM_CTL_SEL_TOO_BIG)) {
1128733a535rb			fmd_hdl_error(hdl, "error: bad ctl sub type %d\n",
1129b8677b7rb			    (int)ctl_hdrp->ctl_pp.pp_sub_type);
1130733a535rb			errno = EBADMSG;
1131733a535rb			etm_stats.etm_subtype_bad.fmds_value.ui64++;
1132733a535rb			return (NULL);
1133733a535rb		}
1134733a535rb
1135733a535rb		/* get the control length */
1136733a535rb
1137733a535rb		if ((n = etm_io_op(hdl, "bad io read on ctl len",
1138b8677b7rb		    conn, &ctl_hdrp->ctl_len, sizeof (ctl_hdrp->ctl_len),
1139b8677b7rb		    ETM_IO_OP_RD)) < 0) {
1140733a535rb			errno = (-n);
1141733a535rb			return (NULL);
1142733a535rb		}
1143733a535rb
1144733a535rb		ctl_hdrp->ctl_len = ntohl(ctl_hdrp->ctl_len);
1145733a535rb
1146733a535rb		etm_stats.etm_rd_hdr_control.fmds_value.ui64++;
1147733a535rb
1148733a535rb	} else if (pp.pp_msg_type == ETM_MSG_TYPE_RESPONSE) {
1149733a535rb
1150733a535rb		resp_hdrp = (void*)&misc_buf[0];
1151733a535rb		hdr_sz = sizeof (*resp_hdrp);
1152733a535rb		(void) memcpy(&resp_hdrp->resp_pp, &pp, sizeof (pp));
1153733a535rb
1154733a535rb		/* sanity check the header's timeout */
1155733a535rb
1156733a535rb		if (resp_hdrp->resp_pp.pp_timeout !=
1157b8677b7rb		    ETM_PROTO_V1_TIMEOUT_NONE) {
1158733a535rb			errno = ETIME;
1159733a535rb			etm_stats.etm_timeout_bad.fmds_value.ui64++;
1160733a535rb			return (NULL);
1161733a535rb		}
1162733a535rb
1163733a535rb		/* get the response code and length */
1164733a535rb
1165733a535rb		if ((n = etm_io_op(hdl, "bad io read on resp code+len",
1166b8677b7rb		    conn, &resp_hdrp->resp_code,
1167b8677b7rb		    sizeof (resp_hdrp->resp_code)
1168b8677b7rb		    + sizeof (resp_hdrp->resp_len),
1169b8677b7rb		    ETM_IO_OP_RD)) < 0) {
1170733a535rb			errno = (-n);
1171733a535rb			return (NULL);
1172733a535rb		}
1173733a535rb
1174733a535rb		resp_hdrp->resp_code = ntohl(resp_hdrp->resp_code);
1175733a535rb		resp_hdrp->resp_len = ntohl(resp_hdrp->resp_len);
1176733a535rb
1177733a535rb		etm_stats.etm_rd_hdr_response.fmds_value.ui64++;
1178733a535rb
11794b476eddarudy	} else if (pp.pp_msg_type == ETM_MSG_TYPE_ALERT) {
11804b476eddarudy
11814b476eddarudy		sa_hdrp = (void*)&misc_buf[0];
11824b476eddarudy		hdr_sz = sizeof (*sa_hdrp);
11834b476eddarudy		(void) memcpy(&sa_hdrp->sa_pp, &pp, sizeof (pp));
11844b476eddarudy
11854b476eddarudy		/* sanity check the header's protocol version */
11864b476eddarudy
11874b476eddarudy		if (sa_hdrp->sa_pp.pp_proto_ver != ETM_PROTO_V3) {
11884b476eddarudy			errno = EPROTO;
11894b476eddarudy			etm_stats.etm_ver_bad.fmds_value.ui64++;
11904b476eddarudy			return (NULL);
11914b476eddarudy		}
11924b476eddarudy
11934b476eddarudy		/* get the priority and length */
11944b476eddarudy
11954b476eddarudy		if ((n = etm_io_op(hdl, "bad io read on sa priority+len",
1196