xref: /illumos-gate/usr/src/cmd/idmap/idmapd/idmapd.c (revision fea136a0)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright 2018 Nexenta Systems, Inc.  All rights reserved.
24  */
25 
26 
27 /*
28  * main() of idmapd(1M)
29  */
30 
31 #include "idmapd.h"
32 #include <atomic.h>
33 #include <signal.h>
34 #include <rpc/pmap_clnt.h> /* for pmap_unset */
35 #include <string.h> /* strcmp */
36 #include <unistd.h> /* setsid */
37 #include <sys/types.h>
38 #include <memory.h>
39 #include <stropts.h>
40 #include <netconfig.h>
41 #include <sys/resource.h> /* rlimit */
42 #include <rpcsvc/daemon_utils.h> /* DAEMON_UID and DAEMON_GID */
43 #include <priv_utils.h> /* privileges */
44 #include <locale.h>
45 #include <sys/systeminfo.h>
46 #include <errno.h>
47 #include <sys/wait.h>
48 #include <sys/time.h>
49 #include <zone.h>
50 #include <door.h>
51 #include <port.h>
52 #include <tsol/label.h>
53 #include <sys/resource.h>
54 #include <sys/sid.h>
55 #include <sys/idmap.h>
56 #include <pthread.h>
57 #include <stdarg.h>
58 #include <assert.h>
59 #include <note.h>
60 
61 #define	CBUFSIZ 26	/* ctime(3c) */
62 
63 static void	term_handler(int);
64 static void	init_idmapd();
65 static void	fini_idmapd();
66 
67 /* The DC Locator lives inside idmap (for now). */
68 extern void	init_dc_locator(void);
69 extern void	fini_dc_locator(void);
70 
71 idmapd_state_t	_idmapdstate;
72 
73 SVCXPRT *xprt = NULL;
74 
75 static int dfd = -1;		/* our door server fildes, for unregistration */
76 static boolean_t degraded = B_FALSE;
77 
78 
79 static uint32_t		num_threads = 0;
80 static pthread_key_t	create_threads_key;
81 static uint32_t		max_threads = 40;
82 
83 /*
84  * Server door thread start routine.
85  *
86  * Set a TSD value to the door thread. This enables the destructor to
87  * be called when this thread exits. Note that we need a non-NULL
88  * value for this or the TSD destructor is not called.
89  */
90 /*ARGSUSED*/
91 static void *
idmapd_door_thread_start(void * arg)92 idmapd_door_thread_start(void *arg)
93 {
94 	static void *value = "NON-NULL TSD";
95 
96 	/*
97 	 * Disable cancellation to avoid memory leaks from not running
98 	 * the thread cleanup code.
99 	 */
100 	(void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL);
101 	(void) pthread_setspecific(create_threads_key, value);
102 	(void) door_return(NULL, 0, NULL, 0);
103 
104 	/* make lint happy */
105 	return (NULL);
106 }
107 
108 /*
109  * Server door threads creation
110  */
111 /*ARGSUSED*/
112 static void
idmapd_door_thread_create(door_info_t * dip)113 idmapd_door_thread_create(door_info_t *dip)
114 {
115 	int		num;
116 	pthread_t	thread_id;
117 
118 	if ((num = atomic_inc_32_nv(&num_threads)) > max_threads) {
119 		atomic_dec_32(&num_threads);
120 		idmapdlog(LOG_DEBUG,
121 		    "thread creation refused - %d threads currently active",
122 		    num - 1);
123 		return;
124 	}
125 	(void) pthread_create(&thread_id, NULL, idmapd_door_thread_start, NULL);
126 	idmapdlog(LOG_DEBUG,
127 	    "created thread ID %d - %d threads currently active",
128 	    thread_id, num);
129 }
130 
131 /*
132  * Server door thread cleanup
133  */
134 /*ARGSUSED*/
135 static void
idmapd_door_thread_cleanup(void * arg)136 idmapd_door_thread_cleanup(void *arg)
137 {
138 	int num;
139 
140 	/* set TSD to NULL so we don't loop infinitely */
141 	(void) pthread_setspecific(create_threads_key, NULL);
142 	num = atomic_dec_32_nv(&num_threads);
143 	idmapdlog(LOG_DEBUG,
144 	    "exiting thread ID %d - %d threads currently active",
145 	    pthread_self(), num);
146 }
147 
148 /*
149  * This is needed for mech_krb5 -- we run as daemon, yes, but we want
150  * mech_krb5 to think we're root so it can get host/nodename.fqdn
151  * tickets for us so we can authenticate to AD as the machine account
152  * that we are.  For more details look at the entry point in mech_krb5
153  * corresponding to gss_init_sec_context().
154  *
155  * As a side effect of faking our effective UID to mech_krb5 we will use
156  * root's default ccache (/tmp/krb5cc_0).  But if that's created by
157  * another process then we won't have access to it: we run as daemon and
158  * keep PRIV_FILE_DAC_READ, which is insufficient to share the ccache
159  * with others.  We putenv("KRB5CCNAME=/var/run/idmap/ccache") in main()
160  * to avoid this issue; see main().
161  *
162  * Someday we'll have gss/mech_krb5 extensions for acquiring initiator
163  * creds with keytabs/raw keys, and someday we'll have extensions to
164  * libsasl to specify creds/name to use on the initiator side, and
165  * someday we'll have extensions to libldap to pass those through to
166  * libsasl.  Until then this interposer will have to do.
167  *
168  * Also, we have to tell lint to shut up: it thinks app_krb5_user_uid()
169  * is defined but not used.
170  */
171 /*LINTLIBRARY*/
172 uid_t
app_krb5_user_uid(void)173 app_krb5_user_uid(void)
174 {
175 	return (0);
176 }
177 
178 /*ARGSUSED*/
179 static void
term_handler(int sig)180 term_handler(int sig)
181 {
182 	idmapdlog(LOG_INFO, "Terminating.");
183 	fini_dc_locator();
184 	fini_idmapd();
185 	_exit(0);
186 }
187 
188 /*ARGSUSED*/
189 static void
usr1_handler(int sig)190 usr1_handler(int sig)
191 {
192 	NOTE(ARGUNUSED(sig))
193 	print_idmapdstate();
194 }
195 
196 static int pipe_fd = -1;
197 
198 static void
daemonize_ready(void)199 daemonize_ready(void)
200 {
201 	char data = '\0';
202 	/*
203 	 * wake the parent
204 	 */
205 	(void) write(pipe_fd, &data, 1);
206 	(void) close(pipe_fd);
207 }
208 
209 static int
daemonize_start(void)210 daemonize_start(void)
211 {
212 	char	data;
213 	int	status;
214 	int	devnull;
215 	int	filedes[2];
216 	pid_t	pid;
217 
218 	(void) sigset(SIGPIPE, SIG_IGN);
219 	devnull = open("/dev/null", O_RDONLY);
220 	if (devnull < 0)
221 		return (-1);
222 	(void) dup2(devnull, 0);
223 	(void) dup2(2, 1);	/* stderr only */
224 	if (pipe(filedes) < 0)
225 		return (-1);
226 	if ((pid = fork1()) < 0)
227 		return (-1);
228 	if (pid != 0) {
229 		/*
230 		 * parent
231 		 */
232 		(void) close(filedes[1]);
233 		if (read(filedes[0], &data, 1) == 1) {
234 			/* presume success */
235 			_exit(0);
236 		}
237 		status = -1;
238 		(void) wait4(pid, &status, 0, NULL);
239 		if (WIFEXITED(status))
240 			_exit(WEXITSTATUS(status));
241 		else
242 			_exit(-1);
243 	}
244 
245 	/*
246 	 * child
247 	 */
248 	pipe_fd = filedes[1];
249 	(void) close(filedes[0]);
250 	(void) setsid();
251 	(void) umask(0077);
252 	openlog("idmap", LOG_PID, LOG_DAEMON);
253 
254 	return (0);
255 }
256 
257 
258 int
main(int argc,char ** argv)259 main(int argc, char **argv)
260 {
261 	int c;
262 	struct rlimit rl;
263 
264 	if (rwlock_init(&_idmapdstate.rwlk_cfg, USYNC_THREAD, NULL) != 0)
265 		return (-1);
266 	if (mutex_init(&_idmapdstate.addisc_lk, USYNC_THREAD, NULL) != 0)
267 		return (-1);
268 	if (cond_init(&_idmapdstate.addisc_cv, USYNC_THREAD, NULL) != 0)
269 		return (-1);
270 
271 	_idmapdstate.daemon_mode = TRUE;
272 	while ((c = getopt(argc, argv, "d")) != -1) {
273 		switch (c) {
274 			case 'd':
275 				_idmapdstate.daemon_mode = FALSE;
276 				break;
277 			default:
278 				(void) fprintf(stderr,
279 				    "Usage: /usr/lib/idmapd [-d]\n");
280 				return (SMF_EXIT_ERR_CONFIG);
281 		}
282 	}
283 
284 	/* set locale and domain for internationalization */
285 	(void) setlocale(LC_ALL, "");
286 	(void) textdomain(TEXT_DOMAIN);
287 
288 	idmap_set_logger(idmapdlog);
289 	adutils_set_logger(idmapdlog);
290 
291 	if (is_system_labeled() && getzoneid() != GLOBAL_ZONEID) {
292 		idmapdlog(LOG_ERR,
293 		    "with Trusted Extensions idmapd runs only in the "
294 		    "global zone");
295 		exit(1);
296 	}
297 
298 	/*
299 	 * Raise the fd limit to max
300 	 */
301 	if (getrlimit(RLIMIT_NOFILE, &rl) != 0) {
302 		idmapdlog(LOG_ERR, "getrlimit failed");
303 	} else if (rl.rlim_cur < rl.rlim_max) {
304 		rl.rlim_cur = rl.rlim_max;
305 		if (setrlimit(RLIMIT_NOFILE, &rl) != 0)
306 			idmapdlog(LOG_ERR,
307 			    "Unable to raise RLIMIT_NOFILE to %d",
308 			    rl.rlim_cur);
309 	}
310 
311 	(void) mutex_init(&_svcstate_lock, USYNC_THREAD, NULL);
312 
313 	if (_idmapdstate.daemon_mode == TRUE) {
314 		if (daemonize_start() < 0) {
315 			idmapdlog(LOG_ERR, "unable to daemonize");
316 			exit(-1);
317 		}
318 	} else
319 		(void) umask(0077);
320 
321 	idmap_init_tsd_key();
322 
323 	init_idmapd();
324 	init_dc_locator();
325 
326 	/* signal handlers that should run only after we're initialized */
327 	(void) sigset(SIGTERM, term_handler);
328 	(void) sigset(SIGUSR1, usr1_handler);
329 	(void) sigset(SIGHUP, idmap_cfg_hup_handler);
330 
331 	if (__init_daemon_priv(PU_RESETGROUPS|PU_CLEARLIMITSET,
332 	    DAEMON_UID, DAEMON_GID,
333 	    PRIV_PROC_AUDIT, PRIV_FILE_DAC_READ,
334 	    (char *)NULL) == -1) {
335 		idmapdlog(LOG_ERR, "unable to drop privileges");
336 		exit(1);
337 	}
338 
339 	__fini_daemon_priv(PRIV_PROC_FORK, PRIV_PROC_EXEC, PRIV_PROC_SESSION,
340 	    PRIV_FILE_LINK_ANY, PRIV_PROC_INFO, (char *)NULL);
341 
342 	if (_idmapdstate.daemon_mode == TRUE)
343 		daemonize_ready();
344 
345 	/* With doors RPC this just wastes this thread, oh well */
346 	svc_run();
347 	return (0);
348 }
349 
350 static void
init_idmapd()351 init_idmapd()
352 {
353 	int	error;
354 	int	connmaxrec = IDMAP_MAX_DOOR_RPC;
355 
356 
357 	/* create directories as root and chown to daemon uid */
358 	if (create_directory(IDMAP_DBDIR, DAEMON_UID, DAEMON_GID) < 0)
359 		exit(1);
360 	if (create_directory(IDMAP_CACHEDIR, DAEMON_UID, DAEMON_GID) < 0)
361 		exit(1);
362 
363 	/*
364 	 * Set KRB5CCNAME in the environment.  See app_krb5_user_uid()
365 	 * for more details.  We blow away the existing one, if there is
366 	 * one.
367 	 */
368 	(void) unlink(IDMAP_CACHEDIR "/ccache");
369 	(void) putenv("KRB5CCNAME=" IDMAP_CACHEDIR "/ccache");
370 	(void) putenv("MS_INTEROP=1");
371 
372 	if (sysinfo(SI_HOSTNAME, _idmapdstate.hostname,
373 	    sizeof (_idmapdstate.hostname)) == -1) {
374 		error = errno;
375 		idmapdlog(LOG_ERR, "unable to determine hostname, error: %d",
376 		    error);
377 		exit(1);
378 	}
379 
380 	if ((error = init_mapping_system()) < 0) {
381 		idmapdlog(LOG_ERR, "unable to initialize mapping system");
382 		exit(error < -2 ? SMF_EXIT_ERR_CONFIG : 1);
383 	}
384 
385 	/*
386 	 * This means max_threads can't be updated without restarting idmap.
387 	 */
388 	RDLOCK_CONFIG();
389 	max_threads = _idmapdstate.cfg->pgcfg.max_threads;
390 	UNLOCK_CONFIG();
391 
392 	(void) door_server_create(idmapd_door_thread_create);
393 	if ((error = pthread_key_create(&create_threads_key,
394 	    idmapd_door_thread_cleanup)) != 0) {
395 		idmapdlog(LOG_ERR, "unable to create threads key (%s)",
396 		    strerror(error));
397 		goto errout;
398 	}
399 
400 	xprt = svc_door_create(idmap_prog_1, IDMAP_PROG, IDMAP_V1, connmaxrec);
401 	if (xprt == NULL) {
402 		idmapdlog(LOG_ERR, "unable to create door RPC service");
403 		goto errout;
404 	}
405 
406 	if (!svc_control(xprt, SVCSET_CONNMAXREC, &connmaxrec)) {
407 		idmapdlog(LOG_ERR, "unable to limit RPC request size");
408 		goto errout;
409 	}
410 
411 	dfd = xprt->xp_fd;
412 
413 	if (dfd == -1) {
414 		idmapdlog(LOG_ERR, "unable to register door");
415 		goto errout;
416 	}
417 	if ((error = __idmap_reg(dfd)) != 0) {
418 		idmapdlog(LOG_ERR, "unable to register door (%s)",
419 		    strerror(errno));
420 		goto errout;
421 	}
422 
423 	if ((error = allocids(_idmapdstate.new_eph_db,
424 	    8192, &_idmapdstate.next_uid,
425 	    8192, &_idmapdstate.next_gid)) != 0) {
426 		idmapdlog(LOG_ERR, "unable to allocate ephemeral IDs (%s)",
427 		    strerror(errno));
428 		_idmapdstate.next_uid = IDMAP_SENTINEL_PID;
429 		_idmapdstate.limit_uid = IDMAP_SENTINEL_PID;
430 		_idmapdstate.next_gid = IDMAP_SENTINEL_PID;
431 		_idmapdstate.limit_gid = IDMAP_SENTINEL_PID;
432 	} else {
433 		_idmapdstate.limit_uid = _idmapdstate.next_uid + 8192;
434 		_idmapdstate.limit_gid = _idmapdstate.next_gid + 8192;
435 	}
436 
437 	if (DBG(CONFIG, 1))
438 		print_idmapdstate();
439 
440 	return;
441 
442 errout:
443 	fini_idmapd();
444 	exit(1);
445 }
446 
447 static void
fini_idmapd()448 fini_idmapd()
449 {
450 	(void) __idmap_unreg(dfd);
451 	fini_mapping_system();
452 	if (xprt != NULL)
453 		svc_destroy(xprt);
454 }
455 
456 static
457 const char *
get_fmri(void)458 get_fmri(void)
459 {
460 	static char *fmri = NULL;
461 	static char buf[60];
462 	char *s;
463 
464 	membar_consumer();
465 	s = fmri;
466 	if (s != NULL && *s == '\0')
467 		return (NULL);
468 	else if (s != NULL)
469 		return (s);
470 
471 	if ((s = getenv("SMF_FMRI")) == NULL || strlen(s) >= sizeof (buf))
472 		buf[0] = '\0';
473 	else
474 		(void) strlcpy(buf, s, sizeof (buf));
475 
476 	membar_producer();
477 	fmri = buf;
478 
479 	return (get_fmri());
480 }
481 
482 /*
483  * Wrappers for smf_degrade/restore_instance()
484  *
485  * smf_restore_instance() is too heavy duty to be calling every time we
486  * have a successful AD name<->SID lookup.
487  */
488 void
degrade_svc(int poke_discovery,const char * reason)489 degrade_svc(int poke_discovery, const char *reason)
490 {
491 	const char *fmri;
492 
493 	membar_consumer();
494 	if (degraded)
495 		return;
496 
497 	idmapdlog(LOG_ERR, "Degraded operation (%s).", reason);
498 
499 	membar_producer();
500 	degraded = B_TRUE;
501 
502 	if ((fmri = get_fmri()) != NULL)
503 		(void) smf_degrade_instance(fmri, 0);
504 
505 	/*
506 	 * If the config update thread is in a state where auto-discovery could
507 	 * be re-tried, then this will make it try it -- a sort of auto-refresh.
508 	 */
509 	if (poke_discovery)
510 		idmap_cfg_poke_updates();
511 }
512 
513 void
restore_svc(void)514 restore_svc(void)
515 {
516 	const char *fmri;
517 
518 	membar_consumer();
519 	if (!degraded)
520 		return;
521 
522 	if ((fmri = get_fmri()) == NULL)
523 		(void) smf_restore_instance(fmri);
524 
525 	membar_producer();
526 	degraded = B_FALSE;
527 
528 	idmapdlog(LOG_NOTICE, "Normal operation restored");
529 }
530 
531 
532 /* printflike */
533 void
idmapdlog(int pri,const char * format,...)534 idmapdlog(int pri, const char *format, ...)
535 {
536 	static time_t prev_ts;
537 	va_list args;
538 	char cbuf[CBUFSIZ];
539 	time_t ts;
540 
541 	ts = time(NULL);
542 	if (prev_ts != ts) {
543 		prev_ts = ts;
544 		/* NB: cbuf has \n */
545 		(void) fprintf(stderr, "@ %s",
546 		    ctime_r(&ts, cbuf, sizeof (cbuf)));
547 	}
548 
549 	va_start(args, format);
550 	(void) vfprintf(stderr, format, args);
551 	(void) fprintf(stderr, "\n");
552 	va_end(args);
553 
554 	/*
555 	 * We don't want to fill up the logs with useless messages when
556 	 * we're degraded, but we still want to log.
557 	 */
558 	if (degraded)
559 		pri = LOG_DEBUG;
560 
561 	va_start(args, format);
562 	vsyslog(pri, format, args);
563 	va_end(args);
564 }
565 
566 static void
trace_str(nvlist_t * entry,char * n1,char * n2,char * str)567 trace_str(nvlist_t *entry, char *n1, char *n2, char *str)
568 {
569 	char name[IDMAP_TRACE_NAME_MAX+1];	/* Max used is only about 11 */
570 
571 	(void) strlcpy(name, n1, sizeof (name));
572 	if (n2 != NULL)
573 		(void) strlcat(name, n2, sizeof (name));
574 
575 	(void) nvlist_add_string(entry, name, str);
576 }
577 
578 static void
trace_int(nvlist_t * entry,char * n1,char * n2,int64_t i)579 trace_int(nvlist_t *entry, char *n1, char *n2, int64_t i)
580 {
581 	char name[IDMAP_TRACE_NAME_MAX+1];	/* Max used is only about 11 */
582 
583 	(void) strlcpy(name, n1, sizeof (name));
584 	if (n2 != NULL)
585 		(void) strlcat(name, n2, sizeof (name));
586 
587 	(void) nvlist_add_int64(entry, name, i);
588 }
589 
590 static void
trace_sid(nvlist_t * entry,char * n1,char * n2,idmap_sid * sid)591 trace_sid(nvlist_t *entry, char *n1, char *n2, idmap_sid *sid)
592 {
593 	char *str;
594 
595 	(void) asprintf(&str, "%s-%u", sid->prefix, sid->rid);
596 	if (str == NULL)
597 		return;
598 
599 	trace_str(entry, n1, n2, str);
600 	free(str);
601 }
602 
603 static void
trace_id(nvlist_t * entry,char * fromto,idmap_id * id,char * name,char * domain)604 trace_id(nvlist_t *entry, char *fromto, idmap_id *id, char *name, char *domain)
605 {
606 	trace_int(entry, fromto, IDMAP_TRACE_TYPE, (int64_t)id->idtype);
607 	if (IS_ID_SID(*id)) {
608 		if (name != NULL) {
609 			char *str;
610 
611 			(void) asprintf(&str, "%s%s%s", name,
612 			    domain == NULL ? "" : "@",
613 			    domain == NULL ? "" : domain);
614 			if (str != NULL) {
615 				trace_str(entry, fromto, IDMAP_TRACE_NAME, str);
616 				free(str);
617 			}
618 		}
619 		if (id->idmap_id_u.sid.prefix != NULL) {
620 			trace_sid(entry, fromto, IDMAP_TRACE_SID,
621 			    &id->idmap_id_u.sid);
622 		}
623 	} else if (IS_ID_POSIX(*id)) {
624 		if (name != NULL)
625 			trace_str(entry, fromto, IDMAP_TRACE_NAME, name);
626 		if (id->idmap_id_u.uid != IDMAP_SENTINEL_PID) {
627 			trace_int(entry, fromto, IDMAP_TRACE_UNIXID,
628 			    (int64_t)id->idmap_id_u.uid);
629 		}
630 	}
631 }
632 
633 /*
634  * Record a trace event.  TRACE() has already decided whether or not
635  * tracing is required; what we do here is collect the data and send it
636  * to its destination - to the trace log in the response, if
637  * IDMAP_REQ_FLG_TRACE is set, and to the SMF service log, if debug/mapping
638  * is greater than zero.
639  */
640 int
trace(idmap_mapping * req,idmap_id_res * res,char * fmt,...)641 trace(idmap_mapping *req, idmap_id_res *res, char *fmt, ...)
642 {
643 	va_list va;
644 	char *buf;
645 	int err;
646 	nvlist_t *entry;
647 
648 	assert(req != NULL);
649 	assert(res != NULL);
650 
651 	err = nvlist_alloc(&entry, NV_UNIQUE_NAME, 0);
652 	if (err != 0) {
653 		(void) fprintf(stderr, "trace nvlist_alloc(entry):  %s\n",
654 		    strerror(err));
655 		return (0);
656 	}
657 
658 	trace_id(entry, "from", &req->id1, req->id1name, req->id1domain);
659 	trace_id(entry, "to", &res->id, req->id2name, req->id2domain);
660 
661 	if (IDMAP_ERROR(res->retcode)) {
662 		trace_int(entry, IDMAP_TRACE_ERROR, NULL,
663 		    (int64_t)res->retcode);
664 	}
665 
666 	va_start(va, fmt);
667 	(void) vasprintf(&buf, fmt, va);
668 	va_end(va);
669 	if (buf != NULL) {
670 		trace_str(entry, IDMAP_TRACE_MESSAGE, NULL, buf);
671 		free(buf);
672 	}
673 
674 	if (DBG(MAPPING, 1))
675 		idmap_trace_print_1(stderr, "", entry);
676 
677 	if (req->flag & IDMAP_REQ_FLG_TRACE) {
678 		/* Lazily allocate the trace list */
679 		if (res->info.trace == NULL) {
680 			err = nvlist_alloc(&res->info.trace, 0, 0);
681 			if (err != 0) {
682 				res->info.trace = NULL; /* just in case */
683 				(void) fprintf(stderr,
684 				    "trace nvlist_alloc(trace):  %s\n",
685 				    strerror(err));
686 				nvlist_free(entry);
687 				return (0);
688 			}
689 		}
690 		(void) nvlist_add_nvlist(res->info.trace, "", entry);
691 		/* Note that entry is copied, so we must still free our copy */
692 	}
693 
694 	nvlist_free(entry);
695 
696 	return (0);
697 }
698