xref: /illumos-gate/usr/src/cmd/mdb/common/modules/genunix/findstack.c (revision c9a6ea2e938727c95af7108c5e00eee4c890c7ae)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 #include <mdb/mdb_modapi.h>
27 #include <mdb/mdb_ctf.h>
28 
29 #include <sys/types.h>
30 #include <sys/regset.h>
31 #include <sys/stack.h>
32 #include <sys/thread.h>
33 #include <sys/modctl.h>
34 #include <assert.h>
35 
36 #include "findstack.h"
37 #include "thread.h"
38 #include "sobj.h"
39 
40 int findstack_debug_on = 0;
41 
42 /*
43  * "sp" is a kernel VA.
44  */
45 static int
46 print_stack(uintptr_t sp, uintptr_t pc, uintptr_t addr,
47     int argc, const mdb_arg_t *argv, int free_state)
48 {
49 	int showargs = 0, count, err;
50 
51 	count = mdb_getopts(argc, argv,
52 	    'v', MDB_OPT_SETBITS, TRUE, &showargs, NULL);
53 	argc -= count;
54 	argv += count;
55 
56 	if (argc > 1 || (argc == 1 && argv->a_type != MDB_TYPE_STRING))
57 		return (DCMD_USAGE);
58 
59 	mdb_printf("stack pointer for thread %p%s: %p\n",
60 	    addr, (free_state ? " (TS_FREE)" : ""), sp);
61 	if (pc != 0)
62 		mdb_printf("[ %0?lr %a() ]\n", sp, pc);
63 
64 	mdb_inc_indent(2);
65 	mdb_set_dot(sp);
66 
67 	if (argc == 1)
68 		err = mdb_eval(argv->a_un.a_str);
69 	else if (showargs)
70 		err = mdb_eval("<.$C");
71 	else
72 		err = mdb_eval("<.$C0");
73 
74 	mdb_dec_indent(2);
75 
76 	return ((err == -1) ? DCMD_ABORT : DCMD_OK);
77 }
78 
79 int
80 findstack(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
81 {
82 	findstack_info_t fsi;
83 	int retval;
84 
85 	if (!(flags & DCMD_ADDRSPEC))
86 		return (DCMD_USAGE);
87 
88 	bzero(&fsi, sizeof (fsi));
89 
90 	if ((retval = stacks_findstack(addr, &fsi, 1)) != DCMD_OK ||
91 	    fsi.fsi_failed)
92 		return (retval);
93 
94 	return (print_stack(fsi.fsi_sp, fsi.fsi_pc, addr,
95 	    argc, argv, fsi.fsi_tstate == TS_FREE));
96 }
97 
98 /*ARGSUSED*/
99 int
100 findstack_debug(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *av)
101 {
102 	findstack_debug_on ^= 1;
103 
104 	mdb_printf("findstack: debugging is now %s\n",
105 	    findstack_debug_on ? "on" : "off");
106 
107 	return (DCMD_OK);
108 }
109 
110 static void
111 uppercase(char *p)
112 {
113 	for (; *p != '\0'; p++) {
114 		if (*p >= 'a' && *p <= 'z')
115 			*p += 'A' - 'a';
116 	}
117 }
118 
119 static void
120 sobj_to_text(uintptr_t addr, char *out, size_t out_sz)
121 {
122 	sobj_ops_to_text(addr, out, out_sz);
123 	uppercase(out);
124 }
125 
126 #define	SOBJ_ALL	1
127 
128 static int
129 text_to_sobj(const char *text, uintptr_t *out)
130 {
131 	if (strcasecmp(text, "ALL") == 0) {
132 		*out = SOBJ_ALL;
133 		return (0);
134 	}
135 
136 	return (sobj_text_to_ops(text, out));
137 }
138 
139 #define	TSTATE_PANIC	-2U
140 static int
141 text_to_tstate(const char *text, uint_t *out)
142 {
143 	if (strcasecmp(text, "panic") == 0)
144 		*out = TSTATE_PANIC;
145 	else if (thread_text_to_state(text, out) != 0) {
146 		mdb_warn("tstate \"%s\" not recognized\n", text);
147 		return (-1);
148 	}
149 	return (0);
150 }
151 
152 static void
153 tstate_to_text(uint_t tstate, uint_t paniced, char *out, size_t out_sz)
154 {
155 	if (paniced)
156 		mdb_snprintf(out, out_sz, "panic");
157 	else
158 		thread_state_to_text(tstate, out, out_sz);
159 	uppercase(out);
160 }
161 
162 typedef struct stacks_entry {
163 	struct stacks_entry	*se_next;
164 	struct stacks_entry	*se_dup;	/* dups of this stack */
165 	uintptr_t		se_thread;
166 	uintptr_t		se_sp;
167 	uintptr_t		se_sobj_ops;
168 	uint32_t		se_tstate;
169 	uint32_t		se_count;	/* # threads w/ this stack */
170 	uint8_t			se_overflow;
171 	uint8_t			se_depth;
172 	uint8_t			se_failed;	/* failure reason; FSI_FAIL_* */
173 	uint8_t			se_panic;
174 	uintptr_t		se_stack[1];
175 } stacks_entry_t;
176 #define	STACKS_ENTRY_SIZE(x) OFFSETOF(stacks_entry_t, se_stack[(x)])
177 
178 #define	STACKS_HSIZE 127
179 
180 /* Maximum stack depth reported in stacks */
181 #define	STACKS_MAX_DEPTH	254
182 
183 typedef struct stacks_info {
184 	size_t		si_count;	/* total stacks_entry_ts (incl dups) */
185 	size_t		si_entries;	/* # entries in hash table */
186 	stacks_entry_t	**si_hash;	/* hash table */
187 	findstack_info_t si_fsi;	/* transient callback state */
188 } stacks_info_t;
189 
190 /* global state cached between invocations */
191 #define	STACKS_STATE_CLEAN	0
192 #define	STACKS_STATE_DIRTY	1
193 #define	STACKS_STATE_DONE	2
194 static uint_t stacks_state = STACKS_STATE_CLEAN;
195 static stacks_entry_t **stacks_hash;
196 static stacks_entry_t **stacks_array;
197 static size_t stacks_array_size;
198 
199 size_t
200 stacks_hash_entry(stacks_entry_t *sep)
201 {
202 	size_t depth = sep->se_depth;
203 	uintptr_t *stack = sep->se_stack;
204 
205 	uint64_t total = depth;
206 
207 	while (depth > 0) {
208 		total += *stack;
209 		stack++; depth--;
210 	}
211 
212 	return (total % STACKS_HSIZE);
213 }
214 
215 /*
216  * This is used to both compare stacks for equality and to sort the final
217  * list of unique stacks.  forsort specifies the latter behavior, which
218  * additionally:
219  *	compares se_count, and
220  *	sorts the stacks by text function name.
221  *
222  * The equality test is independent of se_count, and doesn't care about
223  * relative ordering, so we don't do the extra work of looking up symbols
224  * for the stack addresses.
225  */
226 int
227 stacks_entry_comp_impl(stacks_entry_t *l, stacks_entry_t *r,
228     uint_t forsort)
229 {
230 	int idx;
231 
232 	int depth = MIN(l->se_depth, r->se_depth);
233 
234 	/* no matter what, panic stacks come last. */
235 	if (l->se_panic > r->se_panic)
236 		return (1);
237 	if (l->se_panic < r->se_panic)
238 		return (-1);
239 
240 	if (forsort) {
241 		/* put large counts earlier */
242 		if (l->se_count > r->se_count)
243 			return (-1);
244 		if (l->se_count < r->se_count)
245 			return (1);
246 	}
247 
248 	if (l->se_tstate > r->se_tstate)
249 		return (1);
250 	if (l->se_tstate < r->se_tstate)
251 		return (-1);
252 
253 	if (l->se_failed > r->se_failed)
254 		return (1);
255 	if (l->se_failed < r->se_failed)
256 		return (-1);
257 
258 	for (idx = 0; idx < depth; idx++) {
259 		char lbuf[MDB_SYM_NAMLEN];
260 		char rbuf[MDB_SYM_NAMLEN];
261 
262 		int rval;
263 		uintptr_t laddr = l->se_stack[idx];
264 		uintptr_t raddr = r->se_stack[idx];
265 
266 		if (laddr == raddr)
267 			continue;
268 
269 		if (forsort &&
270 		    mdb_lookup_by_addr(laddr, MDB_SYM_FUZZY,
271 		    lbuf, sizeof (lbuf), NULL) != -1 &&
272 		    mdb_lookup_by_addr(raddr, MDB_SYM_FUZZY,
273 		    rbuf, sizeof (rbuf), NULL) != -1 &&
274 		    (rval = strcmp(lbuf, rbuf)) != 0)
275 			return (rval);
276 
277 		if (laddr > raddr)
278 			return (1);
279 		return (-1);
280 	}
281 
282 	if (l->se_overflow > r->se_overflow)
283 		return (-1);
284 	if (l->se_overflow < r->se_overflow)
285 		return (1);
286 
287 	if (l->se_depth > r->se_depth)
288 		return (1);
289 	if (l->se_depth < r->se_depth)
290 		return (-1);
291 
292 	if (l->se_sobj_ops > r->se_sobj_ops)
293 		return (1);
294 	if (l->se_sobj_ops < r->se_sobj_ops)
295 		return (-1);
296 
297 	return (0);
298 }
299 
300 int
301 stacks_entry_comp(const void *l_arg, const void *r_arg)
302 {
303 	stacks_entry_t * const *lp = l_arg;
304 	stacks_entry_t * const *rp = r_arg;
305 
306 	return (stacks_entry_comp_impl(*lp, *rp, 1));
307 }
308 
309 void
310 stacks_cleanup(int force)
311 {
312 	int idx = 0;
313 	stacks_entry_t *cur, *next;
314 
315 	if (stacks_state == STACKS_STATE_CLEAN)
316 		return;
317 
318 	if (!force && stacks_state == STACKS_STATE_DONE)
319 		return;
320 
321 	/*
322 	 * Until the array is sorted and stable, stacks_hash will be non-NULL.
323 	 * This way, we can get at all of the data, even if qsort() was
324 	 * interrupted while mucking with the array.
325 	 */
326 	if (stacks_hash != NULL) {
327 		for (idx = 0; idx < STACKS_HSIZE; idx++) {
328 			while ((cur = stacks_hash[idx]) != NULL) {
329 				while ((next = cur->se_dup) != NULL) {
330 					cur->se_dup = next->se_dup;
331 					mdb_free(next,
332 					    STACKS_ENTRY_SIZE(next->se_depth));
333 				}
334 				next = cur->se_next;
335 				stacks_hash[idx] = next;
336 				mdb_free(cur, STACKS_ENTRY_SIZE(cur->se_depth));
337 			}
338 		}
339 		if (stacks_array != NULL)
340 			mdb_free(stacks_array,
341 			    stacks_array_size * sizeof (*stacks_array));
342 
343 	} else if (stacks_array != NULL) {
344 		for (idx = 0; idx < stacks_array_size; idx++) {
345 			if ((cur = stacks_array[idx]) != NULL) {
346 				while ((next = cur->se_dup) != NULL) {
347 					cur->se_dup = next->se_dup;
348 					mdb_free(next,
349 					    STACKS_ENTRY_SIZE(next->se_depth));
350 				}
351 				stacks_array[idx] = NULL;
352 				mdb_free(cur, STACKS_ENTRY_SIZE(cur->se_depth));
353 			}
354 		}
355 		mdb_free(stacks_array,
356 		    stacks_array_size * sizeof (*stacks_array));
357 	}
358 
359 	stacks_findstack_cleanup();
360 
361 	stacks_array_size = 0;
362 	stacks_state = STACKS_STATE_CLEAN;
363 }
364 
365 /*ARGSUSED*/
366 int
367 stacks_thread_cb(uintptr_t addr, const void *ignored, void *cbarg)
368 {
369 	stacks_info_t *sip = cbarg;
370 	findstack_info_t *fsip = &sip->si_fsi;
371 
372 	stacks_entry_t **sepp, *nsep, *sep;
373 	int idx;
374 	size_t depth;
375 
376 	if (stacks_findstack(addr, fsip, 0) != DCMD_OK &&
377 	    fsip->fsi_failed == FSI_FAIL_BADTHREAD) {
378 		mdb_warn("couldn't read thread at %p\n", addr);
379 		return (WALK_NEXT);
380 	}
381 
382 	sip->si_count++;
383 
384 	depth = fsip->fsi_depth;
385 	nsep = mdb_zalloc(STACKS_ENTRY_SIZE(depth), UM_SLEEP);
386 	nsep->se_thread = addr;
387 	nsep->se_sp = fsip->fsi_sp;
388 	nsep->se_sobj_ops = fsip->fsi_sobj_ops;
389 	nsep->se_tstate = fsip->fsi_tstate;
390 	nsep->se_count = 1;
391 	nsep->se_overflow = fsip->fsi_overflow;
392 	nsep->se_depth = depth;
393 	nsep->se_failed = fsip->fsi_failed;
394 	nsep->se_panic = fsip->fsi_panic;
395 
396 	for (idx = 0; idx < depth; idx++)
397 		nsep->se_stack[idx] = fsip->fsi_stack[idx];
398 
399 	for (sepp = &sip->si_hash[stacks_hash_entry(nsep)];
400 	    (sep = *sepp) != NULL;
401 	    sepp = &sep->se_next) {
402 
403 		if (stacks_entry_comp_impl(sep, nsep, 0) != 0)
404 			continue;
405 
406 		nsep->se_dup = sep->se_dup;
407 		sep->se_dup = nsep;
408 		sep->se_count++;
409 		return (WALK_NEXT);
410 	}
411 
412 	nsep->se_next = NULL;
413 	*sepp = nsep;
414 	sip->si_entries++;
415 
416 	return (WALK_NEXT);
417 }
418 
419 int
420 stacks_run_tlist(mdb_pipe_t *tlist, stacks_info_t *si)
421 {
422 	size_t idx;
423 	size_t found = 0;
424 	int ret;
425 
426 	for (idx = 0; idx < tlist->pipe_len; idx++) {
427 		uintptr_t addr = tlist->pipe_data[idx];
428 
429 		found++;
430 
431 		ret = stacks_thread_cb(addr, NULL, si);
432 		if (ret == WALK_DONE)
433 			break;
434 		if (ret != WALK_NEXT)
435 			return (-1);
436 	}
437 
438 	if (found)
439 		return (0);
440 	return (-1);
441 }
442 
443 int
444 stacks_run(int verbose, mdb_pipe_t *tlist)
445 {
446 	stacks_info_t si;
447 	findstack_info_t *fsip = &si.si_fsi;
448 	size_t idx;
449 	stacks_entry_t **cur;
450 
451 	bzero(&si, sizeof (si));
452 
453 	stacks_state = STACKS_STATE_DIRTY;
454 
455 	stacks_hash = si.si_hash =
456 	    mdb_zalloc(STACKS_HSIZE * sizeof (*si.si_hash), UM_SLEEP);
457 	si.si_entries = 0;
458 	si.si_count = 0;
459 
460 	fsip->fsi_max_depth = STACKS_MAX_DEPTH;
461 	fsip->fsi_stack =
462 	    mdb_alloc(fsip->fsi_max_depth * sizeof (*fsip->fsi_stack),
463 	    UM_SLEEP | UM_GC);
464 
465 	if (verbose)
466 		mdb_warn("stacks: processing kernel threads\n");
467 
468 	if (tlist != NULL) {
469 		if (stacks_run_tlist(tlist, &si))
470 			return (DCMD_ERR);
471 	} else {
472 		if (mdb_walk("thread", stacks_thread_cb, &si) != 0) {
473 			mdb_warn("cannot walk \"thread\"");
474 			return (DCMD_ERR);
475 		}
476 	}
477 
478 	if (verbose)
479 		mdb_warn("stacks: %d unique stacks / %d threads\n",
480 		    si.si_entries, si.si_count);
481 
482 	stacks_array_size = si.si_entries;
483 	stacks_array =
484 	    mdb_zalloc(si.si_entries * sizeof (*stacks_array), UM_SLEEP);
485 	cur = stacks_array;
486 	for (idx = 0; idx < STACKS_HSIZE; idx++) {
487 		stacks_entry_t *sep;
488 		for (sep = si.si_hash[idx]; sep != NULL; sep = sep->se_next)
489 			*(cur++) = sep;
490 	}
491 
492 	if (cur != stacks_array + si.si_entries) {
493 		mdb_warn("stacks: miscounted array size (%d != size: %d)\n",
494 		    (cur - stacks_array), stacks_array_size);
495 		return (DCMD_ERR);
496 	}
497 	qsort(stacks_array, si.si_entries, sizeof (*stacks_array),
498 	    stacks_entry_comp);
499 
500 	/* Now that we're done, free the hash table */
501 	stacks_hash = NULL;
502 	mdb_free(si.si_hash, STACKS_HSIZE * sizeof (*si.si_hash));
503 
504 	if (tlist == NULL)
505 		stacks_state = STACKS_STATE_DONE;
506 
507 	if (verbose)
508 		mdb_warn("stacks: done\n");
509 
510 	return (DCMD_OK);
511 }
512 
513 static int
514 stacks_has_caller(stacks_entry_t *sep, uintptr_t addr)
515 {
516 	uintptr_t laddr = addr;
517 	uintptr_t haddr = addr + 1;
518 	int idx;
519 	char c[MDB_SYM_NAMLEN];
520 	GElf_Sym sym;
521 
522 	if (mdb_lookup_by_addr(addr, MDB_SYM_FUZZY,
523 	    c, sizeof (c), &sym) != -1 &&
524 	    addr == (uintptr_t)sym.st_value) {
525 		laddr = (uintptr_t)sym.st_value;
526 		haddr = (uintptr_t)sym.st_value + sym.st_size;
527 	}
528 
529 	for (idx = 0; idx < sep->se_depth; idx++)
530 		if (sep->se_stack[idx] >= laddr && sep->se_stack[idx] < haddr)
531 			return (1);
532 
533 	return (0);
534 }
535 
536 static int
537 stacks_has_module(stacks_entry_t *sep, stacks_module_t *mp)
538 {
539 	int idx;
540 
541 	for (idx = 0; idx < sep->se_depth; idx++) {
542 		if (sep->se_stack[idx] >= mp->sm_text &&
543 		    sep->se_stack[idx] < mp->sm_text + mp->sm_size)
544 			return (1);
545 	}
546 
547 	return (0);
548 }
549 
550 static int
551 stacks_module_find(const char *name, stacks_module_t *mp)
552 {
553 	(void) strncpy(mp->sm_name, name, sizeof (mp->sm_name));
554 
555 	if (stacks_module(mp) != 0)
556 		return (-1);
557 
558 	if (mp->sm_size == 0) {
559 		mdb_warn("stacks: module \"%s\" is unknown\n", name);
560 		return (-1);
561 	}
562 
563 	return (0);
564 }
565 
566 static int
567 uintptrcomp(const void *lp, const void *rp)
568 {
569 	uintptr_t lhs = *(const uintptr_t *)lp;
570 	uintptr_t rhs = *(const uintptr_t *)rp;
571 	if (lhs > rhs)
572 		return (1);
573 	if (lhs < rhs)
574 		return (-1);
575 	return (0);
576 }
577 
578 /*ARGSUSED*/
579 int
580 stacks(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
581 {
582 	size_t idx;
583 
584 	char *seen = NULL;
585 
586 	const char *caller_str = NULL;
587 	const char *excl_caller_str = NULL;
588 	uintptr_t caller = 0, excl_caller = 0;
589 	const char *module_str = NULL;
590 	const char *excl_module_str = NULL;
591 	stacks_module_t module, excl_module;
592 	const char *sobj = NULL;
593 	const char *excl_sobj = NULL;
594 	uintptr_t sobj_ops = 0, excl_sobj_ops = 0;
595 	const char *tstate_str = NULL;
596 	const char *excl_tstate_str = NULL;
597 	uint_t tstate = -1U;
598 	uint_t excl_tstate = -1U;
599 	uint_t printed = 0;
600 
601 	uint_t all = 0;
602 	uint_t force = 0;
603 	uint_t interesting = 0;
604 	uint_t verbose = 0;
605 
606 	/*
607 	 * We have a slight behavior difference between having piped
608 	 * input and 'addr::stacks'.  Without a pipe, we assume the
609 	 * thread pointer given is a representative thread, and so
610 	 * we include all similar threads in the system in our output.
611 	 *
612 	 * With a pipe, we filter down to just the threads in our
613 	 * input.
614 	 */
615 	uint_t addrspec = (flags & DCMD_ADDRSPEC);
616 	uint_t only_matching = addrspec && (flags & DCMD_PIPE);
617 
618 	mdb_pipe_t p;
619 
620 	bzero(&module, sizeof (module));
621 	bzero(&excl_module, sizeof (excl_module));
622 
623 	if (mdb_getopts(argc, argv,
624 	    'a', MDB_OPT_SETBITS, TRUE, &all,
625 	    'f', MDB_OPT_SETBITS, TRUE, &force,
626 	    'i', MDB_OPT_SETBITS, TRUE, &interesting,
627 	    'v', MDB_OPT_SETBITS, TRUE, &verbose,
628 	    'c', MDB_OPT_STR, &caller_str,
629 	    'C', MDB_OPT_STR, &excl_caller_str,
630 	    'm', MDB_OPT_STR, &module_str,
631 	    'M', MDB_OPT_STR, &excl_module_str,
632 	    's', MDB_OPT_STR, &sobj,
633 	    'S', MDB_OPT_STR, &excl_sobj,
634 	    't', MDB_OPT_STR, &tstate_str,
635 	    'T', MDB_OPT_STR, &excl_tstate_str,
636 	    NULL) != argc)
637 		return (DCMD_USAGE);
638 
639 	if (interesting) {
640 		if (sobj != NULL || excl_sobj != NULL ||
641 		    tstate_str != NULL || excl_tstate_str != NULL) {
642 			mdb_warn(
643 			    "stacks: -i is incompatible with -[sStT]\n");
644 			return (DCMD_USAGE);
645 		}
646 		excl_sobj = "CV";
647 		excl_tstate_str = "FREE";
648 	}
649 
650 	if (caller_str != NULL) {
651 		mdb_set_dot(0);
652 		if (mdb_eval(caller_str) != 0) {
653 			mdb_warn("stacks: evaluation of \"%s\" failed",
654 			    caller_str);
655 			return (DCMD_ABORT);
656 		}
657 		caller = mdb_get_dot();
658 	}
659 
660 	if (excl_caller_str != NULL) {
661 		mdb_set_dot(0);
662 		if (mdb_eval(excl_caller_str) != 0) {
663 			mdb_warn("stacks: evaluation of \"%s\" failed",
664 			    excl_caller_str);
665 			return (DCMD_ABORT);
666 		}
667 		excl_caller = mdb_get_dot();
668 	}
669 	mdb_set_dot(addr);
670 
671 	if (module_str != NULL && stacks_module_find(module_str, &module) != 0)
672 		return (DCMD_ABORT);
673 
674 	if (excl_module_str != NULL &&
675 	    stacks_module_find(excl_module_str, &excl_module) != 0)
676 		return (DCMD_ABORT);
677 
678 	if (sobj != NULL && text_to_sobj(sobj, &sobj_ops) != 0)
679 		return (DCMD_USAGE);
680 
681 	if (excl_sobj != NULL && text_to_sobj(excl_sobj, &excl_sobj_ops) != 0)
682 		return (DCMD_USAGE);
683 
684 	if (sobj_ops != 0 && excl_sobj_ops != 0) {
685 		mdb_warn("stacks: only one of -s and -S can be specified\n");
686 		return (DCMD_USAGE);
687 	}
688 
689 	if (tstate_str != NULL && text_to_tstate(tstate_str, &tstate) != 0)
690 		return (DCMD_USAGE);
691 
692 	if (excl_tstate_str != NULL &&
693 	    text_to_tstate(excl_tstate_str, &excl_tstate) != 0)
694 		return (DCMD_USAGE);
695 
696 	if (tstate != -1U && excl_tstate != -1U) {
697 		mdb_warn("stacks: only one of -t and -T can be specified\n");
698 		return (DCMD_USAGE);
699 	}
700 
701 	/*
702 	 * If there's an address specified, we're going to further filter
703 	 * to only entries which have an address in the input.  To reduce
704 	 * overhead (and make the sorted output come out right), we
705 	 * use mdb_get_pipe() to grab the entire pipeline of input, then
706 	 * use qsort() and bsearch() to speed up the search.
707 	 */
708 	if (addrspec) {
709 		mdb_get_pipe(&p);
710 		if (p.pipe_data == NULL || p.pipe_len == 0) {
711 			p.pipe_data = &addr;
712 			p.pipe_len = 1;
713 		}
714 		qsort(p.pipe_data, p.pipe_len, sizeof (uintptr_t),
715 		    uintptrcomp);
716 
717 		/* remove any duplicates in the data */
718 		idx = 0;
719 		while (idx < p.pipe_len - 1) {
720 			uintptr_t *data = &p.pipe_data[idx];
721 			size_t len = p.pipe_len - idx;
722 
723 			if (data[0] == data[1]) {
724 				memmove(data, data + 1,
725 				    (len - 1) * sizeof (*data));
726 				p.pipe_len--;
727 				continue; /* repeat without incrementing idx */
728 			}
729 			idx++;
730 		}
731 
732 		seen = mdb_zalloc(p.pipe_len, UM_SLEEP | UM_GC);
733 	}
734 
735 	/*
736 	 * Force a cleanup if we're connected to a live system. Never
737 	 * do a cleanup after the first invocation around the loop.
738 	 */
739 	force |= (mdb_get_state() == MDB_STATE_RUNNING);
740 	if (force && (flags & (DCMD_LOOPFIRST|DCMD_LOOP)) == DCMD_LOOP)
741 		force = 0;
742 
743 	stacks_cleanup(force);
744 
745 	if (stacks_state == STACKS_STATE_CLEAN) {
746 		int res = stacks_run(verbose, addrspec ? &p : NULL);
747 		if (res != DCMD_OK)
748 			return (res);
749 	}
750 
751 	for (idx = 0; idx < stacks_array_size; idx++) {
752 		stacks_entry_t *sep = stacks_array[idx];
753 		stacks_entry_t *cur = sep;
754 		int frame;
755 		size_t count = sep->se_count;
756 
757 		if (addrspec) {
758 			stacks_entry_t *head = NULL, *tail = NULL, *sp;
759 			size_t foundcount = 0;
760 			/*
761 			 * We use the now-unused hash chain field se_next to
762 			 * link together the dups which match our list.
763 			 */
764 			for (sp = sep; sp != NULL; sp = sp->se_dup) {
765 				uintptr_t *entry = bsearch(&sp->se_thread,
766 				    p.pipe_data, p.pipe_len, sizeof (uintptr_t),
767 				    uintptrcomp);
768 				if (entry != NULL) {
769 					foundcount++;
770 					seen[entry - p.pipe_data]++;
771 					if (head == NULL)
772 						head = sp;
773 					else
774 						tail->se_next = sp;
775 					tail = sp;
776 					sp->se_next = NULL;
777 				}
778 			}
779 			if (head == NULL)
780 				continue;	/* no match, skip entry */
781 
782 			if (only_matching) {
783 				cur = sep = head;
784 				count = foundcount;
785 			}
786 		}
787 
788 		if (caller != 0 && !stacks_has_caller(sep, caller))
789 			continue;
790 
791 		if (excl_caller != 0 && stacks_has_caller(sep, excl_caller))
792 			continue;
793 
794 		if (module.sm_size != 0 && !stacks_has_module(sep, &module))
795 			continue;
796 
797 		if (excl_module.sm_size != 0 &&
798 		    stacks_has_module(sep, &excl_module))
799 			continue;
800 
801 		if (tstate != -1U) {
802 			if (tstate == TSTATE_PANIC) {
803 				if (!sep->se_panic)
804 					continue;
805 			} else if (sep->se_panic || sep->se_tstate != tstate)
806 				continue;
807 		}
808 		if (excl_tstate != -1U) {
809 			if (excl_tstate == TSTATE_PANIC) {
810 				if (sep->se_panic)
811 					continue;
812 			} else if (!sep->se_panic &&
813 			    sep->se_tstate == excl_tstate)
814 				continue;
815 		}
816 
817 		if (sobj_ops == SOBJ_ALL) {
818 			if (sep->se_sobj_ops == 0)
819 				continue;
820 		} else if (sobj_ops != 0) {
821 			if (sobj_ops != sep->se_sobj_ops)
822 				continue;
823 		}
824 
825 		if (!(interesting && sep->se_panic)) {
826 			if (excl_sobj_ops == SOBJ_ALL) {
827 				if (sep->se_sobj_ops != 0)
828 					continue;
829 			} else if (excl_sobj_ops != 0) {
830 				if (excl_sobj_ops == sep->se_sobj_ops)
831 					continue;
832 			}
833 		}
834 
835 		if (flags & DCMD_PIPE_OUT) {
836 			while (sep != NULL) {
837 				mdb_printf("%lr\n", sep->se_thread);
838 				sep = only_matching ?
839 				    sep->se_next : sep->se_dup;
840 			}
841 			continue;
842 		}
843 
844 		if (all || !printed) {
845 			mdb_printf("%<u>%-?s %-8s %-?s %8s%</u>\n",
846 			    "THREAD", "STATE", "SOBJ", "COUNT");
847 			printed = 1;
848 		}
849 
850 		do {
851 			char state[20];
852 			char sobj[100];
853 
854 			tstate_to_text(cur->se_tstate, cur->se_panic,
855 			    state, sizeof (state));
856 			sobj_to_text(cur->se_sobj_ops,
857 			    sobj, sizeof (sobj));
858 
859 			if (cur == sep)
860 				mdb_printf("%-?p %-8s %-?s %8d\n",
861 				    cur->se_thread, state, sobj, count);
862 			else
863 				mdb_printf("%-?p %-8s %-?s %8s\n",
864 				    cur->se_thread, state, sobj, "-");
865 
866 			cur = only_matching ? cur->se_next : cur->se_dup;
867 		} while (all && cur != NULL);
868 
869 		if (sep->se_failed != 0) {
870 			char *reason;
871 			switch (sep->se_failed) {
872 			case FSI_FAIL_NOTINMEMORY:
873 				reason = "thread not in memory";
874 				break;
875 			case FSI_FAIL_THREADCORRUPT:
876 				reason = "thread structure stack info corrupt";
877 				break;
878 			case FSI_FAIL_STACKNOTFOUND:
879 				reason = "no consistent stack found";
880 				break;
881 			default:
882 				reason = "unknown failure";
883 				break;
884 			}
885 			mdb_printf("%?s <%s>\n", "", reason);
886 		}
887 
888 		for (frame = 0; frame < sep->se_depth; frame++)
889 			mdb_printf("%?s %a\n", "", sep->se_stack[frame]);
890 		if (sep->se_overflow)
891 			mdb_printf("%?s ... truncated ...\n", "");
892 		mdb_printf("\n");
893 	}
894 
895 	if (flags & DCMD_ADDRSPEC) {
896 		for (idx = 0; idx < p.pipe_len; idx++)
897 			if (seen[idx] == 0)
898 				mdb_warn("stacks: %p not in thread list\n",
899 				    p.pipe_data[idx]);
900 	}
901 	return (DCMD_OK);
902 }
903