xref: /illumos-gate/usr/src/cmd/mdb/common/modules/genunix/kmem.c (revision 24537d3e)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright 2018 Joyent, Inc.  All rights reserved.
 * Copyright (c) 2012 by Delphix. All rights reserved.
 */

#include <mdb/mdb_param.h>
#include <mdb/mdb_modapi.h>
#include <mdb/mdb_ctf.h>
#include <mdb/mdb_whatis.h>
#include <sys/cpuvar.h>
#include <sys/kmem_impl.h>
#include <sys/vmem_impl.h>
#include <sys/machelf.h>
#include <sys/modctl.h>
#include <sys/kobj.h>
#include <sys/panic.h>
#include <sys/stack.h>
#include <sys/sysmacros.h>
#include <vm/page.h>

#include "avl.h"
#include "combined.h"
#include "dist.h"
#include "kmem.h"
#include "list.h"

#define	dprintf(x) if (mdb_debug_level) { \
	mdb_printf("kmem debug: ");  \
	/*CSTYLED*/\
	mdb_printf x ;\
}

#define	KM_ALLOCATED		0x01
#define	KM_FREE			0x02
#define	KM_BUFCTL		0x04
#define	KM_CONSTRUCTED		0x08	/* only constructed free buffers */
#define	KM_HASH			0x10

static int mdb_debug_level = 0;

/*ARGSUSED*/
static int
kmem_init_walkers(uintptr_t addr, const kmem_cache_t *c, void *ignored)
{
	mdb_walker_t w;
	char descr[64];

	(void) mdb_snprintf(descr, sizeof (descr),
	    "walk the %s cache", c->cache_name);

	w.walk_name = c->cache_name;
	w.walk_descr = descr;
	w.walk_init = kmem_walk_init;
	w.walk_step = kmem_walk_step;
	w.walk_fini = kmem_walk_fini;
	w.walk_init_arg = (void *)addr;

	if (mdb_add_walker(&w) == -1)
		mdb_warn("failed to add %s walker", c->cache_name);

	return (WALK_NEXT);
}

/*ARGSUSED*/
int
kmem_debug(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
	mdb_debug_level ^= 1;

	mdb_printf("kmem: debugging is now %s\n",
	    mdb_debug_level ? "on" : "off");

	return (DCMD_OK);
}

int
kmem_cache_walk_init(mdb_walk_state_t *wsp)
{
	GElf_Sym sym;

	if (mdb_lookup_by_name("kmem_caches", &sym) == -1) {
		mdb_warn("couldn't find kmem_caches");
		return (WALK_ERR);
	}

	wsp->walk_addr = (uintptr_t)sym.st_value;

	return (list_walk_init_named(wsp, "cache list", "cache"));
}

int
kmem_cpu_cache_walk_init(mdb_walk_state_t *wsp)
{
	if (wsp->walk_addr == 0) {
		mdb_warn("kmem_cpu_cache doesn't support global walks");
		return (WALK_ERR);
	}

	if (mdb_layered_walk("cpu", wsp) == -1) {
		mdb_warn("couldn't walk 'cpu'");
		return (WALK_ERR);
	}

	wsp->walk_data = (void *)wsp->walk_addr;

	return (WALK_NEXT);
}

int
kmem_cpu_cache_walk_step(mdb_walk_state_t *wsp)
{
	uintptr_t caddr = (uintptr_t)wsp->walk_data;
	const cpu_t *cpu = wsp->walk_layer;
	kmem_cpu_cache_t cc;

	caddr += OFFSETOF(kmem_cache_t, cache_cpu[cpu->cpu_seqid]);

	if (mdb_vread(&cc, sizeof (kmem_cpu_cache_t), caddr) == -1) {
		mdb_warn("couldn't read kmem_cpu_cache at %p", caddr);
		return (WALK_ERR);
	}

	return (wsp->walk_callback(caddr, &cc, wsp->walk_cbdata));
}

static int
kmem_slab_check(void *p, uintptr_t saddr, void *arg)
{
	kmem_slab_t *sp = p;
	uintptr_t caddr = (uintptr_t)arg;
	if ((uintptr_t)sp->slab_cache != caddr) {
		mdb_warn("slab %p isn't in cache %p (in cache %p)\n",
		    saddr, caddr, sp->slab_cache);
		return (-1);
	}

	return (0);
}

static int
kmem_partial_slab_check(void *p, uintptr_t saddr, void *arg)
{
	kmem_slab_t *sp = p;

	int rc = kmem_slab_check(p, saddr, arg);
	if (rc != 0) {
		return (rc);
	}

	if (!KMEM_SLAB_IS_PARTIAL(sp)) {
		mdb_warn("slab %p is not a partial slab\n", saddr);
		return (-1);
	}

	return (0);
}

static int
kmem_complete_slab_check(void *p, uintptr_t saddr, void *arg)
{
	kmem_slab_t *sp = p;

	int rc = kmem_slab_check(p, saddr, arg);
	if (rc != 0) {
		return (rc);
	}

	if (!KMEM_SLAB_IS_ALL_USED(sp)) {
		mdb_warn("slab %p is not completely allocated\n", saddr);
		return (-1);
	}

	return (0);
}

typedef struct {
	uintptr_t kns_cache_addr;
	int kns_nslabs;
} kmem_nth_slab_t;

static int
kmem_nth_slab_check(void *p, uintptr_t saddr, void *arg)
{
	kmem_nth_slab_t *chkp = arg;

	int rc = kmem_slab_check(p, saddr, (void *)chkp->kns_cache_addr);
	if (rc != 0) {
		return (rc);
	}

	return (chkp->kns_nslabs-- == 0 ? 1 : 0);
}

static int
kmem_complete_slab_walk_init(mdb_walk_state_t *wsp)
{
	uintptr_t caddr = wsp->walk_addr;

	wsp->walk_addr = (uintptr_t)(caddr +
	    offsetof(kmem_cache_t, cache_complete_slabs));

	return (list_walk_init_checked(wsp, "slab list", "slab",
	    kmem_complete_slab_check, (void *)caddr));
}

static int
kmem_partial_slab_walk_init(mdb_walk_state_t *wsp)
{
	uintptr_t caddr = wsp->walk_addr;

	wsp->walk_addr = (uintptr_t)(caddr +
	    offsetof(kmem_cache_t, cache_partial_slabs));

	return (avl_walk_init_checked(wsp, "slab list", "slab",
	    kmem_partial_slab_check, (void *)caddr));
}

int
kmem_slab_walk_init(mdb_walk_state_t *wsp)
{
	uintptr_t caddr = wsp->walk_addr;

	if (caddr == 0) {
		mdb_warn("kmem_slab doesn't support global walks\n");
		return (WALK_ERR);
	}

	combined_walk_init(wsp);
	combined_walk_add(wsp,
	    kmem_complete_slab_walk_init, list_walk_step, list_walk_fini);
	combined_walk_add(wsp,
	    kmem_partial_slab_walk_init, avl_walk_step, avl_walk_fini);

	return (WALK_NEXT);
}

static int
kmem_first_complete_slab_walk_init(mdb_walk_state_t *wsp)
{
	uintptr_t caddr = wsp->walk_addr;
	kmem_nth_slab_t *chk;

	chk = mdb_alloc(sizeof (kmem_nth_slab_t),
	    UM_SLEEP | UM_GC);
	chk->kns_cache_addr = caddr;
	chk->kns_nslabs = 1;
	wsp->walk_addr = (uintptr_t)(caddr +
	    offsetof(kmem_cache_t, cache_complete_slabs));

	return (list_walk_init_checked(wsp, "slab list", "slab",
	    kmem_nth_slab_check, chk));
}

int
kmem_slab_walk_partial_init(mdb_walk_state_t *wsp)
{
	uintptr_t caddr = wsp->walk_addr;
	kmem_cache_t c;

	if (caddr == 0) {
		mdb_warn("kmem_slab_partial doesn't support global walks\n");
		return (WALK_ERR);
	}

	if (mdb_vread(&c, sizeof (c), caddr) == -1) {
		mdb_warn("couldn't read kmem_cache at %p", caddr);
		return (WALK_ERR);
	}

	combined_walk_init(wsp);

	/*
	 * Some consumers (umem_walk_step(), in particular) require at
	 * least one callback if there are any buffers in the cache.  So
	 * if there are *no* partial slabs, report the first full slab, if
	 * any.
	 *
	 * Yes, this is ugly, but it's cleaner than the other possibilities.
	 */
	if (c.cache_partial_slabs.avl_numnodes == 0) {
		combined_walk_add(wsp, kmem_first_complete_slab_walk_init,
		    list_walk_step, list_walk_fini);
	} else {
		combined_walk_add(wsp, kmem_partial_slab_walk_init,
		    avl_walk_step, avl_walk_fini);
	}

	return (WALK_NEXT);
}

int
kmem_cache(uintptr_t addr, uint_t flags, int ac, const mdb_arg_t *argv)
{
	kmem_cache_t c;
	const char *filter = NULL;

	if (mdb_getopts(ac, argv,
	    'n', MDB_OPT_STR, &filter,
	    NULL) != ac) {
		return (DCMD_USAGE);
	}

	if (!(flags & DCMD_ADDRSPEC)) {
		if (mdb_walk_dcmd("kmem_cache", "kmem_cache", ac, argv) == -1) {
			mdb_warn("can't walk kmem_cache");
			return (DCMD_ERR);
		}
		return (DCMD_OK);
	}

	if (DCMD_HDRSPEC(flags))
		mdb_printf("%-?s %-25s %4s %6s %8s %8s\n", "ADDR", "NAME",
		    "FLAG", "CFLAG", "BUFSIZE", "BUFTOTL");

	if (mdb_vread(&c, sizeof (c), addr) == -1) {
		mdb_warn("couldn't read kmem_cache at %p", addr);
		return (DCMD_ERR);
	}

	if ((filter != NULL) && (strstr(c.cache_name, filter) == NULL))
		return (DCMD_OK);

	mdb_printf("%0?p %-25s %04x %06x %8ld %8lld\n", addr, c.cache_name,
	    c.cache_flags, c.cache_cflags, c.cache_bufsize, c.cache_buftotal);

	return (DCMD_OK);
}

void
kmem_cache_help(void)
{
	mdb_printf("%s", "Print kernel memory caches.\n\n");
	mdb_dec_indent(2);
	mdb_printf("%<b>OPTIONS%</b>\n");
	mdb_inc_indent(2);
	mdb_printf("%s",
"  -n name\n"
"        name of kmem cache (or matching partial name)\n"
"\n"
"Column\tDescription\n"
"\n"
"ADDR\t\taddress of kmem cache\n"
"NAME\t\tname of kmem cache\n"
"FLAG\t\tvarious cache state flags\n"
"CFLAG\t\tcache creation flags\n"
"BUFSIZE\tobject size in bytes\n"
"BUFTOTL\tcurrent total buffers in cache (allocated and free)\n");
}

#define	LABEL_WIDTH	11
static void
kmem_slabs_print_dist(uint_t *ks_bucket, size_t buffers_per_slab,
    size_t maxbuckets, size_t minbucketsize)
{
	uint64_t total;
	int buckets;
	int i;
	const int *distarray;
	int complete[2];

	buckets = buffers_per_slab;

	total = 0;
	for (i = 0; i <= buffers_per_slab; i++)
		total += ks_bucket[i];

	if (maxbuckets > 1)
		buckets = MIN(buckets, maxbuckets);

	if (minbucketsize > 1) {
		/*
		 * minbucketsize does not apply to the first bucket reserved
		 * for completely allocated slabs
		 */
		buckets = MIN(buckets, 1 + ((buffers_per_slab - 1) /
		    minbucketsize));
		if ((buckets < 2) && (buffers_per_slab > 1)) {
			buckets = 2;
			minbucketsize = (buffers_per_slab - 1);
		}
	}

	/*
	 * The first printed bucket is reserved for completely allocated slabs.
	 * Passing (buckets - 1) excludes that bucket from the generated
	 * distribution, since we're handling it as a special case.
	 */
	complete[0] = buffers_per_slab;
	complete[1] = buffers_per_slab + 1;
	distarray = dist_linear(buckets - 1, 1, buffers_per_slab - 1);

	mdb_printf("%*s\n", LABEL_WIDTH, "Allocated");
	dist_print_header("Buffers", LABEL_WIDTH, "Slabs");

	dist_print_bucket(complete, 0, ks_bucket, total, LABEL_WIDTH);
	/*
	 * Print bucket ranges in descending order after the first bucket for
	 * completely allocated slabs, so a person can see immediately whether
	 * or not there is fragmentation without having to scan possibly
	 * multiple screens of output. Starting at (buckets - 2) excludes the
	 * extra terminating bucket.
	 */
	for (i = buckets - 2; i >= 0; i--) {
		dist_print_bucket(distarray, i, ks_bucket, total, LABEL_WIDTH);
	}
	mdb_printf("\n");
}
#undef LABEL_WIDTH

/*ARGSUSED*/
static int
kmem_first_slab(uintptr_t addr, const kmem_slab_t *sp, boolean_t *is_slab)
{
	*is_slab = B_TRUE;
	return (WALK_DONE);
}

/*ARGSUSED*/
static int
kmem_first_partial_slab(uintptr_t addr, const kmem_slab_t *sp,
    boolean_t *is_slab)
{
	/*
	 * The "kmem_partial_slab" walker reports the first full slab if there
	 * are no partial slabs (for the sake of consumers that require at least
	 * one callback if there are any buffers in the cache).
	 */
	*is_slab = KMEM_SLAB_IS_PARTIAL(sp);
	return (WALK_DONE);
}

typedef struct kmem_slab_usage {
	int ksu_refcnt;			/* count of allocated buffers on slab */
	boolean_t ksu_nomove;		/* slab marked non-reclaimable */
} kmem_slab_usage_t;

typedef struct kmem_slab_stats {
	const kmem_cache_t *ks_cp;
	int ks_slabs;			/* slabs in cache */
	int ks_partial_slabs;		/* partially allocated slabs in cache */
	uint64_t ks_unused_buffers;	/* total unused buffers in cache */
	int ks_max_buffers_per_slab;	/* max buffers per slab */
	int ks_usage_len;		/* ks_usage array length */
	kmem_slab_usage_t *ks_usage;	/* partial slab usage */
	uint_t *ks_bucket;		/* slab usage distribution */
} kmem_slab_stats_t;

/*ARGSUSED*/
static int
kmem_slablist_stat(uintptr_t addr, const kmem_slab_t *sp,
    kmem_slab_stats_t *ks)
{
	kmem_slab_usage_t *ksu;
	long unused;

	ks->ks_slabs++;
	ks->ks_bucket[sp->slab_refcnt]++;

	unused = (sp->slab_chunks - sp->slab_refcnt);
	if (unused == 0) {
		return (WALK_NEXT);
	}

	ks->ks_partial_slabs++;
	ks->ks_unused_buffers += unused;

	if (ks->ks_partial_slabs > ks->ks_usage_len) {
		kmem_slab_usage_t *usage;
		int len = ks->ks_usage_len;

		len = (len == 0 ? 16 : len * 2);
		usage = mdb_zalloc(len * sizeof (kmem_slab_usage_t), UM_SLEEP);
		if (ks->ks_usage != NULL) {
			bcopy(ks->ks_usage, usage,
			    ks->ks_usage_len * sizeof (kmem_slab_usage_t));
			mdb_free(ks->ks_usage,
			    ks->ks_usage_len * sizeof (kmem_slab_usage_t));
		}
		ks->ks_usage = usage;
		ks->ks_usage_len = len;
	}

	ksu = &ks->ks_usage[ks->ks_partial_slabs - 1];
	ksu->ksu_refcnt = sp->slab_refcnt;
	ksu->ksu_nomove = (sp->slab_flags & KMEM_SLAB_NOMOVE);
	return (WALK_NEXT);
}

static void
kmem_slabs_header()
{
	mdb_printf("%-25s %8s %8s %9s %9s %6s\n",
	    "", "", "Partial", "", "Unused", "");
	mdb_printf("%-25s %8s %8s %9s %9s %6s\n",
	    "Cache Name", "Slabs", "Slabs", "Buffers", "Buffers", "Waste");
	mdb_printf("%-25s %8s %8s %9s %9s %6s\n",
	    "-------------------------", "--------", "--------", "---------",
	    "---------", "------");
}

int
kmem_slabs(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
	kmem_cache_t c;
	kmem_slab_stats_t stats;
	mdb_walk_cb_t cb;
	int pct;
	int tenths_pct;
	size_t maxbuckets = 1;
	size_t minbucketsize = 0;
	const char *filter = NULL;
	const char *name = NULL;
	uint_t opt_v = FALSE;
	boolean_t buckets = B_FALSE;
	boolean_t skip = B_FALSE;

	if (mdb_getopts(argc, argv,
	    'B', MDB_OPT_UINTPTR, &minbucketsize,
	    'b', MDB_OPT_UINTPTR, &maxbuckets,
	    'n', MDB_OPT_STR, &filter,
	    'N', MDB_OPT_STR, &name,
	    'v', MDB_OPT_SETBITS, TRUE, &opt_v,
	    NULL) != argc) {
		return (DCMD_USAGE);
	}

	if ((maxbuckets != 1) || (minbucketsize != 0)) {
		buckets = B_TRUE;
	}

	if (!(flags & DCMD_ADDRSPEC)) {
		if (mdb_walk_dcmd("kmem_cache", "kmem_slabs", argc,
		    argv) == -1) {
			mdb_warn("can't walk kmem_cache");
			return (DCMD_ERR);
		}
		return (DCMD_OK);
	}

	if (mdb_vread(&c, sizeof (c), addr) == -1) {
		mdb_warn("couldn't read kmem_cache at %p", addr);
		return (DCMD_ERR);
	}

	if (name == NULL) {
		skip = ((filter != NULL) &&
		    (strstr(c.cache_name, filter) == NULL));
	} else if (filter == NULL) {
		skip = (strcmp(c.cache_name, name) != 0);
	} else {
		/* match either -n or -N */
		skip = ((strcmp(c.cache_name, name) != 0) &&
		    (strstr(c.cache_name, filter) == NULL));
	}

	if (!(opt_v || buckets) && DCMD_HDRSPEC(flags)) {
		kmem_slabs_header();
	} else if ((opt_v || buckets) && !skip) {
		if (DCMD_HDRSPEC(flags)) {
			kmem_slabs_header();
		} else {
			boolean_t is_slab = B_FALSE;
			const char *walker_name;
			if (opt_v) {
				cb = (mdb_walk_cb_t)kmem_first_partial_slab;
				walker_name = "kmem_slab_partial";
			} else {
				cb = (mdb_walk_cb_t)kmem_first_slab;
				walker_name = "kmem_slab";
			}
			(void) mdb_pwalk(walker_name, cb, &is_slab, addr);
			if (is_slab) {
				kmem_slabs_header();
			}
		}
	}

	if (skip) {
		return (DCMD_OK);
	}

	bzero(&stats, sizeof (kmem_slab_stats_t));
	stats.ks_cp = &c;
	stats.ks_max_buffers_per_slab = c.cache_maxchunks;
	/* +1 to include a zero bucket */
	stats.ks_bucket = mdb_zalloc((stats.ks_max_buffers_per_slab + 1) *
	    sizeof (*stats.ks_bucket), UM_SLEEP);
	cb = (mdb_walk_cb_t)kmem_slablist_stat;
	(void) mdb_pwalk("kmem_slab", cb, &stats, addr);

	if (c.cache_buftotal == 0) {
		pct = 0;
		tenths_pct = 0;
	} else {
		uint64_t n = stats.ks_unused_buffers * 10000;
		pct = (int)(n / c.cache_buftotal);
		tenths_pct = pct - ((pct / 100) * 100);
		tenths_pct = (tenths_pct + 5) / 10; /* round nearest tenth */
		if (tenths_pct == 10) {
			pct += 100;
			tenths_pct = 0;
		}
	}

	pct /= 100;
	mdb_printf("%-25s %8d %8d %9lld %9lld %3d.%1d%%\n", c.cache_name,
	    stats.ks_slabs, stats.ks_partial_slabs, c.cache_buftotal,
	    stats.ks_unused_buffers, pct, tenths_pct);

	if (maxbuckets == 0) {
		maxbuckets = stats.ks_max_buffers_per_slab;
	}

	if (((maxbuckets > 1) || (minbucketsize > 0)) &&
	    (stats.ks_slabs > 0)) {
		mdb_printf("\n");
		kmem_slabs_print_dist(stats.ks_bucket,
		    stats.ks_max_buffers_per_slab, maxbuckets, minbucketsize);
	}

	mdb_free(stats.ks_bucket, (stats.ks_max_buffers_per_slab + 1) *
	    sizeof (*stats.ks_bucket));

	if (!opt_v) {
		return (DCMD_OK);
	}

	if (opt_v && (stats.ks_partial_slabs > 0)) {
		int i;
		kmem_slab_usage_t *ksu;

		mdb_printf("  %d complete (%d), %d partial:",
		    (stats.ks_slabs - stats.ks_partial_slabs),
		    stats.ks_max_buffers_per_slab,
		    stats.ks_partial_slabs);

		for (i = 0; i < stats.ks_partial_slabs; i++) {
			ksu = &stats.ks_usage[i];
			mdb_printf(" %d%s", ksu->ksu_refcnt,
			    (ksu->ksu_nomove ? "*" : ""));
		}
		mdb_printf("\n\n");
	}

	if (stats.ks_usage_len > 0) {
		mdb_free(stats.ks_usage,
		    stats.ks_usage_len * sizeof (kmem_slab_usage_t));
	}

	return (DCMD_OK);
}

void
kmem_slabs_help(void)
{
	mdb_printf("%s",
"Display slab usage per kmem cache.\n\n");
	mdb_dec_indent(2);
	mdb_printf("%<b>OPTIONS%</b>\n");
	mdb_inc_indent(2);
	mdb_printf("%s",
"  -n name\n"
"        name of kmem cache (or matching partial name)\n"
"  -N name\n"
"        exact name of kmem cache\n"
"  -b maxbins\n"
"        Print a distribution of allocated buffers per slab using at\n"
"        most maxbins bins. The first bin is reserved for completely\n"
"        allocated slabs. Setting maxbins to zero (-b 0) has the same\n"
"        effect as specifying the maximum allocated buffers per slab\n"
"        or setting minbinsize to 1 (-B 1).\n"
"  -B minbinsize\n"
"        Print a distribution of allocated buffers per slab, making\n"
"        all bins (except the first, reserved for completely allocated\n"
"        slabs) at least minbinsize buffers apart.\n"
"  -v    verbose output: List the allocated buffer count of each partial\n"
"        slab on the free list in order from front to back to show how\n"
"        closely the slabs are ordered by usage. For example\n"
"\n"
"          10 complete, 3 partial (8): 7 3 1\n"
"\n"
"        means there are thirteen slabs with eight buffers each, including\n"
"        three partially allocated slabs with less than all eight buffers\n"
"        allocated.\n"
"\n"
"        Buffer allocations are always from the front of the partial slab\n"
"        list. When a buffer is freed from a completely used slab, that\n"
"        slab is added to the front of the partial slab list. Assuming\n"
"        that all buffers are equally likely to be freed soon, the\n"
"        desired order of partial slabs is most-used at the front of the\n"
"        list and least-used at the back (as in the example above).\n"
"        However, if a slab contains an allocated buffer that will not\n"
"        soon be freed, it would be better for that slab to be at the\n"
"        front where all of its buffers can be allocated. Taking a slab\n"
"        off the partial slab list (either with all buffers freed or all\n"
"        buffers allocated) reduces cache fragmentation.\n"
"\n"
"        A slab's allocated buffer count representing a partial slab (9 in\n"
"        the example below) may be marked as follows:\n"
"\n"
"        9*   An asterisk indicates that kmem has marked the slab non-\n"
"        reclaimable because the kmem client refused to move one of the\n"
"        slab's buffers. Since kmem does not expect to completely free the\n"
"        slab, it moves it to the front of the list in the hope of\n"
"        completely allocating it instead. A slab marked with an asterisk\n"
"        stays marked for as long as it remains on the partial slab list.\n"
"\n"
"Column\t\tDescription\n"
"\n"
"Cache Name\t\tname of kmem cache\n"
"Slabs\t\t\ttotal slab count\n"
"Partial Slabs\t\tcount of partially allocated slabs on the free list\n"
"Buffers\t\ttotal buffer count (Slabs * (buffers per slab))\n"
"Unused Buffers\tcount of unallocated buffers across all partial slabs\n"
"Waste\t\t\t(Unused Buffers / Buffers) does not include space\n"
"\t\t\t  for accounting structures (debug mode), slab\n"
"\t\t\t  coloring (incremental small offsets to stagger\n"
"\t\t\t  buffer alignment), or the per-CPU magazine layer\n");
}

static int
addrcmp(const void *lhs, const void *rhs)
{
	uintptr_t p1 = *((uintptr_t *)lhs);
	uintptr_t p2 = *((uintptr_t *)rhs);

	if (p1 < p2)
		return (-1);
	if (p1 > p2)
		return (1);
	return (0);
}

static int
bufctlcmp(const kmem_bufctl_audit_t **lhs, const kmem_bufctl_audit_t **rhs)
{
	const kmem_bufctl_audit_t *bcp1 = *lhs;
	const kmem_bufctl_audit_t *bcp2 = *rhs;

	if (bcp1->bc_timestamp > bcp2->bc_timestamp)
		return (-1);

	if (bcp1->bc_timestamp < bcp2->bc_timestamp)
		return (1);

	return (0);
}

typedef struct kmem_hash_walk {
	uintptr_t *kmhw_table;
	size_t kmhw_nelems;
	size_t kmhw_pos;
	kmem_bufctl_t kmhw_cur;
} kmem_hash_walk_t;

int
kmem_hash_walk_init(mdb_walk_state_t *wsp)
{
	kmem_hash_walk_t *kmhw;
	uintptr_t *hash;
	kmem_cache_t c;
	uintptr_t haddr, addr = wsp->walk_addr;
	size_t nelems;
	size_t hsize;

	if (addr == 0) {
		mdb_warn("kmem_hash doesn't support global walks\n");
		return (WALK_ERR);
	}

	if (mdb_vread(&c, sizeof (c), addr) == -1) {
		mdb_warn("couldn't read cache at addr %p", addr);
		return (WALK_ERR);
	}

	if (!(c.cache_flags & KMF_HASH)) {
		mdb_warn("cache %p doesn't have a hash table\n", addr);
		return (WALK_DONE);		/* nothing to do */
	}

	kmhw = mdb_zalloc(sizeof (kmem_hash_walk_t), UM_SLEEP);
	kmhw->kmhw_cur.bc_next = NULL;
	kmhw->kmhw_pos = 0;

	kmhw->kmhw_nelems = nelems = c.cache_hash_mask + 1;
	hsize = nelems * sizeof (uintptr_t);
	haddr = (uintptr_t)c.cache_hash_table;

	kmhw->kmhw_table = hash = mdb_alloc(hsize, UM_SLEEP);
	if (mdb_vread(hash, hsize, haddr) == -1) {
		mdb_warn("failed to read hash table at %p", haddr);
		mdb_free(hash, hsize);
		mdb_free(kmhw, sizeof (kmem_hash_walk_t));
		return (WALK_ERR);
	}

	wsp->walk_data = kmhw;

	return (WALK_NEXT);
}

int
kmem_hash_walk_step(mdb_walk_state_t *wsp)
{
	kmem_hash_walk_t *kmhw = wsp->walk_data;
	uintptr_t addr = 0;

	if ((addr = (uintptr_t)kmhw->kmhw_cur.bc_next) == 0) {
		while (kmhw->kmhw_pos < kmhw->kmhw_nelems) {
			if ((addr = kmhw->kmhw_table[kmhw->kmhw_pos++]) != 0)
				break;
		}
	}
	if (addr == 0)
		return (WALK_DONE);

	if (mdb_vread(&kmhw->kmhw_cur, sizeof (kmem_bufctl_t), addr) == -1) {
		mdb_warn("couldn't read kmem_bufctl_t at addr %p", addr);
		return (WALK_ERR);
	}

	return (wsp->walk_callback(addr, &kmhw->kmhw_cur, wsp->walk_cbdata));
}

void
kmem_hash_walk_fini(mdb_walk_state_t *wsp)
{
	kmem_hash_walk_t *kmhw = wsp->walk_data;

	if (kmhw == NULL)
		return;

	mdb_free(kmhw->kmhw_table, kmhw->kmhw_nelems * sizeof (uintptr_t));
	mdb_free(kmhw, sizeof (kmem_hash_walk_t));
}

/*
 * Find the address of the bufctl structure for the address 'buf' in cache
 * 'cp', which is at address caddr, and place it in *out.
 */
static int
kmem_hash_lookup(kmem_cache_t *cp, uintptr_t caddr, void *buf, uintptr_t *out)
{
	uintptr_t bucket = (uintptr_t)KMEM_HASH(cp, buf);
	kmem_bufctl_t *bcp;
	kmem_bufctl_t bc;

	if (mdb_vread(&bcp, sizeof (kmem_bufctl_t *), bucket) == -1) {
		mdb_warn("unable to read hash bucket for %p in cache %p",
		    buf, caddr);
		return (-1);
	}

	while (bcp != NULL) {
		if (mdb_vread(&bc, sizeof (kmem_bufctl_t),
		    (uintptr_t)bcp) == -1) {
			mdb_warn("unable to read bufctl at %p", bcp);
			return (-1);
		}
		if (bc.bc_addr == buf) {
			*out = (uintptr_t)bcp;
			return (0);
		}
		bcp = bc.bc_next;
	}

	mdb_warn("unable to find bufctl for %p in cache %p\n", buf, caddr);
	return (-1);
}

int
kmem_get_magsize(const kmem_cache_t *cp)
{
	uintptr_t addr = (uintptr_t)cp->cache_magtype;
	GElf_Sym mt_sym;
	kmem_magtype_t mt;
	int res;

	/*
	 * if cpu 0 has a non-zero magsize, it must be correct.  caches
	 * with KMF_NOMAGAZINE have disabled their magazine layers, so
	 * it is okay to return 0 for them.
	 */
	if ((res = cp->cache_cpu[0].cc_magsize) != 0 ||
	    (cp->cache_flags & KMF_NOMAGAZINE))
		return (res);

	if (mdb_lookup_by_name("kmem_magtype", &mt_sym) == -1) {
		mdb_warn("unable to read 'kmem_magtype'");
	} else if (addr < mt_sym.st_value ||
	    addr + sizeof (mt) - 1 > mt_sym.st_value + mt_sym.st_size - 1 ||
	    ((addr - mt_sym.st_value) % sizeof (mt)) != 0) {
		mdb_warn("cache '%s' has invalid magtype pointer (%p)\n",
		    cp->cache_name, addr);
		return (0);
	}
	if (mdb_vread(&mt, sizeof (mt), addr) == -1) {
		mdb_warn("unable to read magtype at %a", addr);
		return (0);
	}
	return (mt.mt_magsize);
}

/*ARGSUSED*/
static int
kmem_estimate_slab(uintptr_t addr, const kmem_slab_t *sp, size_t *est)
{
	*est -= (sp->slab_chunks - sp->slab_refcnt);

	return (WALK_NEXT);
}

/*
 * Returns an upper bound on the number of allocated buffers in a given
 * cache.
 */
size_t
kmem_estimate_allocated(uintptr_t addr, const kmem_cache_t *cp)
{
	int magsize;
	size_t cache_est;

	cache_est = cp->cache_buftotal;

	(void) mdb_pwalk("kmem_slab_partial",
	    (mdb_walk_cb_t)kmem_estimate_slab, &cache_est, addr);

	if ((magsize = kmem_get_magsize(cp)) != 0) {
		size_t mag_est = cp->cache_full.ml_total * magsize;

		if (cache_est >= mag_est) {
			cache_est -= mag_est;
		} else {
			mdb_warn("cache %p's magazine layer holds more buffers "
			    "than the slab layer.\n", addr);
		}
	}
	return (cache_est);
}

#define	READMAG_ROUNDS(rounds) { \
	if (mdb_vread(mp, magbsize, (uintptr_t)kmp) == -1) { \
		mdb_warn("couldn't read magazine at %p", kmp); \
		goto fail; \
	} \
	for (i = 0; i < rounds; i++) { \
		maglist[magcnt++] = mp->mag_round[i]; \
		if (magcnt == magmax) { \
			mdb_warn("%d magazines exceeds fudge factor\n", \
			    magcnt); \
			goto fail; \
		} \
	} \
}

int
kmem_read_magazines(kmem_cache_t *cp, uintptr_t addr, int ncpus,
    void ***maglistp, size_t *magcntp, size_t *magmaxp, int alloc_flags)
{
	kmem_magazine_t *kmp, *mp;
	void **maglist = NULL;
	int i, cpu;
	size_t magsize, magmax, magbsize;
	size_t magcnt = 0;

	/*
	 * Read the magtype out of the cache, after verifying the pointer's
	 * correctness.
	 */
	magsize = kmem_get_magsize(cp);
	if (magsize == 0) {
		*maglistp = NULL;
		*magcntp = 0;
		*magmaxp = 0;
		return (WALK_NEXT);
	}

	/*
	 * There are several places where we need to go buffer hunting:
	 * the per-CPU loaded magazine, the per-CPU spare full magazine,
	 * and the full magazine list in the depot.
	 *
	 * For an upper bound on the number of buffers in the magazine
	 * layer, we have the number of magazines on the cache_full
	 * list plus at most two magazines per CPU (the loaded and the
	 * spare).  Toss in 100 magazines as a fudge factor in case this
	 * is live (the number "100" comes from the same fudge factor in
	 * crash(1M)).
	 */
	magmax = (cp->cache_full.ml_total + 2 * ncpus + 100) * magsize;
	magbsize = offsetof(kmem_magazine_t, mag_round[magsize]);

	if (magbsize >= PAGESIZE / 2) {
		mdb_warn("magazine size for cache %p unreasonable (%x)\n",
		    addr, magbsize);
		return (WALK_ERR);
	}

	maglist = mdb_alloc(magmax * sizeof (void *), alloc_flags);
	mp = mdb_alloc(magbsize, alloc_flags);
	if (mp == NULL || maglist == NULL)
		goto fail;

	/*
	 * First up: the magazines in the depot (i.e. on the cache_full list).
	 */
	for (kmp = cp->cache_full.ml_list; kmp != NULL; ) {
		READMAG_ROUNDS(magsize);
		kmp = mp->mag_next;

		if (kmp == cp->cache_full.ml_list)
			break; /* cache_full list loop detected */
	}

	dprintf(("cache_full list done\n"));

	/*
	 * Now whip through the CPUs, snagging the loaded magazines
	 * and full spares.
	 *
	 * In order to prevent inconsistent dumps, rounds and prounds
	 * are copied aside before dumping begins.
	 */
	for (cpu = 0; cpu < ncpus; cpu++) {
		kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu];
		short rounds, prounds;

		if (KMEM_DUMPCC(ccp)) {
			rounds = ccp->cc_dump_rounds;
			prounds = ccp->cc_dump_prounds;
		} else {
			rounds = ccp->cc_rounds;
			prounds = ccp->cc_prounds;
		}

		dprintf(("reading cpu cache %p\n",
		    (uintptr_t)ccp - (uintptr_t)cp + addr));

		if (rounds > 0 &&
		    (kmp = ccp->cc_loaded) != NULL) {
			dprintf(("reading %d loaded rounds\n", rounds));
			READMAG_ROUNDS(rounds);
		}

		if (prounds > 0 &&
		    (kmp = ccp->cc_ploaded) != NULL) {
			dprintf(("reading %d previously loaded rounds\n",
			    prounds));
			READMAG_ROUNDS(prounds);
		}
	}

	dprintf(("magazine layer: %d buffers\n", magcnt));

	if (!(alloc_flags & UM_GC))
		mdb_free(mp, magbsize);

	*maglistp = maglist;
	*magcntp = magcnt;
	*magmaxp = magmax;

	return (WALK_NEXT);

fail:
	if (!(alloc_flags & UM_GC)) {
		if (mp)
			mdb_free(mp, magbsize);
		if (maglist)
			mdb_free(maglist, magmax * sizeof (void *));