1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2018 Joyent, Inc.
25 * Copyright 2018 Nexenta Systems, Inc. All rights reserved.
26 */
27
28#include <sys/types.h>
29#include <sys/param.h>
30#include <sys/systm.h>
31#include <sys/vm.h>
32#include <sys/proc.h>
33#include <sys/file.h>
34#include <sys/conf.h>
35#include <sys/kmem.h>
36#include <sys/mem.h>
37#include <sys/mman.h>
38#include <sys/vnode.h>
39#include <sys/errno.h>
40#include <sys/memlist.h>
41#include <sys/dumphdr.h>
42#include <sys/dumpadm.h>
43#include <sys/ksyms.h>
44#include <sys/compress.h>
45#include <sys/stream.h>
46#include <sys/strsun.h>
47#include <sys/cmn_err.h>
48#include <sys/bitmap.h>
49#include <sys/modctl.h>
50#include <sys/utsname.h>
51#include <sys/systeminfo.h>
52#include <sys/vmem.h>
53#include <sys/log.h>
54#include <sys/var.h>
55#include <sys/debug.h>
56#include <sys/sunddi.h>
57#include <fs/fs_subr.h>
58#include <sys/fs/snode.h>
59#include <sys/ontrap.h>
60#include <sys/panic.h>
61#include <sys/dkio.h>
62#include <sys/vtoc.h>
63#include <sys/errorq.h>
64#include <sys/fm/util.h>
65#include <sys/fs/zfs.h>
66
67#include <vm/hat.h>
68#include <vm/as.h>
69#include <vm/page.h>
70#include <vm/pvn.h>
71#include <vm/seg.h>
72#include <vm/seg_kmem.h>
73#include <sys/clock_impl.h>
74#include <sys/hold_page.h>
75#include <sys/cpu.h>
76
77#include <bzip2/bzlib.h>
78
79#define	ONE_GIG	(1024 * 1024 * 1024UL)
80
81/*
82 * Crash dump time is dominated by disk write time.  To reduce this,
83 * the stronger compression method bzip2 is applied to reduce the dump
84 * size and hence reduce I/O time.  However, bzip2 is much more
85 * computationally expensive than the existing lzjb algorithm, so to
86 * avoid increasing compression time, CPUs that are otherwise idle
87 * during panic are employed to parallelize the compression task.
88 * Many helper CPUs are needed to prevent bzip2 from being a
89 * bottleneck, and on systems with too few CPUs, the lzjb algorithm is
90 * parallelized instead. Lastly, I/O and compression are performed by
91 * different CPUs, and are hence overlapped in time, unlike the older
92 * serial code.
93 *
94 * Another important consideration is the speed of the dump
95 * device. Faster disks need less CPUs in order to benefit from
96 * parallel lzjb versus parallel bzip2. Therefore, the CPU count
97 * threshold for switching from parallel lzjb to paralled bzip2 is
98 * elevated for faster disks. The dump device speed is adduced from
99 * the setting for dumpbuf.iosize, see dump_update_clevel.
100 */
101
102/*
103 * exported vars
104 */
105kmutex_t	dump_lock;		/* lock for dump configuration */
106dumphdr_t	*dumphdr;		/* dump header */
107int		dump_conflags = DUMP_KERNEL; /* dump configuration flags */
108vnode_t		*dumpvp;		/* dump device vnode pointer */
109u_offset_t	dumpvp_size;		/* size of dump device, in bytes */
110char		*dumppath;		/* pathname of dump device */
111int		dump_timeout = 120;	/* timeout for dumping pages */
112int		dump_timeleft;		/* portion of dump_timeout remaining */
113int		dump_ioerr;		/* dump i/o error */
114int		dump_check_used;	/* enable check for used pages */
115char	    *dump_stack_scratch; /* scratch area for saving stack summary */
116
117/*
118 * Tunables for dump compression and parallelism. These can be set via
119 * /etc/system.
120 *
121 * dump_ncpu_low	number of helpers for parallel lzjb
122 *	This is also the minimum configuration.
123 *
124 * dump_bzip2_level	bzip2 compression level: 1-9
125 *	Higher numbers give greater compression, but take more memory
126 *	and time. Memory used per helper is ~(dump_bzip2_level * 1MB).
127 *
128 * dump_plat_mincpu	the cross-over limit for using bzip2 (per platform):
129 *	if dump_plat_mincpu == 0, then always do single threaded dump
130 *	if ncpu >= dump_plat_mincpu then try to use bzip2
131 *
132 * dump_metrics_on	if set, metrics are collected in the kernel, passed
133 *	to savecore via the dump file, and recorded by savecore in
134 *	METRICS.txt.
135 */
136uint_t dump_ncpu_low = 4;	/* minimum config for parallel lzjb */
137uint_t dump_bzip2_level = 1;	/* bzip2 level (1-9) */
138
139/* Use dump_plat_mincpu_default unless this variable is set by /etc/system */
140#define	MINCPU_NOT_SET	((uint_t)-1)
141uint_t dump_plat_mincpu = MINCPU_NOT_SET;
142
143/* tunables for pre-reserved heap */
144uint_t dump_kmem_permap = 1024;
145uint_t dump_kmem_pages = 0;
146
147/* Define multiple buffers per helper to avoid stalling */
148#define	NCBUF_PER_HELPER	2
149#define	NCMAP_PER_HELPER	4
150
151/* minimum number of helpers configured */
152#define	MINHELPERS	(dump_ncpu_low)
153#define	MINCBUFS	(MINHELPERS * NCBUF_PER_HELPER)
154
155/*
156 * Define constant parameters.
157 *
158 * CBUF_SIZE		size of an output buffer
159 *
160 * CBUF_MAPSIZE		size of virtual range for mapping pages
161 *
162 * CBUF_MAPNP		size of virtual range in pages
163 *
164 */
165#define	DUMP_1KB	((size_t)1 << 10)
166#define	DUMP_1MB	((size_t)1 << 20)
167#define	CBUF_SIZE	((size_t)1 << 17)
168#define	CBUF_MAPSHIFT	(22)
169#define	CBUF_MAPSIZE	((size_t)1 << CBUF_MAPSHIFT)
170#define	CBUF_MAPNP	((size_t)1 << (CBUF_MAPSHIFT - PAGESHIFT))
171
172/*
173 * Compression metrics are accumulated nano-second subtotals. The
174 * results are normalized by the number of pages dumped. A report is
175 * generated when dumpsys() completes and is saved in the dump image
176 * after the trailing dump header.
177 *
178 * Metrics are always collected. Set the variable dump_metrics_on to
179 * cause metrics to be saved in the crash file, where savecore will
180 * save it in the file METRICS.txt.
181 */
182#define	PERPAGES \
183	PERPAGE(bitmap) PERPAGE(map) PERPAGE(unmap) \
184	PERPAGE(copy) PERPAGE(compress) \
185	PERPAGE(write) \
186	PERPAGE(inwait) PERPAGE(outwait)
187
188typedef struct perpage {
189#define	PERPAGE(x) hrtime_t x;
190	PERPAGES
191#undef PERPAGE
192} perpage_t;
193
194/*
195 * This macro controls the code generation for collecting dump
196 * performance information. By default, the code is generated, but
197 * automatic saving of the information is disabled. If dump_metrics_on
198 * is set to 1, the timing information is passed to savecore via the
199 * crash file, where it is appended to the file dump-dir/METRICS.txt.
200 */
201#define	COLLECT_METRICS
202
203#ifdef COLLECT_METRICS
204uint_t dump_metrics_on = 0;	/* set to 1 to enable recording metrics */
205
206#define	HRSTART(v, m)		v##ts.m = gethrtime()
207#define	HRSTOP(v, m)		v.m += gethrtime() - v##ts.m
208#define	HRBEGIN(v, m, s)	v##ts.m = gethrtime(); v.size += s
209#define	HREND(v, m)		v.m += gethrtime() - v##ts.m
210#define	HRNORM(v, m, n)		v.m /= (n)
211
212#else
213#define	HRSTART(v, m)
214#define	HRSTOP(v, m)
215#define	HRBEGIN(v, m, s)
216#define	HREND(v, m)
217#define	HRNORM(v, m, n)
218#endif	/* COLLECT_METRICS */
219
220/*
221 * Buffers for copying and compressing memory pages.
222 *
223 * cbuf_t buffer controllers: used for both input and output.
224 *
225 * The buffer state indicates how it is being used:
226 *
227 * CBUF_FREEMAP: CBUF_MAPSIZE virtual address range is available for
228 * mapping input pages.
229 *
230 * CBUF_INREADY: input pages are mapped and ready for compression by a
231 * helper.
232 *
233 * CBUF_USEDMAP: mapping has been consumed by a helper. Needs unmap.
234 *
235 * CBUF_FREEBUF: CBUF_SIZE output buffer, which is available.
236 *
237 * CBUF_WRITE: CBUF_SIZE block of compressed pages from a helper,
238 * ready to write out.
239 *
240 * CBUF_ERRMSG: CBUF_SIZE block of error messages from a helper
241 * (reports UE errors.)
242 */
243
244typedef enum cbufstate {
245	CBUF_FREEMAP,
246	CBUF_INREADY,
247	CBUF_USEDMAP,
248	CBUF_FREEBUF,
249	CBUF_WRITE,
250	CBUF_ERRMSG
251} cbufstate_t;
252
253typedef struct cbuf cbuf_t;
254
255struct cbuf {
256	cbuf_t *next;			/* next in list */
257	cbufstate_t state;		/* processing state */
258	size_t used;			/* amount used */
259	size_t size;			/* mem size */
260	char *buf;			/* kmem or vmem */
261	pgcnt_t pagenum;		/* index to pfn map */
262	pgcnt_t bitnum;			/* first set bitnum */
263	pfn_t pfn;			/* first pfn in mapped range */
264	int off;			/* byte offset to first pfn */
265};
266
267static char dump_osimage_uuid[36 + 1];
268
269#define	isdigit(ch)	((ch) >= '0' && (ch) <= '9')
270#define	isxdigit(ch)	(isdigit(ch) || ((ch) >= 'a' && (ch) <= 'f') || \
271			((ch) >= 'A' && (ch) <= 'F'))
272
273/*
274 * cqueue_t queues: a uni-directional channel for communication
275 * from the master to helper tasks or vice-versa using put and
276 * get primitives. Both mappings and data buffers are passed via
277 * queues. Producers close a queue when done. The number of
278 * active producers is reference counted so the consumer can
279 * detect end of data. Concurrent access is mediated by atomic
280 * operations for panic dump, or mutex/cv for live dump.
281 *
282 * There a four queues, used as follows:
283 *
284 * Queue		Dataflow		NewState
285 * --------------------------------------------------
286 * mainq		master -> master	FREEMAP
287 * master has initialized or unmapped an input buffer
288 * --------------------------------------------------
289 * helperq		master -> helper	INREADY
290 * master has mapped input for use by helper
291 * --------------------------------------------------
292 * mainq		master <- helper	USEDMAP
293 * helper is done with input
294 * --------------------------------------------------
295 * freebufq		master -> helper	FREEBUF
296 * master has initialized or written an output buffer
297 * --------------------------------------------------
298 * mainq		master <- helper	WRITE
299 * block of compressed pages from a helper
300 * --------------------------------------------------
301 * mainq		master <- helper	ERRMSG
302 * error messages from a helper (memory error case)
303 * --------------------------------------------------
304 * writerq		master <- master	WRITE
305 * non-blocking queue of blocks to write
306 * --------------------------------------------------
307 */
308typedef struct cqueue {
309	cbuf_t *volatile first;		/* first in list */
310	cbuf_t *last;			/* last in list */
311	hrtime_t ts;			/* timestamp */
312	hrtime_t empty;			/* total time empty */
313	kmutex_t mutex;			/* live state lock */
314	kcondvar_t cv;			/* live wait var */
315	lock_t spinlock;		/* panic mode spin lock */
316	volatile uint_t open;		/* producer ref count */
317} cqueue_t;
318
319/*
320 * Convenience macros for using the cqueue functions
321 * Note that the caller must have defined "dumpsync_t *ds"
322 */
323#define	CQ_IS_EMPTY(q)					\
324	(ds->q.first == NULL)
325
326#define	CQ_OPEN(q)					\
327	atomic_inc_uint(&ds->q.open)
328
329#define	CQ_CLOSE(q)					\
330	dumpsys_close_cq(&ds->q, ds->live)
331
332#define	CQ_PUT(q, cp, st)				\
333	dumpsys_put_cq(&ds->q, cp, st, ds->live)
334
335#define	CQ_GET(q)					\
336	dumpsys_get_cq(&ds->q, ds->live)
337
338/*
339 * Dynamic state when dumpsys() is running.
340 */
341typedef struct dumpsync {
342	pgcnt_t npages;			/* subtotal of pages dumped */
343	pgcnt_t pages_mapped;		/* subtotal of pages mapped */
344	pgcnt_t pages_used;		/* subtotal of pages used per map */
345	size_t nwrite;			/* subtotal of bytes written */
346	uint_t live;			/* running live dump */
347	uint_t neednl;			/* will need to print a newline */
348	uint_t percent;			/* dump progress */
349	uint_t percent_done;		/* dump progress reported */
350	int sec_done;			/* dump progress last report time */
351	cqueue_t freebufq;		/* free kmem bufs for writing */
352	cqueue_t mainq;			/* input for main task */
353	cqueue_t helperq;		/* input for helpers */
354	cqueue_t writerq;		/* input for writer */
355	hrtime_t start;			/* start time */
356	hrtime_t elapsed;		/* elapsed time when completed */
357	hrtime_t iotime;		/* time spent writing nwrite bytes */
358	hrtime_t iowait;		/* time spent waiting for output */
359	hrtime_t iowaitts;		/* iowait timestamp */
360	perpage_t perpage;		/* metrics */
361	perpage_t perpagets;
362	int dumpcpu;			/* master cpu */
363} dumpsync_t;
364
365static dumpsync_t dumpsync;		/* synchronization vars */
366
367/*
368 * helper_t helpers: contains the context for a stream. CPUs run in
369 * parallel at dump time; each CPU creates a single stream of
370 * compression data.  Stream data is divided into CBUF_SIZE blocks.
371 * The blocks are written in order within a stream. But, blocks from
372 * multiple streams can be interleaved. Each stream is identified by a
373 * unique tag.
374 */
375typedef struct helper {
376	int helper;			/* bound helper id */
377	int tag;			/* compression stream tag */
378	perpage_t perpage;		/* per page metrics */
379	perpage_t perpagets;		/* per page metrics (timestamps) */
380	taskqid_t taskqid;		/* live dump task ptr */
381	int in, out;			/* buffer offsets */
382	cbuf_t *cpin, *cpout, *cperr;	/* cbuf objects in process */
383	dumpsync_t *ds;			/* pointer to sync vars */
384	size_t used;			/* counts input consumed */
385	char *page;			/* buffer for page copy */
386	char *lzbuf;			/* lzjb output */
387	bz_stream bzstream;		/* bzip2 state */
388} helper_t;
389
390#define	MAINHELPER	(-1)		/* helper is also the main task */
391#define	FREEHELPER	(-2)		/* unbound helper */
392#define	DONEHELPER	(-3)		/* helper finished */
393
394/*
395 * configuration vars for dumpsys
396 */
397typedef struct dumpcfg {
398	int	threshold;	/* ncpu threshold for bzip2 */
399	int	nhelper;	/* number of helpers */
400	int	nhelper_used;	/* actual number of helpers used */
401	int	ncmap;		/* number VA pages for compression */
402	int	ncbuf;		/* number of bufs for compression */
403	int	ncbuf_used;	/* number of bufs in use */
404	uint_t	clevel;		/* dump compression level */
405	helper_t *helper;	/* array of helpers */
406	cbuf_t	*cmap;		/* array of input (map) buffers */
407	cbuf_t	*cbuf;		/* array of output  buffers */
408	ulong_t	*helpermap;	/* set of dumpsys helper CPU ids */
409	ulong_t	*bitmap;	/* bitmap for marking pages to dump */
410	ulong_t	*rbitmap;	/* bitmap for used CBUF_MAPSIZE ranges */
411	pgcnt_t	bitmapsize;	/* size of bitmap */
412	pgcnt_t	rbitmapsize;	/* size of bitmap for ranges */
413	pgcnt_t found4m;	/* number ranges allocated by dump */
414	pgcnt_t foundsm;	/* number small pages allocated by dump */
415	pid_t	*pids;		/* list of process IDs at dump time */
416	size_t	maxsize;	/* memory size needed at dump time */
417	size_t	maxvmsize;	/* size of reserved VM */
418	char	*maxvm;		/* reserved VM for spare pages */
419	lock_t	helper_lock;	/* protect helper state */
420	char	helpers_wanted;	/* flag to enable parallelism */
421} dumpcfg_t;
422
423static dumpcfg_t dumpcfg;	/* config vars */
424
425/*
426 * The dump I/O buffer.
427 *
428 * There is one I/O buffer used by dumpvp_write and dumvp_flush. It is
429 * sized according to the optimum device transfer speed.
430 */
431typedef struct dumpbuf {
432	vnode_t	*cdev_vp;	/* VCHR open of the dump device */
433	len_t	vp_limit;	/* maximum write offset */
434	offset_t vp_off;	/* current dump device offset */
435	char	*cur;		/* dump write pointer */
436	char	*start;		/* dump buffer address */
437	char	*end;		/* dump buffer end */
438	size_t	size;		/* size of dumpbuf in bytes */
439	size_t	iosize;		/* best transfer size for device */
440} dumpbuf_t;
441
442dumpbuf_t dumpbuf;		/* I/O buffer */
443
444/*
445 * For parallel dump, defines maximum time main task thread will wait
446 * for at least one helper to register in dumpcfg.helpermap, before
447 * assuming there are no helpers and falling back to serial mode.
448 * Value is chosen arbitrary and provides *really* long wait for any
449 * available helper to register.
450 */
451#define	DUMP_HELPER_MAX_WAIT	1000	/* millisec */
452
453/*
454 * The dump I/O buffer must be at least one page, at most xfer_size
455 * bytes, and should scale with physmem in between.  The transfer size
456 * passed in will either represent a global default (maxphys) or the
457 * best size for the device.  The size of the dumpbuf I/O buffer is
458 * limited by dumpbuf_limit (8MB by default) because the dump
459 * performance saturates beyond a certain size.  The default is to
460 * select 1/4096 of the memory.
461 */
462static int	dumpbuf_fraction = 12;	/* memory size scale factor */
463static size_t	dumpbuf_limit = 8 * DUMP_1MB;	/* max I/O buf size */
464
465static size_t
466dumpbuf_iosize(size_t xfer_size)
467{
468	size_t iosize = ptob(physmem >> dumpbuf_fraction);
469
470	if (iosize < PAGESIZE)
471		iosize = PAGESIZE;
472	else if (iosize > xfer_size)
473		iosize = xfer_size;
474	if (iosize > dumpbuf_limit)
475		iosize = dumpbuf_limit;
476	return (iosize & PAGEMASK);
477}
478
479/*
480 * resize the I/O buffer
481 */
482static void
483dumpbuf_resize(void)
484{
485	char *old_buf = dumpbuf.start;
486	size_t old_size = dumpbuf.size;
487	char *new_buf;
488	size_t new_size;
489
490	ASSERT(MUTEX_HELD(&dump_lock));
491
492	new_size = dumpbuf_iosize(MAX(dumpbuf.iosize, maxphys));
493	if (new_size <= old_size)
494		return; /* no need to reallocate buffer */
495
496	new_buf = kmem_alloc(new_size, KM_SLEEP);
497	dumpbuf.size = new_size;
498	dumpbuf.start = new_buf;
499	dumpbuf.end = new_buf + new_size;
500	kmem_free(old_buf, old_size);
501}
502
503/*
504 * dump_update_clevel is called when dumpadm configures the dump device.
505 *	Calculate number of helpers and buffers.
506 *	Allocate the minimum configuration for now.
507 *
508 * When the dump file is configured we reserve a minimum amount of
509 * memory for use at crash time. But we reserve VA for all the memory
510 * we really want in order to do the fastest dump possible. The VA is
511 * backed by pages not being dumped, according to the bitmap. If
512 * there is insufficient spare memory, however, we fall back to the
513 * minimum.
514 *
515 * Live dump (savecore -L) always uses the minimum config.
516 *
517 * clevel 0 is single threaded lzjb
518 * clevel 1 is parallel lzjb
519 * clevel 2 is parallel bzip2
520 *
521 * The ncpu threshold is selected with dump_plat_mincpu.
522 * On OPL, set_platform_defaults() overrides the sun4u setting.
523 * The actual values are defined via DUMP_PLAT_*_MINCPU macros.
524 *
525 * Architecture		Threshold	Algorithm
526 * sun4u		<  51		parallel lzjb
527 * sun4u		>= 51		parallel bzip2(*)
528 * sun4u OPL		<  8		parallel lzjb
529 * sun4u OPL		>= 8		parallel bzip2(*)
530 * sun4v		<  128		parallel lzjb
531 * sun4v		>= 128		parallel bzip2(*)
532 * x86			< 11		parallel lzjb
533 * x86			>= 11		parallel bzip2(*)
534 * 32-bit		N/A		single-threaded lzjb
535 *
536 * (*) bzip2 is only chosen if there is sufficient available
537 * memory for buffers at dump time. See dumpsys_get_maxmem().
538 *
539 * Faster dump devices have larger I/O buffers. The threshold value is
540 * increased according to the size of the dump I/O buffer, because
541 * parallel lzjb performs better with faster disks. For buffers >= 1MB
542 * the threshold is 3X; for buffers >= 256K threshold is 2X.
543 *
544 * For parallel dumps, the number of helpers is ncpu-1. The CPU
545 * running panic runs the main task. For single-threaded dumps, the
546 * panic CPU does lzjb compression (it is tagged as MAINHELPER.)
547 *
548 * Need multiple buffers per helper so that they do not block waiting
549 * for the main task.
550 *				parallel	single-threaded
551 * Number of output buffers:	nhelper*2		1
552 * Number of mapping buffers:	nhelper*4		1
553 *
554 */
555static void
556dump_update_clevel()
557{
558	int tag;
559	size_t bz2size;
560	helper_t *hp, *hpend;
561	cbuf_t *cp, *cpend;
562	dumpcfg_t *old = &dumpcfg;
563	dumpcfg_t newcfg = *old;
564	dumpcfg_t *new = &newcfg;
565
566	ASSERT(MUTEX_HELD(&dump_lock));
567
568	/*
569	 * Free the previously allocated bufs and VM.
570	 */
571	if (old->helper != NULL) {
572
573		/* helpers */
574		hpend = &old->helper[old->nhelper];
575		for (hp = old->helper; hp != hpend; hp++) {
576			if (hp->lzbuf != NULL)
577				kmem_free(hp->lzbuf, PAGESIZE);
578			if (hp->page != NULL)
579				kmem_free(hp->page, PAGESIZE);
580		}
581		kmem_free(old->helper, old->nhelper * sizeof (helper_t));
582
583		/* VM space for mapping pages */
584		cpend = &old->cmap[old->ncmap];
585		for (cp = old->cmap; cp != cpend; cp++)
586			vmem_xfree(heap_arena, cp->buf, CBUF_MAPSIZE);
587		kmem_free(old->cmap, old->ncmap * sizeof (cbuf_t));
588
589		/* output bufs */
590		cpend = &old->cbuf[old->ncbuf];
591		for (cp = old->cbuf; cp != cpend; cp++)
592			if (cp->buf != NULL)
593				kmem_free(cp->buf, cp->size);
594		kmem_free(old->cbuf, old->ncbuf * sizeof (cbuf_t));
595
596		/* reserved VM for dumpsys_get_maxmem */
597		if (old->maxvmsize > 0)
598			vmem_xfree(heap_arena, old->maxvm, old->maxvmsize);
599	}
600
601	/*
602	 * Allocate memory and VM.
603	 * One CPU runs dumpsys, the rest are helpers.
604	 */
605	new->nhelper = ncpus - 1;
606	if (new->nhelper < 1)
607		new->nhelper = 1;
608
609	if (new->nhelper > DUMP_MAX_NHELPER)
610		new->nhelper = DUMP_MAX_NHELPER;
611
612	/* use platform default, unless /etc/system overrides */
613	if (dump_plat_mincpu == MINCPU_NOT_SET)
614		dump_plat_mincpu = dump_plat_mincpu_default;
615
616	/* increase threshold for faster disks */
617	new->threshold = dump_plat_mincpu;
618	if (dumpbuf.iosize >= DUMP_1MB)
619		new->threshold *= 3;
620	else if (dumpbuf.iosize >= (256 * DUMP_1KB))
621		new->threshold *= 2;
622
623	/* figure compression level based upon the computed threshold. */
624	if (dump_plat_mincpu == 0 || new->nhelper < 2) {
625		new->clevel = 0;
626		new->nhelper = 1;
627	} else if ((new->nhelper + 1) >= new->threshold) {
628		new->clevel = DUMP_CLEVEL_BZIP2;
629	} else {
630		new->clevel = DUMP_CLEVEL_LZJB;
631	}
632
633	if (new->clevel == 0) {
634		new->ncbuf = 1;
635		new->ncmap = 1;
636	} else {
637		new->ncbuf = NCBUF_PER_HELPER * new->nhelper;
638		new->ncmap = NCMAP_PER_HELPER * new->nhelper;
639	}
640
641	/*
642	 * Allocate new data structures and buffers for MINHELPERS,
643	 * and also figure the max desired size.
644	 */
645	bz2size = BZ2_bzCompressInitSize(dump_bzip2_level);
646	new->maxsize = 0;
647	new->maxvmsize = 0;
648	new->maxvm = NULL;
649	tag = 1;
650	new->helper = kmem_zalloc(new->nhelper * sizeof (helper_t), KM_SLEEP);
651	hpend = &new->helper[new->nhelper];
652	for (hp = new->helper; hp != hpend; hp++) {
653		hp->tag = tag++;
654		if (hp < &new->helper[MINHELPERS]) {
655			hp->lzbuf = kmem_alloc(PAGESIZE, KM_SLEEP);
656			hp->page = kmem_alloc(PAGESIZE, KM_SLEEP);
657		} else if (new->clevel < DUMP_CLEVEL_BZIP2) {
658			new->maxsize += 2 * PAGESIZE;
659		} else {
660			new->maxsize += PAGESIZE;
661		}
662		if (new->clevel >= DUMP_CLEVEL_BZIP2)
663			new->maxsize += bz2size;
664	}
665
666	new->cbuf = kmem_zalloc(new->ncbuf * sizeof (cbuf_t), KM_SLEEP);
667	cpend = &new->cbuf[new->ncbuf];
668	for (cp = new->cbuf; cp != cpend; cp++) {
669		cp->state = CBUF_FREEBUF;
670		cp->size = CBUF_SIZE;
671		if (cp < &new->cbuf[MINCBUFS])
672			cp->buf = kmem_alloc(cp->size, KM_SLEEP);
673		else
674			new->maxsize += cp->size;
675	}
676
677	new->cmap = kmem_zalloc(new->ncmap * sizeof (cbuf_t), KM_SLEEP);
678	cpend = &new->cmap[new->ncmap];
679	for (cp = new->cmap; cp != cpend; cp++) {
680		cp->state = CBUF_FREEMAP;
681		cp->size = CBUF_MAPSIZE;
682		cp->buf = vmem_xalloc(heap_arena, CBUF_MAPSIZE, CBUF_MAPSIZE,
683		    0, 0, NULL, NULL, VM_SLEEP);
684	}
685
686	/* reserve VA to be backed with spare pages at crash time */
687	if (new->maxsize > 0) {
688		new->maxsize = P2ROUNDUP(new->maxsize, PAGESIZE);
689		new->maxvmsize = P2ROUNDUP(new->maxsize, CBUF_MAPSIZE);
690		new->maxvm = vmem_xalloc(heap_arena, new->maxvmsize,
691		    CBUF_MAPSIZE, 0, 0, NULL, NULL, VM_SLEEP);
692	}
693
694	/*
695	 * Reserve memory for kmem allocation calls made during crash dump.  The
696	 * hat layer allocates memory for each mapping created, and the I/O path
697	 * allocates buffers and data structs.
698	 *
699	 * On larger systems, we easily exceed the lower amount, so we need some
700	 * more space; the cut-over point is relatively arbitrary.  If we run
701	 * out, the only impact is that kmem state in the dump becomes
702	 * inconsistent.
703	 */
704
705	if (dump_kmem_pages == 0) {
706		if (physmem > (16 * ONE_GIG) / PAGESIZE)
707			dump_kmem_pages = 20;
708		else
709			dump_kmem_pages = 8;
710	}
711
712	kmem_dump_init((new->ncmap * dump_kmem_permap) +
713	    (dump_kmem_pages * PAGESIZE));
714
715	/* set new config pointers */
716	*old = *new;
717}
718
719/*
720 * Define a struct memlist walker to optimize bitnum to pfn
721 * lookup. The walker maintains the state of the list traversal.
722 */
723typedef struct dumpmlw {
724	struct memlist	*mp;		/* current memlist */
725	pgcnt_t		basenum;	/* bitnum base offset */
726	pgcnt_t		mppages;	/* current memlist size */
727	pgcnt_t		mpleft;		/* size to end of current memlist */
728	pfn_t		mpaddr;		/* first pfn in memlist */
729} dumpmlw_t;
730
731/* initialize the walker */
732static inline void
733dump_init_memlist_walker(dumpmlw_t *pw)
734{
735	pw->mp = phys_install;
736	pw->basenum = 0;
737	pw->mppages = pw->mp->ml_size >> PAGESHIFT;
738	pw->mpleft = pw->mppages;
739	pw->mpaddr = pw->mp->ml_address >> PAGESHIFT;
740}
741
742/*
743 * Lookup pfn given bitnum. The memlist can be quite long on some
744 * systems (e.g.: one per board). To optimize sequential lookups, the
745 * caller initializes and presents a memlist walker.
746 */
747static pfn_t
748dump_bitnum_to_pfn(pgcnt_t bitnum, dumpmlw_t *pw)
749{
750	bitnum -= pw->basenum;
751	while (pw->mp != NULL) {
752		if (bitnum < pw->mppages) {
753			pw->mpleft = pw->mppages - bitnum;
754			return (pw->mpaddr + bitnum);
755		}
756		bitnum -= pw->mppages;
757		pw->basenum += pw->mppages;
758		pw->mp = pw->mp->ml_next;
759		if (pw->mp != NULL) {
760			pw->mppages = pw->mp->ml_size >> PAGESHIFT;
761			pw->mpleft = pw->mppages;
762			pw->mpaddr = pw->mp->ml_address >> PAGESHIFT;
763		}
764	}
765	return (PFN_INVALID);
766}
767
768static pgcnt_t
769dump_pfn_to_bitnum(pfn_t pfn)
770{
771	struct memlist *mp;
772	pgcnt_t bitnum = 0;
773
774	for (mp = phys_install; mp != NULL; mp = mp->ml_next) {
775		if (pfn >= (mp->ml_address >> PAGESHIFT) &&
776		    pfn < ((mp->ml_address + mp->ml_size) >> PAGESHIFT))
777			return (bitnum + pfn - (mp->ml_address >> PAGESHIFT));
778		bitnum += mp->ml_size >> PAGESHIFT;
779	}
780	return ((pgcnt_t)-1);
781}
782
783/*
784 * Set/test bitmap for a CBUF_MAPSIZE range which includes pfn. The
785 * mapping of pfn to range index is imperfect because pfn and bitnum
786 * do not have the same phase. To make sure a CBUF_MAPSIZE range is
787 * covered, call this for both ends:
788 *	dump_set_used(base)
789 *	dump_set_used(base+CBUF_MAPNP-1)
790 *
791 * This is used during a panic dump to mark pages allocated by
792 * dumpsys_get_maxmem(). The macro IS_DUMP_PAGE(pp) is used by
793 * page_get_mnode_freelist() to make sure pages used by dump are never
794 * allocated.
795 */
796#define	CBUF_MAPP2R(pfn)	((pfn) >> (CBUF_MAPSHIFT - PAGESHIFT))
797
798static void
799dump_set_used(pfn_t pfn)
800{
801
802	pgcnt_t bitnum, rbitnum;
803
804	bitnum = dump_pfn_to_bitnum(pfn);
805	ASSERT(bitnum != (pgcnt_t)-1);
806
807	rbitnum = CBUF_MAPP2R(bitnum);
808	ASSERT(rbitnum < dumpcfg.rbitmapsize);
809
810	BT_SET(dumpcfg.rbitmap, rbitnum);
811}
812
813int
814dump_test_used(pfn_t pfn)
815{
816	pgcnt_t bitnum, rbitnum;
817
818	bitnum = dump_pfn_to_bitnum(pfn);
819	ASSERT(bitnum != (pgcnt_t)-1);
820
821	rbitnum = CBUF_MAPP2R(bitnum);
822	ASSERT(rbitnum < dumpcfg.rbitmapsize);
823
824	return (BT_TEST(dumpcfg.rbitmap, rbitnum));
825}
826
827/*
828 * dumpbzalloc and dumpbzfree are callbacks from the bzip2 library.
829 * dumpsys_get_maxmem() uses them for BZ2_bzCompressInit().
830 */
831static void *
832dumpbzalloc(void *opaque, int items, int size)
833{
834	size_t *sz;
835	char *ret;
836
837	ASSERT(opaque != NULL);
838	sz = opaque;
839	ret = dumpcfg.maxvm + *sz;
840	*sz += items * size;
841	*sz = P2ROUNDUP(*sz, BZ2_BZALLOC_ALIGN);
842	ASSERT(*sz <= dumpcfg.maxvmsize);
843	return (ret);
844}
845
846/*ARGSUSED*/
847static void
848dumpbzfree(void *opaque, void *addr)
849{
850}
851
852/*
853 * Perform additional checks on the page to see if we can really use
854 * it. The kernel (kas) pages are always set in the bitmap. However,
855 * boot memory pages (prom_ppages or P_BOOTPAGES) are not in the
856 * bitmap. So we check for them.
857 */
858static inline int
859dump_pfn_check(pfn_t pfn)
860{
861	page_t *pp = page_numtopp_nolock(pfn);
862	if (pp == NULL || pp->p_pagenum != pfn ||
863#if defined(__sparc)
864	    pp->p_vnode == &promvp ||
865#else
866	    PP_ISBOOTPAGES(pp) ||
867#endif
868	    pp->p_toxic != 0)
869		return (0);
870	return (1);
871}
872
873/*
874 * Check a range to see if all contained pages are available and
875 * return non-zero if the range can be used.
876 */
877static inline int
878dump_range_check(pgcnt_t start, pgcnt_t end, pfn_t pfn)
879{
880	for (; start < end; start++, pfn++) {
881		if (BT_TEST(dumpcfg.bitmap, start))
882			return (0);
883		if (!dump_pfn_check(pfn))
884			return (0);
885	}
886	return (1);
887}
888
889/*
890 * dumpsys_get_maxmem() is called during panic. Find unused ranges
891 * and use them for buffers. If we find enough memory switch to
892 * parallel bzip2, otherwise use parallel lzjb.
893 *
894 * It searches the dump bitmap in 2 passes. The first time it looks
895 * for CBUF_MAPSIZE ranges. On the second pass it uses small pages.
896 */
897static void
898dumpsys_get_maxmem()
899{
900	dumpcfg_t *cfg = &dumpcfg;
901	cbuf_t *endcp = &cfg->cbuf[cfg->ncbuf];
902	helper_t *endhp = &cfg->helper[cfg->nhelper];
903	pgcnt_t bitnum, end;
904	size_t sz, endsz, bz2size;
905	pfn_t pfn, off;
906	cbuf_t *cp;
907	helper_t *hp, *ohp;
908	dumpmlw_t mlw;
909	int k;
910
911	/*
912	 * Setting dump_plat_mincpu to 0 at any time forces a serial
913	 * dump.
914	 */
915	if (dump_plat_mincpu == 0) {
916		cfg->clevel = 0;
917		return;
918	}
919
920	/*
921	 * There may be no point in looking for spare memory. If
922	 * dumping all memory, then none is spare. If doing a serial
923	 * dump, then already have buffers.
924	 */
925	if (cfg->maxsize == 0 || cfg->clevel < DUMP_CLEVEL_LZJB ||
926	    (dump_conflags & DUMP_ALL) != 0) {
927		if (cfg->clevel > DUMP_CLEVEL_LZJB)
928			cfg->clevel = DUMP_CLEVEL_LZJB;
929		return;
930	}
931
932	sz = 0;
933	cfg->found4m = 0;
934	cfg->foundsm = 0;
935
936	/* bitmap of ranges used to estimate which pfns are being used */
937	bzero(dumpcfg.rbitmap, BT_SIZEOFMAP(dumpcfg.rbitmapsize));
938
939	/* find ranges that are not being dumped to use for buffers */
940	dump_init_memlist_walker(&mlw);
941	for (bitnum = 0; bitnum < dumpcfg.bitmapsize; bitnum = end) {
942		dump_timeleft = dump_timeout;
943		end = bitnum + CBUF_MAPNP;
944		pfn = dump_bitnum_to_pfn(bitnum, &mlw);
945		ASSERT(pfn != PFN_INVALID);
946
947		/* skip partial range at end of mem segment */
948		if (mlw.mpleft < CBUF_MAPNP) {
949			end = bitnum + mlw.mpleft;
950			continue;
951		}
952
953		/* skip non aligned pages */
954		off = P2PHASE(pfn, CBUF_MAPNP);
955		if (off != 0) {
956			end -= off;
957			continue;
958		}
959
960		if (!dump_range_check(bitnum, end, pfn))
961			continue;
962
963		ASSERT((sz + CBUF_MAPSIZE) <= cfg->maxvmsize);
964		hat_devload(kas.a_hat, cfg->maxvm + sz, CBUF_MAPSIZE, pfn,
965		    PROT_READ | PROT_WRITE, HAT_LOAD_NOCONSIST);
966		sz += CBUF_MAPSIZE;
967		cfg->found4m++;
968
969		/* set the bitmap for both ends to be sure to cover the range */
970		dump_set_used(pfn);
971		dump_set_used(pfn + CBUF_MAPNP - 1);
972
973		if (sz >= cfg->maxsize)
974			goto foundmax;
975	}
976
977	/* Add small pages if we can't find enough large pages. */
978	dump_init_memlist_walker(&mlw);
979	for (bitnum = 0; bitnum < dumpcfg.bitmapsize; bitnum = end) {
980		dump_timeleft = dump_timeout;
981		end = bitnum + CBUF_MAPNP;
982		pfn = dump_bitnum_to_pfn(bitnum, &mlw);
983		ASSERT(pfn != PFN_INVALID);
984
985		/* Find any non-aligned pages at start and end of segment. */
986		off = P2PHASE(pfn, CBUF_MAPNP);
987		if (mlw.mpleft < CBUF_MAPNP) {
988			end = bitnum + mlw.mpleft;
989		} else if (off != 0) {
990			end -= off;
991		} else if (cfg->found4m && dump_test_used(pfn)) {
992			continue;
993		}
994
995		for (; bitnum < end; bitnum++, pfn++) {
996			dump_timeleft = dump_timeout;
997			if (BT_TEST(dumpcfg.bitmap, bitnum))
998				continue;
999			if (!dump_pfn_check(pfn))
1000				continue;
1001			ASSERT((sz + PAGESIZE) <= cfg->maxvmsize);
1002			hat_devload(kas.a_hat, cfg->maxvm + sz, PAGESIZE, pfn,
1003			    PROT_READ | PROT_WRITE, HAT_LOAD_NOCONSIST);
1004			sz += PAGESIZE;
1005			cfg->foundsm++;
1006			dump_set_used(pfn);
1007			if (sz >= cfg->maxsize)
1008				goto foundmax;
1009		}
1010	}
1011
1012	/* Fall back to lzjb if we did not get enough memory for bzip2. */
1013	endsz = (cfg->maxsize * cfg->threshold) / cfg->nhelper;
1014	if (sz < endsz) {
1015		cfg->clevel = DUMP_CLEVEL_LZJB;
1016	}
1017
1018	/* Allocate memory for as many helpers as we can. */
1019foundmax:
1020
1021	/* Byte offsets into memory found and mapped above */
1022	endsz = sz;
1023	sz = 0;
1024
1025	/* Set the size for bzip2 state. Only bzip2 needs it. */
1026	bz2size = BZ2_bzCompressInitSize(dump_bzip2_level);
1027
1028	/* Skip the preallocate output buffers. */
1029	cp = &cfg->cbuf[MINCBUFS];
1030
1031	/* Use this to move memory up from the preallocated helpers. */
1032	ohp = cfg->helper;
1033
1034	/* Loop over all helpers and allocate memory. */
1035	for (hp = cfg->helper; hp < endhp; hp++) {
1036
1037		/* Skip preallocated helpers by checking hp->page. */
1038		if (hp->page == NULL) {
1039			if (cfg->clevel <= DUMP_CLEVEL_LZJB) {
1040				/* lzjb needs 2 1-page buffers */
1041				if ((sz + (2 * PAGESIZE)) > endsz)
1042					break;
1043				hp->page = cfg->maxvm + sz;
1044				sz += PAGESIZE;
1045				hp->lzbuf = cfg->maxvm + sz;
1046				sz += PAGESIZE;
1047
1048			} else if (ohp->lzbuf != NULL) {
1049				/* re-use the preallocted lzjb page for bzip2 */
1050				hp->page = ohp->lzbuf;
1051				ohp->lzbuf = NULL;
1052				++ohp;
1053
1054			} else {
1055				/* bzip2 needs a 1-page buffer */
1056				if ((sz + PAGESIZE) > endsz)
1057					break;
1058				hp->page = cfg->maxvm + sz;
1059				sz += PAGESIZE;
1060			}
1061		}
1062
1063		/*
1064		 * Add output buffers per helper. The number of
1065		 * buffers per helper is determined by the ratio of
1066		 * ncbuf to nhelper.
1067		 */
1068		for (k = 0; cp < endcp && (sz + CBUF_SIZE) <= endsz &&
1069		    k < NCBUF_PER_HELPER; k++) {
1070			cp->state = CBUF_FREEBUF;
1071			cp->size = CBUF_SIZE;
1072			cp->buf = cfg->maxvm + sz;
1073			sz += CBUF_SIZE;
1074			++cp;
1075		}
1076
1077		/*
1078		 * bzip2 needs compression state. Use the dumpbzalloc
1079		 * and dumpbzfree callbacks to allocate the memory.
1080		 * bzip2 does allocation only at init time.
1081		 */
1082		if (cfg->clevel >= DUMP_CLEVEL_BZIP2) {
1083			if ((sz + bz2size) > endsz) {
1084				hp->page = NULL;
1085				break;
1086			} else {
1087				hp->bzstream.opaque = &sz;
1088				hp->bzstream.bzalloc = dumpbzalloc;
1089				hp->bzstream.bzfree = dumpbzfree;
1090				(void) BZ2_bzCompressInit(&hp->bzstream,
1091				    dump_bzip2_level, 0, 0);
1092				hp->bzstream.opaque = NULL;
1093			}
1094		}
1095	}
1096
1097	/* Finish allocating output buffers */
1098	for (; cp < endcp && (sz + CBUF_SIZE) <= endsz; cp++) {
1099		cp->state = CBUF_FREEBUF;
1100		cp->size = CBUF_SIZE;
1101		cp->buf = cfg->maxvm + sz;
1102		sz += CBUF_SIZE;
1103	}
1104
1105	/* Enable IS_DUMP_PAGE macro, which checks for pages we took. */
1106	if (cfg->found4m || cfg->foundsm)
1107		dump_check_used = 1;
1108
1109	ASSERT(sz <= endsz);
1110}
1111
1112static void
1113dumphdr_init(void)
1114{
1115	pgcnt_t npages = 0;
1116
1117	ASSERT(MUTEX_HELD(&dump_lock));
1118
1119	if (dumphdr == NULL) {
1120		dumphdr = kmem_zalloc(sizeof (dumphdr_t), KM_SLEEP);
1121		dumphdr->dump_magic = DUMP_MAGIC;
1122		dumphdr->dump_version = DUMP_VERSION;
1123		dumphdr->dump_wordsize = DUMP_WORDSIZE;
1124		dumphdr->dump_pageshift = PAGESHIFT;
1125		dumphdr->dump_pagesize = PAGESIZE;
1126		dumphdr->dump_utsname = utsname;
1127		(void) strcpy(dumphdr->dump_platform, platform);
1128		dumpbuf.size = dumpbuf_iosize(maxphys);
1129		dumpbuf.start = kmem_alloc(dumpbuf.size, KM_SLEEP);
1130		dumpbuf.end = dumpbuf.start + dumpbuf.size;
1131		dumpcfg.pids = kmem_alloc(v.v_proc * sizeof (pid_t), KM_SLEEP);
1132		dumpcfg.helpermap = kmem_zalloc(BT_SIZEOFMAP(NCPU), KM_SLEEP);
1133		LOCK_INIT_HELD(&dumpcfg.helper_lock);
1134		dump_stack_scratch = kmem_alloc(STACK_BUF_SIZE, KM_SLEEP);
1135		(void) strncpy(dumphdr->dump_uuid, dump_get_uuid(),
1136		    sizeof (dumphdr->dump_uuid));
1137	}
1138
1139	npages = num_phys_pages();
1140
1141	if (dumpcfg.bitmapsize != npages) {
1142		size_t rlen = CBUF_MAPP2R(P2ROUNDUP(npages, CBUF_MAPNP));
1143		void *map = kmem_alloc(BT_SIZEOFMAP(npages), KM_SLEEP);
1144		void *rmap = kmem_alloc(BT_SIZEOFMAP(rlen), KM_SLEEP);
1145
1146		if (dumpcfg.bitmap != NULL)
1147			kmem_free(dumpcfg.bitmap, BT_SIZEOFMAP(dumpcfg.
1148			    bitmapsize));
1149		if (dumpcfg.rbitmap != NULL)
1150			kmem_free(dumpcfg.rbitmap, BT_SIZEOFMAP(dumpcfg.
1151			    rbitmapsize));
1152		dumpcfg.bitmap = map;
1153		dumpcfg.bitmapsize = npages;
1154		dumpcfg.rbitmap = rmap;
1155		dumpcfg.rbitmapsize = rlen;
1156	}
1157}
1158
1159/*
1160 * Establish a new dump device.
1161 */
1162int
1163dumpinit(vnode_t *vp, char *name, int justchecking)
1164{
1165	vnode_t *cvp;
1166	vattr_t vattr;
1167	vnode_t *cdev_vp;
1168	int error = 0;
1169
1170	ASSERT(MUTEX_HELD(&dump_lock));
1171
1172	dumphdr_init();
1173
1174	cvp = common_specvp(vp);
1175	if (cvp == dumpvp)
1176		return (0);
1177
1178	/*
1179	 * Determine whether this is a plausible dump device.  We want either:
1180	 * (1) a real device that's not mounted and has a cb_dump routine, or
1181	 * (2) a swapfile on some filesystem that has a vop_dump routine.
1182	 */
1183	if ((error = VOP_OPEN(&cvp, FREAD | FWRITE, kcred, NULL)) != 0)
1184		return (error);
1185
1186	vattr.va_mask = AT_SIZE | AT_TYPE | AT_RDEV;
1187	if ((error = VOP_GETATTR(cvp, &vattr, 0, kcred, NULL)) == 0) {
1188		if (vattr.va_type == VBLK || vattr.va_type == VCHR) {
1189			if (devopsp[getmajor(vattr.va_rdev)]->
1190			    devo_cb_ops->cb_dump == nodev)
1191				error = ENOTSUP;
1192			else if (vfs_devismounted(vattr.va_rdev))
1193				error = EBUSY;
1194			if (strcmp(ddi_driver_name(VTOS(cvp)->s_dip),
1195			    ZFS_DRIVER) == 0 &&
1196			    IS_SWAPVP(common_specvp(cvp)))
1197					error = EBUSY;
1198		} else {
1199			if (vn_matchopval(cvp, VOPNAME_DUMP, fs_nosys) ||
1200			    !IS_SWAPVP(cvp))
1201				error = ENOTSUP;
1202		}
1203	}
1204
1205	if (error == 0 && vattr.va_size < 2 * DUMP_LOGSIZE + DUMP_ERPTSIZE)
1206		error = ENOSPC;
1207
1208	if (error || justchecking) {
1209		(void) VOP_CLOSE(cvp, FREAD | FWRITE, 1, (offset_t)0,
1210		    kcred, NULL);
1211		return (error);
1212	}
1213
1214	VN_HOLD(cvp);
1215
1216	if (dumpvp != NULL)
1217		dumpfini();	/* unconfigure the old dump device */
1218
1219	dumpvp = cvp;
1220	dumpvp_size = vattr.va_size & -DUMP_OFFSET;
1221	dumppath = kmem_alloc(strlen(name) + 1, KM_SLEEP);
1222	(void) strcpy(dumppath, name);
1223	dumpbuf.iosize = 0;
1224
1225	/*
1226	 * If the dump device is a block device, attempt to open up the
1227	 * corresponding character device and determine its maximum transfer
1228	 * size.  We use this information to potentially resize dumpbuf to a
1229	 * larger and more optimal size for performing i/o to the dump device.
1230	 */
1231	if (cvp->v_type == VBLK &&
1232	    (cdev_vp = makespecvp(VTOS(cvp)->s_dev, VCHR)) != NULL) {
1233		if (VOP_OPEN(&cdev_vp, FREAD | FWRITE, kcred, NULL) == 0) {
1234			size_t blk_size;
1235			struct dk_cinfo dki;
1236			struct dk_minfo minf;
1237
1238			if (VOP_IOCTL(cdev_vp, DKIOCGMEDIAINFO,
1239			    (intptr_t)&minf, FKIOCTL, kcred, NULL, NULL)
1240			    == 0 && minf.dki_lbsize != 0)
1241				blk_size = minf.dki_lbsize;
1242			else
1243				blk_size = DEV_BSIZE;
1244
1245			if (VOP_IOCTL(cdev_vp, DKIOCINFO, (intptr_t)&dki,
1246			    FKIOCTL, kcred, NULL, NULL) == 0) {
1247				dumpbuf.iosize = dki.dki_maxtransfer * blk_size;
1248				dumpbuf_resize();
1249			}
1250			/*
1251			 * If we are working with a zvol then dumpify it
1252			 * if it's not being used as swap.
1253			 */
1254			if (strcmp(dki.dki_dname, ZVOL_DRIVER) == 0) {
1255				if (IS_SWAPVP(common_specvp(cvp)))
1256					error = EBUSY;
1257				else if ((error = VOP_IOCTL(cdev_vp,
1258				    DKIOCDUMPINIT, 0, FKIOCTL, kcred,
1259				    NULL, NULL)) != 0)
1260					dumpfini();
1261			}
1262
1263			(void) VOP_CLOSE(cdev_vp, FREAD | FWRITE, 1, 0,
1264			    kcred, NULL);
1265		}
1266
1267		VN_RELE(cdev_vp);
1268	}
1269
1270	cmn_err(CE_CONT, "?dump on %s size %llu MB\n", name, dumpvp_size >> 20);
1271
1272	dump_update_clevel();
1273
1274	return (error);
1275}
1276
1277void
1278dumpfini(void)
1279{
1280	vattr_t vattr;
1281	boolean_t is_zfs = B_FALSE;
1282	vnode_t *cdev_vp;
1283	ASSERT(MUTEX_HELD(&dump_lock));
1284
1285	kmem_free(dumppath, strlen(dumppath) + 1);
1286
1287	/*
1288	 * Determine if we are using zvols for our dump device
1289	 */
1290	vattr.va_mask = AT_RDEV;
1291	if (VOP_GETATTR(dumpvp, &vattr, 0, kcred, NULL) == 0) {
1292		is_zfs = (getmajor(vattr.va_rdev) ==
1293		    ddi_name_to_major(ZFS_DRIVER)) ? B_TRUE : B_FALSE;
1294	}
1295
1296	/*
1297	 * If we have a zvol dump device then we call into zfs so
1298	 * that it may have a chance to cleanup.
1299	 */
1300	if (is_zfs &&
1301	    (cdev_vp = makespecvp(VTOS(dumpvp)->s_dev, VCHR)) != NULL) {
1302		if (VOP_OPEN(&cdev_vp, FREAD | FWRITE, kcred, NULL) == 0) {
1303			(void) VOP_IOCTL(cdev_vp, DKIOCDUMPFINI, 0, FKIOCTL,
1304			    kcred, NULL, NULL);
1305			(void) VOP_CLOSE(cdev_vp, FREAD | FWRITE, 1, 0,
1306			    kcred, NULL);
1307		}
1308		VN_RELE(cdev_vp);
1309	}
1310
1311	(void) VOP_CLOSE(dumpvp, FREAD | FWRITE, 1, (offset_t)0, kcred, NULL);
1312
1313	VN_RELE(dumpvp);
1314
1315	dumpvp = NULL;
1316	dumpvp_size = 0;
1317	dumppath = NULL;
1318}
1319
1320static offset_t
1321dumpvp_flush(void)
1322{
1323	size_t size = P2ROUNDUP(dumpbuf.cur - dumpbuf.start, PAGESIZE);
1324	hrtime_t iotime;
1325	int err;
1326
1327	if (dumpbuf.vp_off + size > dumpbuf.vp_limit) {
1328		dump_ioerr = ENOSPC;
1329		dumpbuf.vp_off = dumpbuf.vp_limit;
1330	} else if (size != 0) {
1331		iotime = gethrtime();
1332		dumpsync.iowait += iotime - dumpsync.iowaitts;
1333		if (panicstr)
1334			err = VOP_DUMP(dumpvp, dumpbuf.start,
1335			    lbtodb(dumpbuf.vp_off), btod(size), NULL);
1336		else
1337			err = vn_rdwr(UIO_WRITE, dumpbuf.cdev_vp != NULL ?
1338			    dumpbuf.cdev_vp : dumpvp, dumpbuf.start, size,
1339			    dumpbuf.vp_off, UIO_SYSSPACE, 0, dumpbuf.vp_limit,
1340			    kcred, 0);
1341		if (err && dump_ioerr == 0)
1342			dump_ioerr = err;
1343		dumpsync.iowaitts = gethrtime();
1344		dumpsync.iotime += dumpsync.iowaitts - iotime;
1345		dumpsync.nwrite += size;
1346		dumpbuf.vp_off += size;
1347	}
1348	dumpbuf.cur = dumpbuf.start;
1349	dump_timeleft = dump_timeout;
1350	return (dumpbuf.vp_off);
1351}
1352
1353/* maximize write speed by keeping seek offset aligned with size */
1354void
1355dumpvp_write(const void *va, size_t size)
1356{
1357	size_t len, off, sz;
1358
1359	while (size != 0) {
1360		len = MIN(size, dumpbuf.end - dumpbuf.cur);
1361		if (len == 0) {
1362			off = P2PHASE(dumpbuf.vp_off, dumpbuf.size);
1363			if (off == 0 || !ISP2(dumpbuf.size)) {
1364				(void) dumpvp_flush();
1365			} else {
1366				sz = dumpbuf.size - off;
1367				dumpbuf.cur = dumpbuf.start + sz;
1368				(void) dumpvp_flush();
1369				ovbcopy(dumpbuf.start + sz, dumpbuf.start, off);
1370				dumpbuf.cur += off;
1371			}
1372		} else {
1373			bcopy(va, dumpbuf.cur, len);
1374			va = (char *)va + len;
1375			dumpbuf.cur += len;
1376			size -= len;
1377		}
1378	}
1379}
1380
1381/*ARGSUSED*/
1382static void
1383dumpvp_ksyms_write(const void *src, void *dst, size_t size)
1384{
1385	dumpvp_write(src, size);
1386}
1387
1388/*
1389 * Mark 'pfn' in the bitmap and dump its translation table entry.
1390 */
1391void
1392dump_addpage(struct as *as, void *va, pfn_t pfn)
1393{
1394	mem_vtop_t mem_vtop;
1395	pgcnt_t bitnum;
1396
1397	if ((bitnum = dump_pfn_to_bitnum(pfn)) != (pgcnt_t)-1) {
1398		if (!BT_TEST(dumpcfg.bitmap, bitnum)) {
1399			dumphdr->dump_npages++;
1400			BT_SET(dumpcfg.bitmap, bitnum);
1401		}
1402		dumphdr->dump_nvtop++;
1403		mem_vtop.m_as = as;
1404		mem_vtop.m_va = va;
1405		mem_vtop.m_pfn = pfn;
1406		dumpvp_write(&mem_vtop, sizeof (mem_vtop_t));
1407	}
1408	dump_timeleft = dump_timeout;
1409}
1410
1411/*
1412 * Mark 'pfn' in the bitmap
1413 */
1414void
1415dump_page(pfn_t pfn)
1416{
1417	pgcnt_t bitnum;
1418
1419	if ((bitnum = dump_pfn_to_bitnum(pfn)) != (pgcnt_t)-1) {
1420		if (!BT_TEST(dumpcfg.bitmap, bitnum)) {
1421			dumphdr->dump_npages++;
1422			BT_SET(dumpcfg.bitmap, bitnum);
1423		}
1424	}
1425	dump_timeleft = dump_timeout;
1426}
1427
1428/*
1429 * Dump the <as, va, pfn> information for a given address space.
1430 * SEGOP_DUMP() will call dump_addpage() for each page in the segment.
1431 */
1432static void
1433dump_as(struct as *as)
1434{
1435	struct seg *seg;
1436
1437	AS_LOCK_ENTER(as, RW_READER);
1438	for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) {
1439		if (seg->s_as != as)
1440			break;
1441		if (seg->s_ops == NULL)
1442			continue;
1443		SEGOP_DUMP(seg);
1444	}
1445	AS_LOCK_EXIT(as);
1446
1447	if (seg != NULL)
1448		cmn_err(CE_WARN, "invalid segment %p in address space %p",
1449		    (void *)seg, (void *)as);
1450}
1451
1452static int
1453dump_process(pid_t pid)
1454{
1455	proc_t *p = sprlock(pid);
1456
1457	if (p == NULL)
1458		return (-1);
1459	if (p->p_as != &kas) {
1460		mutex_exit(&p->p_lock);
1461		dump_as(p->p_as);
1462		mutex_enter(&p->p_lock);
1463	}
1464
1465	sprunlock(p);
1466
1467	return (0);
1468}
1469
1470/*
1471 * The following functions (dump_summary(), dump_ereports(), and
1472 * dump_messages()), write data to an uncompressed area within the
1473 * crashdump. The layout of these is
1474 *
1475 * +------------------------------------------------------------+
1476 * |     compressed pages       | summary | ereports | messages |
1477 * +------------------------------------------------------------+
1478 *
1479 * With the advent of saving a compressed crash dump by default, we
1480 * need to save a little more data to describe the failure mode in
1481 * an uncompressed buffer available before savecore uncompresses
1482 * the dump. Initially this is a copy of the stack trace. Additional
1483 * summary information should be added here.
1484 */
1485
1486void
1487dump_summary(void)
1488{
1489	u_offset_t dumpvp_start;
1490	summary_dump_t sd;
1491
1492	if (dumpvp == NULL || dumphdr == NULL)
1493		return;
1494
1495	dumpbuf.cur = dumpbuf.start;
1496
1497	dumpbuf.vp_limit = dumpvp_size - (DUMP_OFFSET + DUMP_LOGSIZE +
1498	    DUMP_ERPTSIZE);
1499	dumpvp_start = dumpbuf.vp_limit - DUMP_SUMMARYSIZE;
1500	dumpbuf.vp_off = dumpvp_start;
1501
1502	sd.sd_magic = SUMMARY_MAGIC;
1503	sd.sd_ssum = checksum32(dump_stack_scratch, STACK_BUF_SIZE);
1504	dumpvp_write(&sd, sizeof (sd));
1505	dumpvp_write(dump_stack_scratch, STACK_BUF_SIZE);
1506
1507	sd.sd_magic = 0; /* indicate end of summary */
1508	dumpvp_write(&sd, sizeof (sd));
1509	(void) dumpvp_flush();
1510}
1511
1512void
1513dump_ereports(void)
1514{
1515	u_offset_t dumpvp_start;
1516	erpt_dump_t ed;
1517
1518	if (dumpvp == NULL || dumphdr == NULL)
1519		return;
1520
1521	dumpbuf.cur = dumpbuf.start;
1522	dumpbuf.vp_limit = dumpvp_size - (DUMP_OFFSET + DUMP_LOGSIZE);
1523	dumpvp_start = dumpbuf.vp_limit - DUMP_ERPTSIZE;
1524	dumpbuf.vp_off = dumpvp_start;
1525
1526	fm_ereport_dump();
1527	if (panicstr)
1528		errorq_dump();
1529
1530	bzero(&ed, sizeof (ed)); /* indicate end of ereports */
1531	dumpvp_write(&ed, sizeof (ed));
1532	(void) dumpvp_flush();
1533
1534	if (!panicstr) {
1535		(void) VOP_PUTPAGE(dumpvp, dumpvp_start,
1536		    (size_t)(dumpbuf.vp_off - dumpvp_start),
1537		    B_INVAL | B_FORCE, kcred, NULL);
1538	}
1539}
1540
1541void
1542dump_messages(void)
1543{
1544	log_dump_t ld;
1545	mblk_t *mctl, *mdata;
1546	queue_t *q, *qlast;
1547	u_offset_t dumpvp_start;
1548
1549	if (dumpvp == NULL || dumphdr == NULL || log_consq == NULL)
1550		return;
1551
1552	dumpbuf.cur = dumpbuf.start;
1553	dumpbuf.vp_limit = dumpvp_size - DUMP_OFFSET;
1554	dumpvp_start = dumpbuf.vp_limit - DUMP_LOGSIZE;
1555	dumpbuf.vp_off = dumpvp_start;
1556
1557	qlast = NULL;
1558	do {
1559		for (q = log_consq; q->q_next != qlast; q = q->q_next)
1560			continue;
1561		for (mctl = q->q_first; mctl != NULL; mctl = mctl->b_next) {
1562			dump_timeleft = dump_timeout;
1563			mdata = mctl->b_cont;
1564			ld.ld_magic = LOG_MAGIC;
1565			ld.ld_msgsize = MBLKL(mctl->b_cont);
1566			ld.ld_csum = checksum32(mctl->b_rptr, MBLKL(mctl));
1567			ld.ld_msum = checksum32(mdata->b_rptr, MBLKL(mdata));
1568			dumpvp_write(&ld, sizeof (ld));
1569			dumpvp_write(mctl->b_rptr, MBLKL(mctl));
1570			dumpvp_write(mdata->b_rptr, MBLKL(mdata));
1571		}
1572	} while ((qlast = q) != log_consq);
1573
1574	ld.ld_magic = 0;		/* indicate end of messages */
1575	dumpvp_write(&ld, sizeof (ld));
1576	(void) dumpvp_flush();
1577	if (!panicstr) {
1578		(void) VOP_PUTPAGE(dumpvp, dumpvp_start,
1579		    (size_t)(dumpbuf.vp_off - dumpvp_start),
1580		    B_INVAL | B_FORCE, kcred, NULL);
1581	}
1582}
1583
1584/*
1585 * The following functions are called on multiple CPUs during dump.
1586 * They must not use most kernel services, because all cross-calls are
1587 * disabled during panic. Therefore, blocking locks and cache flushes
1588 * will not work.
1589 */
1590
1591/*
1592 * Copy pages, trapping ECC errors. Also, for robustness, trap data
1593 * access in case something goes wrong in the hat layer and the
1594 * mapping is broken.
1595 */
1596static int
1597dump_pagecopy(void *src, void *dst)
1598{
1599	long *wsrc = (long *)src;
1600	long *wdst = (long *)dst;
1601	const ulong_t ncopies = PAGESIZE / sizeof (long);
1602	volatile int w = 0;
1603	volatile int ueoff = -1;
1604	on_trap_data_t otd;
1605
1606	if (on_trap(&otd, OT_DATA_EC | OT_DATA_ACCESS)) {
1607		if (ueoff == -1)
1608			ueoff = w * sizeof (long);
1609		/* report "bad ECC" or "bad address" */
1610#ifdef _LP64
1611		if (otd.ot_trap & OT_DATA_EC)
1612			wdst[w++] = 0x00badecc00badecc;
1613		else
1614			wdst[w++] = 0x00badadd00badadd;
1615#else
1616		if (otd.ot_trap & OT_DATA_EC)
1617			wdst[w++] = 0x00badecc;
1618		else
1619			wdst[w++] = 0x00badadd;
1620#endif
1621	}
1622	while (w < ncopies) {
1623		wdst[w] = wsrc[w];
1624		w++;
1625	}
1626	no_trap();
1627	return (ueoff);
1628}
1629
1630static void
1631dumpsys_close_cq(cqueue_t *cq, int live)
1632{
1633	if (live) {
1634		mutex_enter(&cq->mutex);
1635		atomic_dec_uint(&cq->open);
1636		cv_signal(&cq->cv);
1637		mutex_exit(&cq->mutex);
1638	} else {
1639		atomic_dec_uint(&cq->open);
1640	}
1641}
1642
1643static inline void
1644dumpsys_spinlock(lock_t *lp)
1645{
1646	uint_t backoff = 0;
1647	int loop_count = 0;
1648
1649	while (LOCK_HELD(lp) || !lock_spin_try(lp)) {
1650		if (++loop_count >= ncpus) {
1651			backoff = mutex_lock_backoff(0);
1652			loop_count = 0;
1653		} else {
1654			backoff = mutex_lock_backoff(backoff);
1655		}
1656		mutex_lock_delay(backoff);
1657	}
1658}
1659
1660static inline void
1661dumpsys_spinunlock(lock_t *lp)
1662{
1663	lock_clear(lp);
1664}
1665
1666static inline void
1667dumpsys_lock(cqueue_t *cq, int live)
1668{
1669	if (live)
1670		mutex_enter(&cq->mutex);
1671	else
1672		dumpsys_spinlock(&cq->spinlock);
1673}
1674
1675static inline void
1676dumpsys_unlock(cqueue_t *cq, int live, int signal)
1677{
1678	if (live) {
1679		if (signal)
1680			cv_signal(&cq->cv);
1681		mutex_exit(&cq->mutex);
1682	} else {
1683		dumpsys_spinunlock(&cq->spinlock);
1684	}
1685}
1686
1687static void
1688dumpsys_wait_cq(cqueue_t *cq, int live)
1689{
1690	if (live) {
1691		cv_wait(&cq->cv, &cq->mutex);
1692	} else {
1693		dumpsys_spinunlock(&cq->spinlock);
1694		while (cq->open)
1695			if (cq->first)
1696				break;
1697		dumpsys_spinlock(&cq->spinlock);
1698	}
1699}
1700
1701static void
1702dumpsys_put_cq(cqueue_t *cq, cbuf_t *cp, int newstate, int live)
1703{
1704	if (cp == NULL)
1705		return;
1706
1707	dumpsys_lock(cq, live);
1708
1709	if (cq->ts != 0) {
1710		cq->empty += gethrtime() - cq->ts;
1711		cq->ts = 0;
1712	}
1713
1714	cp->state = newstate;
1715	cp->next = NULL;
1716	if (cq->last == NULL)
1717		cq->first = cp;
1718	else
1719		cq->last->next = cp;
1720	cq->last = cp;
1721
1722	dumpsys_unlock(cq, live, 1);
1723}
1724
1725static cbuf_t *
1726dumpsys_get_cq(cqueue_t *cq, int live)
1727{
1728	cbuf_t *cp;
1729	hrtime_t now = gethrtime();
1730
1731	dumpsys_lock(cq, live);
1732
1733	/* CONSTCOND */
1734	while (1) {
1735		cp = (cbuf_t *)cq->first;
1736		if (cp == NULL) {
1737			if (cq->open == 0)
1738				break;
1739			dumpsys_wait_cq(cq, live);
1740			continue;
1741		}
1742		cq->first = cp->next;
1743		if (cq->first == NULL) {
1744			cq->last = NULL;
1745			cq->ts = now;
1746		}
1747		break;
1748	}
1749
1750	dumpsys_unlock(cq, live, cq->first != NULL || cq->open == 0);
1751	return (cp);
1752}
1753
1754/*
1755 * Send an error message to the console. If the main task is running
1756 * just write the message via uprintf. If a helper is running the
1757 * message has to be put on a queue for the main task. Setting fmt to
1758 * NULL means flush the error message buffer. If fmt is not NULL, just
1759 * add the text to the existing buffer.
1760 */
1761static void
1762dumpsys_errmsg(helper_t *hp, const char *fmt, ...)
1763{
1764	dumpsync_t *ds = hp->ds;
1765	cbuf_t *cp = hp->cperr;
1766	va_list adx;
1767
1768	if (hp->helper == MAINHELPER) {
1769		if (fmt != NULL) {
1770			if (ds->neednl) {
1771				uprintf("\n");
1772				ds->neednl = 0;
1773			}
1774			va_start(adx, fmt);
1775			vuprintf(fmt, adx);
1776			va_end(adx);
1777		}
1778	} else if (fmt == NULL) {
1779		if (cp != NULL) {
1780			CQ_PUT(mainq, cp, CBUF_ERRMSG);
1781			hp->cperr = NULL;
1782		}
1783	} else {
1784		if (hp->cperr == NULL) {
1785			cp = CQ_GET(freebufq);
1786			hp->cperr = cp;
1787			cp->used = 0;
1788		}
1789		va_start(adx, fmt);
1790		cp->used += vsnprintf(cp->buf + cp->used, cp->size - cp->used,
1791		    fmt, adx);
1792		va_end(adx);
1793		if ((cp->used + LOG_MSGSIZE) > cp->size) {
1794			CQ_PUT(mainq, cp, CBUF_ERRMSG);
1795			hp->cperr = NULL;
1796		}
1797	}
1798}
1799
1800/*
1801 * Write an output buffer to the dump file. If the main task is
1802 * running just write the data. If a helper is running the output is
1803 * placed on a queue for the main task.
1804 */
1805static void
1806dumpsys_swrite(helper_t *hp, cbuf_t *cp, size_t used)
1807{
1808	dumpsync_t *ds = hp->ds;
1809
1810	if (hp->helper == MAINHELPER) {
1811		HRSTART(ds->perpage, write);
1812		dumpvp_write(cp->buf, used);
1813		HRSTOP(ds->perpage, write);
1814		CQ_PUT(freebufq, cp, CBUF_FREEBUF);
1815	} else {
1816		cp->used = used;
1817		CQ_PUT(mainq, cp, CBUF_WRITE);
1818	}
1819}
1820
1821/*
1822 * Copy one page within the mapped range. The offset starts at 0 and
1823 * is relative to the first pfn. cp->buf + cp->off is the address of
1824 * the first pfn. If dump_pagecopy returns a UE offset, create an
1825 * error message.  Returns the offset to the next pfn in the range
1826 * selected by the bitmap.
1827 */
1828static int
1829dumpsys_copy_page(helper_t *hp, int offset)
1830{
1831	cbuf_t *cp = hp->cpin;
1832	int ueoff;
1833
1834	ASSERT(cp->off + offset + PAGESIZE <= cp->size);
1835	ASSERT(BT_TEST(dumpcfg.bitmap, cp->bitnum));
1836
1837	ueoff = dump_pagecopy(cp->buf + cp->off + offset, hp->page);
1838
1839	/* ueoff is the offset in the page to a UE error */
1840	if (ueoff != -1) {
1841		uint64_t pa = ptob(cp->pfn) + offset + ueoff;
1842
1843		dumpsys_errmsg(hp, "cpu %d: memory error at PA 0x%08x.%08x\n",
1844		    CPU->cpu_id, (uint32_t)(pa >> 32), (uint32_t)pa);
1845	}
1846
1847	/*
1848	 * Advance bitnum and offset to the next input page for the
1849	 * next call to this function.
1850	 */
1851	offset += PAGESIZE;
1852	cp->bitnum++;
1853	while (cp->off + offset < cp->size) {
1854		if (BT_TEST(dumpcfg.bitmap, cp->bitnum))
1855			break;
1856		offset += PAGESIZE;
1857		cp->bitnum++;
1858	}
1859
1860	return (offset);
1861}
1862
1863/*
1864 * Read the helper queue, and copy one mapped page. Return 0 when
1865 * done. Return 1 when a page has been copied into hp->page.
1866 */
1867static int
1868dumpsys_sread(helper_t *hp)
1869{
1870	dumpsync_t *ds = hp->ds;
1871
1872	/* CONSTCOND */
1873	while (1) {
1874
1875		/* Find the next input buffer. */
1876		if (hp->cpin == NULL) {
1877			HRSTART(hp->perpage, inwait);
1878
1879			/* CONSTCOND */
1880			while (1) {
1881				hp->cpin = CQ_GET(helperq);
1882				dump_timeleft = dump_timeout;
1883
1884				/*
1885				 * NULL return means the helper queue
1886				 * is closed and empty.
1887				 */
1888				if (hp->cpin == NULL)
1889					break;
1890
1891				/* Have input, check for dump I/O error. */
1892				if (!dump_ioerr)
1893					break;
1894
1895				/*
1896				 * If an I/O error occurs, stay in the
1897				 * loop in order to empty the helper
1898				 * queue. Return the buffers to the
1899				 * main task to unmap and free it.
1900				 */
1901				hp->cpin->used = 0;
1902				CQ_PUT(mainq, hp->cpin, CBUF_USEDMAP);
1903			}
1904			HRSTOP(hp->perpage, inwait);
1905
1906			/* Stop here when the helper queue is closed. */
1907			if (hp->cpin == NULL)
1908				break;
1909
1910			/* Set the offset=0 to get the first pfn. */
1911			hp->in = 0;
1912
1913			/* Set the total processed to 0 */
1914			hp->used = 0;
1915		}
1916
1917		/* Process the next page. */
1918		if (hp->used < hp->cpin->used) {
1919
1920			/*
1921			 * Get the next page from the input buffer and
1922			 * return a copy.
1923			 */
1924			ASSERT(hp->in != -1);
1925			HRSTART(hp->perpage, copy);
1926			hp->in = dumpsys_copy_page(hp, hp->in);
1927			hp->used += PAGESIZE;
1928			HRSTOP(hp->perpage, copy);
1929			break;
1930
1931		} else {
1932
1933			/*
1934			 * Done with the input. Flush the VM and
1935			 * return the buffer to the main task.
1936			 */
1937			if (panicstr && hp->helper != MAINHELPER)
1938				hat_flush_range(kas.a_hat,
1939				    hp->cpin->buf, hp->cpin->size);
1940			dumpsys_errmsg(hp, NULL);
1941			CQ_PUT(mainq, hp->cpin, CBUF_USEDMAP);
1942			hp->cpin = NULL;
1943		}
1944	}
1945
1946	return (hp->cpin != NULL);
1947}
1948
1949/*
1950 * Compress size bytes starting at buf with bzip2
1951 * mode:
1952 *	BZ_RUN		add one more compressed page
1953 *	BZ_FINISH	no more input, flush the state
1954 */
1955static void
1956dumpsys_bzrun(helper_t *hp, void *buf, size_t size, int mode)
1957{
1958	dumpsync_t *ds = hp->ds;
1959	const int CSIZE = sizeof (dumpcsize_t);
1960	bz_stream *ps = &hp->bzstream;
1961	int rc = 0;
1962	uint32_t csize;
1963	dumpcsize_t cs;
1964
1965	/* Set input pointers to new input page */
1966	if (size > 0) {
1967		ps->avail_in = size;
1968		ps->next_in = buf;
1969	}
1970
1971	/* CONSTCOND */
1972	while (1) {
1973
1974		/* Quit when all input has been consumed */
1975		if (ps->avail_in == 0 && mode == BZ_RUN)
1976			break;
1977
1978		/* Get a new output buffer */
1979		if (hp->cpout == NULL) {
1980			HRSTART(hp->perpage, outwait);
1981			hp->cpout = CQ_GET(freebufq);
1982			HRSTOP(hp->perpage, outwait);
1983			ps->avail_out = hp->cpout->size - CSIZE;
1984			ps->next_out = hp->cpout->buf + CSIZE;
1985		}
1986
1987		/* Compress input, or finalize */
1988		HRSTART(hp->perpage, compress);
1989		rc = BZ2_bzCompress(ps, mode);
1990		HRSTOP(hp->perpage, compress);
1991
1992		/* Check for error */
1993		if (mode == BZ_RUN && rc != BZ_RUN_OK) {
1994			dumpsys_errmsg(hp, "%d: BZ_RUN error %s at page %lx\n",
1995			    hp->helper, BZ2_bzErrorString(rc),
1996			    hp->cpin->pagenum);
1997			break;
1998		}
1999
2000		/* Write the buffer if it is full, or we are flushing */
2001		if (ps->avail_out == 0 || mode == BZ_FINISH) {
2002			csize = hp->cpout->size - CSIZE - ps->avail_out;
2003			cs = DUMP_SET_TAG(csize, hp->tag);
2004			if (csize > 0) {
2005				(void) memcpy(hp->cpout->buf, &cs, CSIZE);
2006				dumpsys_swrite(hp, hp->cpout, csize + CSIZE);
2007				hp->cpout = NULL;
2008			}
2009		}
2010
2011		/* Check for final complete */
2012		if (mode == BZ_FINISH) {
2013			if (rc == BZ_STREAM_END)
2014				break;
2015			if (rc != BZ_FINISH_OK) {
2016				dumpsys_errmsg(hp, "%d: BZ_FINISH error %s\n",
2017				    hp->helper, BZ2_bzErrorString(rc));
2018				break;
2019			}
2020		}
2021	}
2022
2023	/* Cleanup state and buffers */
2024	if (mode == BZ_FINISH) {
2025
2026		/* Reset state so that it is re-usable. */
2027		(void) BZ2_bzCompressReset(&hp->bzstream);
2028
2029		/* Give any unused outout buffer to the main task */
2030		if (hp->cpout != NULL) {
2031			hp->cpout->used = 0;
2032			CQ_PUT(mainq, hp->cpout, CBUF_ERRMSG);
2033			hp->cpout = NULL;
2034		}
2035	}
2036}
2037
2038static void
2039dumpsys_bz2compress(helper_t *hp)
2040{
2041	dumpsync_t *ds = hp->ds;
2042	dumpstreamhdr_t sh;
2043
2044	(void) strcpy(sh.stream_magic, DUMP_STREAM_MAGIC);
2045	sh.stream_pagenum = (pgcnt_t)-1;
2046	sh.stream_npages = 0;
2047	hp->cpin = NULL;
2048	hp->cpout = NULL;
2049	hp->cperr = NULL;
2050	hp->in = 0;
2051	hp->out = 0;
2052	hp->bzstream.avail_in = 0;
2053
2054	/* Bump reference to mainq while we are running */
2055	CQ_OPEN(mainq);
2056
2057	/* Get one page at a time */
2058	while (dumpsys_sread(hp)) {
2059		if (sh.stream_pagenum != hp->cpin->pagenum) {
2060			sh.stream_pagenum = hp->cpin->pagenum;
2061			sh.stream_npages = btop(hp->cpin->used);
2062			dumpsys_bzrun(hp, &sh, sizeof (sh), BZ_RUN);
2063		}
2064		dumpsys_bzrun(hp, hp->page, PAGESIZE, 0);
2065	}
2066
2067	/* Done with input, flush any partial buffer */
2068	if (sh.stream_pagenum != (pgcnt_t)-1) {
2069		dumpsys_bzrun(hp, NULL, 0, BZ_FINISH);
2070		dumpsys_errmsg(hp, NULL);
2071	}
2072
2073	ASSERT(hp->cpin == NULL && hp->cpout == NULL && hp->cperr == NULL);
2074
2075	/* Decrement main queue count, we are done */
2076	CQ_CLOSE(mainq);
2077}
2078
2079/*
2080 * Compress with lzjb
2081 * write stream block if full or size==0
2082 * if csize==0 write stream header, else write <csize, data>
2083 * size==0 is a call to flush a buffer
2084 * hp->cpout is the buffer we are flushing or filling
2085 * hp->out is the next index to fill data
2086 * osize is either csize+data, or the size of a stream header
2087 */
2088static void
2089dumpsys_lzjbrun(helper_t *hp, size_t csize, void *buf, size_t size)
2090{
2091	dumpsync_t *ds = hp->ds;
2092	const int CSIZE = sizeof (dumpcsize_t);
2093	dumpcsize_t cs;
2094	size_t osize = csize > 0 ? CSIZE + size : size;
2095
2096	/* If flush, and there is no buffer, just return */
2097	if (size == 0 && hp->cpout == NULL)
2098		return;
2099
2100	/* If flush, or cpout is full, write it out */
2101	if (size == 0 ||
2102	    hp->cpout != NULL && hp->out + osize > hp->cpout->size) {
2103
2104		/* Set tag+size word at the front of the stream block. */
2105		cs = DUMP_SET_TAG(hp->out - CSIZE, hp->tag);
2106		(void) memcpy(hp->cpout->buf, &cs, CSIZE);
2107
2108		/* Write block to dump file. */
2109		dumpsys_swrite(hp, hp->cpout, hp->out);
2110
2111		/* Clear pointer to indicate we need a new buffer */
2112		hp->cpout = NULL;
2113
2114		/* flushing, we are done */
2115		if (size == 0)
2116			return;
2117	}
2118
2119	/* Get an output buffer if we dont have one. */
2120	if (hp->cpout == NULL) {
2121		HRSTART(hp->perpage, outwait);
2122		hp->cpout = CQ_GET(freebufq);
2123		HRSTOP(hp->perpage, outwait);
2124		hp->out = CSIZE;
2125	}
2126
2127	/* Store csize word. This is the size of compressed data. */
2128	if (csize > 0) {
2129		cs = DUMP_SET_TAG(csize, 0);
2130		(void) memcpy(hp->cpout->buf + hp->out, &cs, CSIZE);
2131		hp->out += CSIZE;
2132	}
2133
2134	/* Store the data. */
2135	(void) memcpy(hp->cpout->buf + hp->out, buf, size);
2136	hp->out += size;
2137}
2138
2139static void
2140dumpsys_lzjbcompress(helper_t *hp)
2141{
2142	dumpsync_t *ds = hp->ds;
2143	size_t csize;
2144	dumpstreamhdr_t sh;
2145
2146	(void) strcpy(sh.stream_magic, DUMP_STREAM_MAGIC);
2147	sh.stream_pagenum = (pfn_t)-1;
2148	sh.stream_npages = 0;
2149	hp->cpin = NULL;
2150	hp->cpout = NULL;
2151	hp->cperr = NULL;
2152	hp->in = 0;
2153	hp->out = 0;
2154
2155	/* Bump reference to mainq while we are running */
2156	CQ_OPEN(mainq);
2157
2158	/* Get one page at a time */
2159	while (dumpsys_sread(hp)) {
2160
2161		/* Create a stream header for each new input map */
2162		if (sh.stream_pagenum != hp->cpin->pagenum) {
2163			sh.stream_pagenum = hp->cpin->pagenum;
2164			sh.stream_npages = btop(hp->cpin->used);
2165			dumpsys_lzjbrun(hp, 0, &sh, sizeof (sh));
2166		}
2167
2168		/* Compress one page */
2169		HRSTART(hp->perpage, compress);
2170		csize = compress(hp->page, hp->lzbuf, PAGESIZE);
2171		HRSTOP(hp->perpage, compress);
2172
2173		/* Add csize+data to output block */
2174		ASSERT(csize > 0 && csize <= PAGESIZE);
2175		dumpsys_lzjbrun(hp, csize, hp->lzbuf, csize);
2176	}
2177
2178	/* Done with input, flush any partial buffer */
2179	if (sh.stream_pagenum != (pfn_t)-1) {
2180		dumpsys_lzjbrun(hp, 0, NULL, 0);
2181		dumpsys_errmsg(hp, NULL);
2182	}
2183
2184	ASSERT(hp->cpin == NULL && hp->cpout == NULL && hp->cperr == NULL);
2185
2186	/* Decrement main queue count, we are done */
2187	CQ_CLOSE(mainq);
2188}
2189
2190/*
2191 * Dump helper called from panic_idle() to compress pages.  CPUs in
2192 * this path must not call most kernel services.
2193 *
2194 * During panic, all but one of the CPUs is idle. These CPUs are used
2195 * as helpers working in parallel to copy and compress memory
2196 * pages. During a panic, however, these processors cannot call any
2197 * kernel services. This is because mutexes become no-ops during
2198 * panic, and, cross-call interrupts are inhibited.  Therefore, during
2199 * panic dump the helper CPUs communicate with the panic CPU using
2200 * memory variables. All memory mapping and I/O is performed by the
2201 * panic CPU.
2202 *
2203 * At dump configuration time, helper_lock is set and helpers_wanted
2204 * is 0. dumpsys() decides whether to set helpers_wanted before
2205 * clearing helper_lock.
2206 *
2207 * At panic time, idle CPUs spin-wait on helper_lock, then alternately
2208 * take the lock and become a helper, or return.
2209 */
2210void
2211dumpsys_helper()
2212{
2213	dumpsys_spinlock(&dumpcfg.helper_lock);
2214	if (dumpcfg.helpers_wanted) {
2215		helper_t *hp, *hpend = &dumpcfg.helper[dumpcfg.nhelper];
2216
2217		for (hp = dumpcfg.helper; hp != hpend; hp++) {
2218			if (hp->helper == FREEHELPER) {
2219				hp->helper = CPU->cpu_id;
2220				BT_SET(dumpcfg.helpermap, CPU->cpu_seqid);
2221
2222				dumpsys_spinunlock(&dumpcfg.helper_lock);
2223
2224				if (dumpcfg.clevel < DUMP_CLEVEL_BZIP2)
2225					dumpsys_lzjbcompress(hp);
2226				else
2227					dumpsys_bz2compress(hp);
2228
2229				hp->helper = DONEHELPER;
2230				return;
2231			}
2232		}
2233
2234		/* No more helpers are needed. */
2235		dumpcfg.helpers_wanted = 0;
2236
2237	}
2238	dumpsys_spinunlock(&dumpcfg.helper_lock);
2239}
2240
2241/*
2242 * No-wait helper callable in spin loops.
2243 *
2244 * Do not wait for helper_lock. Just check helpers_wanted. The caller
2245 * may decide to continue. This is the "c)ontinue, s)ync, r)eset? s"
2246 * case.
2247 */
2248void
2249dumpsys_helper_nw()
2250{
2251	if (dumpcfg.helpers_wanted)
2252		dumpsys_helper();
2253}
2254
2255/*
2256 * Dump helper for live dumps.
2257 * These run as a system task.
2258 */
2259static void
2260dumpsys_live_helper(void *arg)
2261{
2262	helper_t *hp = arg;
2263
2264	BT_ATOMIC_SET(dumpcfg.helpermap, CPU->cpu_seqid);
2265	if (dumpcfg.clevel < DUMP_CLEVEL_BZIP2)
2266		dumpsys_lzjbcompress(hp);
2267	else
2268		dumpsys_bz2compress(hp);
2269}
2270
2271/*
2272 * Compress one page with lzjb (single threaded case)
2273 */
2274static void
2275dumpsys_lzjb_page(helper_t *hp, cbuf_t *cp)
2276{
2277	dumpsync_t *ds = hp->ds;
2278	uint32_t csize;
2279
2280	hp->helper = MAINHELPER;
2281	hp->in = 0;
2282	hp->used = 0;
2283	hp->cpin = cp;
2284	while (hp->used < cp->used) {
2285		HRSTART(hp->perpage, copy);
2286		hp->in = dumpsys_copy_page(hp, hp->in);
2287		hp->used += PAGESIZE;
2288		HRSTOP(hp->perpage, copy);
2289
2290		HRSTART(hp->perpage, compress);
2291		csize = compress(hp->page, hp->lzbuf, PAGESIZE);
2292		HRSTOP(hp->perpage, compress);
2293
2294		HRSTART(hp->perpage, write);
2295		dumpvp_write(&csize, sizeof (csize));
2296		dumpvp_write(hp->lzbuf, csize);
2297		HRSTOP(hp->perpage, write);
2298	}
2299	CQ_PUT(mainq, hp->cpin, CBUF_USEDMAP);
2300	hp->cpin = NULL;
2301}
2302
2303/*
2304 * Main task to dump pages. This is called on the dump CPU.
2305 */
2306static void
2307dumpsys_main_task(void *arg)
2308{
2309	dumpsync_t *ds = arg;
2310	pgcnt_t pagenum = 0, bitnum = 0, hibitnum;
2311	dumpmlw_t mlw;
2312	cbuf_t *cp;
2313	pgcnt_t baseoff, pfnoff;
2314	pfn_t base, pfn;
2315	boolean_t dumpserial;
2316	int i;
2317
2318	/*
2319	 * Fall back to serial mode if there are no helpers.
2320	 * dump_plat_mincpu can be set to 0 at any time.
2321	 * dumpcfg.helpermap must contain at least one member.
2322	 *
2323	 * It is possible that the helpers haven't registered
2324	 * in helpermap yet; wait up to DUMP_HELPER_MAX_WAIT for
2325	 * at least one helper to register.
2326	 */
2327	dumpserial = B_TRUE;
2328	if (dump_plat_mincpu != 0 && dumpcfg.clevel != 0) {
2329		hrtime_t hrtmax = MSEC2NSEC(DUMP_HELPER_MAX_WAIT);
2330		hrtime_t hrtstart = gethrtime();
2331
2332		for (;;) {
2333			for (i = 0; i < BT_BITOUL(NCPU); ++i) {
2334				if (dumpcfg.helpermap[i] != 0) {
2335					dumpserial = B_FALSE;
2336					break;
2337				}
2338			}
2339
2340			if ((!dumpserial) ||
2341			    ((gethrtime() - hrtstart) >= hrtmax)) {
2342				break;
2343			}
2344
2345			SMT_PAUSE();
2346		}
2347
2348		if (dumpserial) {
2349			dumpcfg.clevel = 0;
2350			if (dumpcfg.helper[0].lzbuf == NULL) {
2351				dumpcfg.helper[0].lzbuf =
2352				    dumpcfg.helper[1].page;
2353			}
2354		}
2355	}
2356
2357	dump_init_memlist_walker(&mlw);
2358
2359	for (;;) {
2360		int sec = (gethrtime() - ds->start) / NANOSEC;
2361
2362		/*
2363		 * Render a simple progress display on the system console to
2364		 * make clear to the operator that the system has not hung.
2365		 * Emit an update when dump progress has advanced by one
2366		 * percent, or when no update has been drawn in the last
2367		 * second.
2368		 */
2369		if (ds->percent > ds->percent_done || sec > ds->sec_done) {
2370			ds->sec_done = sec;
2371			ds->percent_done = ds->percent;
2372			uprintf("^\rdumping: %2d:%02d %3d%% done",
2373			    sec / 60, sec % 60, ds->percent);
2374			ds->neednl = 1;
2375		}
2376
2377		while (CQ_IS_EMPTY(mainq) && !CQ_IS_EMPTY(writerq)) {
2378
2379			/* the writerq never blocks */
2380			cp = CQ_GET(writerq);
2381			if (cp == NULL)
2382				break;
2383
2384			dump_timeleft = dump_timeout;
2385
2386			HRSTART(ds->perpage, write);
2387			dumpvp_write(cp->buf, cp->used);
2388			HRSTOP(ds->perpage, write);
2389
2390			CQ_PUT(freebufq, cp, CBUF_FREEBUF);
2391		}
2392
2393		/*
2394		 * Wait here for some buffers to process. Returns NULL
2395		 * when all helpers have terminated and all buffers
2396		 * have been processed.
2397		 */
2398		cp = CQ_GET(mainq);
2399
2400		if (cp == NULL) {
2401
2402			/* Drain the write queue. */
2403			if (!CQ_IS_EMPTY(writerq))
2404				continue;
2405
2406			/* Main task exits here. */
2407			break;
2408		}
2409
2410		dump_timeleft = dump_timeout;
2411
2412		switch (cp->state) {
2413
2414		case CBUF_FREEMAP:
2415
2416			/*
2417			 * Note that we drop CBUF_FREEMAP buffers on
2418			 * the floor (they will not be on any cqueue)
2419			 * when we no longer need them.
2420			 */
2421			if (bitnum >= dumpcfg.bitmapsize)
2422				break;
2423
2424			if (dump_ioerr) {
2425				bitnum = dumpcfg.bitmapsize;
2426				CQ_CLOSE(helperq);
2427				break;
2428			}
2429
2430			HRSTART(ds->perpage, bitmap);
2431			for (; bitnum < dumpcfg.bitmapsize; bitnum++)
2432				if (BT_TEST(dumpcfg.bitmap, bitnum))
2433					break;
2434			HRSTOP(ds->perpage, bitmap);
2435			dump_timeleft = dump_timeout;
2436
2437			if (bitnum >= dumpcfg.bitmapsize) {
2438				CQ_CLOSE(helperq);
2439				break;
2440			}
2441
2442			/*
2443			 * Try to map CBUF_MAPSIZE ranges. Can't
2444			 * assume that memory segment size is a
2445			 * multiple of CBUF_MAPSIZE. Can't assume that
2446			 * the segment starts on a CBUF_MAPSIZE
2447			 * boundary.
2448			 */
2449			pfn = dump_bitnum_to_pfn(bitnum, &mlw);
2450			ASSERT(pfn != PFN_INVALID);
2451			ASSERT(bitnum + mlw.mpleft <= dumpcfg.bitmapsize);
2452
2453			base = P2ALIGN(pfn, CBUF_MAPNP);
2454			if (base < mlw.mpaddr) {
2455				base = mlw.mpaddr;
2456				baseoff = P2PHASE(base, CBUF_MAPNP);
2457			} else {
2458				baseoff = 0;
2459			}
2460
2461			pfnoff = pfn - base;
2462			if (pfnoff + mlw.mpleft < CBUF_MAPNP) {
2463				hibitnum = bitnum + mlw.mpleft;
2464				cp->size = ptob(pfnoff + mlw.mpleft);
2465			} else {
2466				hibitnum = bitnum - pfnoff + CBUF_MAPNP -
2467				    baseoff;
2468				cp->size = CBUF_MAPSIZE - ptob(baseoff);
2469			}
2470
2471			cp->pfn = pfn;
2472			cp->bitnum = bitnum++;
2473			cp->pagenum = pagenum++;
2474			cp->off = ptob(pfnoff);
2475
2476			for (; bitnum < hibitnum; bitnum++)
2477				if (BT_TEST(dumpcfg.bitmap, bitnum))
2478					pagenum++;
2479
2480			dump_timeleft = dump_timeout;
2481			cp->used = ptob(pagenum - cp->pagenum);
2482
2483			HRSTART(ds->perpage, map);
2484			hat_devload(kas.a_hat, cp->buf, cp->size, base,
2485			    PROT_READ, HAT_LOAD_NOCONSIST);
2486			HRSTOP(ds->perpage, map);
2487
2488			ds->pages_mapped += btop(cp->size);
2489			ds->pages_used += pagenum - cp->pagenum;
2490
2491			CQ_OPEN(mainq);
2492
2493			/*
2494			 * If there are no helpers the main task does
2495			 * non-streams lzjb compress.
2496			 */
2497			if (dumpserial) {
2498				dumpsys_lzjb_page(dumpcfg.helper, cp);
2499			} else {
2500				/* pass mapped pages to a helper */
2501				CQ_PUT(helperq, cp, CBUF_INREADY);
2502			}
2503
2504			/* the last page was done */
2505			if (bitnum >= dumpcfg.bitmapsize)
2506				CQ_CLOSE(helperq);
2507
2508			break;
2509
2510		case CBUF_USEDMAP:
2511
2512			ds->npages += btop(cp->used);
2513
2514			HRSTART(ds->perpage, unmap);
2515			hat_unload(kas.a_hat, cp->buf, cp->size, HAT_UNLOAD);
2516			HRSTOP(ds->perpage, unmap);
2517
2518			if (bitnum < dumpcfg.bitmapsize)
2519				CQ_PUT(mainq, cp, CBUF_FREEMAP);
2520			CQ_CLOSE(mainq);
2521
2522			ASSERT(ds->npages <= dumphdr->dump_npages);
2523			ds->percent = ds->npages * 100LL / dumphdr->dump_npages;
2524			break;
2525
2526		case CBUF_WRITE:
2527
2528			CQ_PUT(writerq, cp, CBUF_WRITE);
2529			break;
2530
2531		case CBUF_ERRMSG:
2532
2533			if (cp->used > 0) {
2534				cp->buf[cp->size - 2] = '\n';
2535				cp->buf[cp->size - 1] = '\0';
2536				if (ds->neednl) {
2537					uprintf("\n%s", cp->buf);
2538					ds->neednl = 0;
2539				} else {
2540					uprintf("%s", cp->buf);
2541				}
2542				/* wait for console output */
2543				drv_usecwait(200000);
2544				dump_timeleft = dump_timeout;
2545			}
2546			CQ_PUT(freebufq, cp, CBUF_FREEBUF);
2547			break;
2548
2549		default:
2550			uprintf("dump: unexpected buffer state %d, "
2551			    "buffer will be lost\n", cp->state);
2552			break;
2553
2554		} /* end switch */
2555	}
2556}
2557
2558#ifdef	COLLECT_METRICS
2559size_t
2560dumpsys_metrics(dumpsync_t *ds, char *buf, size_t size)
2561{
2562	dumpcfg_t *cfg = &dumpcfg;
2563	int myid = CPU->cpu_seqid;
2564	int i, compress_ratio;
2565	int sec, iorate;
2566	helper_t *hp, *hpend = &cfg->helper[cfg->nhelper];
2567	char *e = buf + size;
2568	char *p = buf;
2569
2570	sec = ds->elapsed / (1000 * 1000 * 1000ULL);
2571	if (sec < 1)
2572		sec = 1;
2573
2574	if (ds->iotime < 1)
2575		ds->iotime = 1;
2576	iorate = (ds->nwrite * 100000ULL) / ds->iotime;
2577
2578	compress_ratio = 100LL * ds->npages / btopr(ds->nwrite + 1);
2579
2580#define	P(...) (p += p < e ? snprintf(p, e - p, __VA_ARGS__) : 0)
2581
2582	P("Master cpu_seqid,%d\n", CPU->cpu_seqid);
2583	P("Master cpu_id,%d\n", CPU->cpu_id);
2584	P("dump_flags,0x%x\n", dumphdr->dump_flags);
2585	P("dump_ioerr,%d\n", dump_ioerr);
2586
2587	P("Helpers:\n");
2588	for (i = 0; i < ncpus; i++) {
2589		if ((i & 15) == 0)
2590			P(",,%03d,", i);
2591		if (i == myid)
2592			P("   M");
2593		else if (BT_TEST(cfg->helpermap, i))
2594			P("%4d", cpu_seq[i]->cpu_id);
2595		else
2596			P("   *");
2597		if ((i & 15) == 15)
2598			P("\n");
2599	}
2600
2601	P("ncbuf_used,%d\n", cfg->ncbuf_used);
2602	P("ncmap,%d\n", cfg->ncmap);
2603
2604	P("Found %ldM ranges,%ld\n", (CBUF_MAPSIZE / DUMP_1MB), cfg->found4m);
2605	P("Found small pages,%ld\n", cfg->foundsm);
2606
2607	P("Compression level,%d\n", cfg->clevel);
2608	P("Compression type,%s %s", cfg->clevel == 0 ? "serial" : "parallel",
2609	    cfg->clevel >= DUMP_CLEVEL_BZIP2 ? "bzip2" : "lzjb");
2610	if (cfg->clevel >= DUMP_CLEVEL_BZIP2)
2611		P(" (level %d)\n", dump_bzip2_level);
2612	else
2613		P("\n");
2614	P("Compression ratio,%d.%02d\n", compress_ratio / 100, compress_ratio %
2615	    100);
2616	P("nhelper_used,%d\n", cfg->nhelper_used);
2617
2618	P("Dump I/O rate MBS,%d.%02d\n", iorate / 100, iorate % 100);
2619	P("..total bytes,%lld\n", (u_longlong_t)ds->nwrite);
2620	P("..total nsec,%lld\n", (u_longlong_t)ds->iotime);
2621	P("dumpbuf.iosize,%ld\n", dumpbuf.iosize);
2622	P("dumpbuf.size,%ld\n", dumpbuf.size);
2623
2624	P("Dump pages/sec,%llu\n", (u_longlong_t)ds->npages / sec);
2625	P("Dump pages,%llu\n", (u_longlong_t)ds->npages);
2626	P("Dump time,%d\n", sec);
2627
2628	if (ds->pages_mapped > 0)
2629		P("per-cent map utilization,%d\n", (int)((100 * ds->pages_used)
2630		    / ds->pages_mapped));
2631
2632	P("\nPer-page metrics:\n");
2633	if (ds->npages > 0) {
2634		for (hp = cfg->helper; hp != hpend; hp++) {
2635#define	PERPAGE(x)	ds->perpage.x += hp->perpage.x;
2636			PERPAGES;
2637#undef PERPAGE
2638		}
2639#define	PERPAGE(x) \
2640		P("%s nsec/page,%d\n", #x, (int)(ds->perpage.x / ds->npages));
2641		PERPAGES;
2642#undef PERPAGE
2643		P("freebufq.empty,%d\n", (int)(ds->freebufq.empty /
2644		    ds->npages));
2645		P("helperq.empty,%d\n", (int)(ds->helperq.empty /
2646		    ds->npages));
2647		P("writerq.empty,%d\n", (int)(ds->writerq.empty /
2648		    ds->npages));
2649		P("mainq.empty,%d\n", (int)(ds->mainq.empty / ds->npages));
2650
2651		P("I/O wait nsec/page,%llu\n", (u_longlong_t)(ds->iowait /
2652		    ds->npages));
2653	}
2654#undef P
2655	if (p < e)
2656		bzero(p, e - p);
2657	return (p - buf);
2658}
2659#endif	/* COLLECT_METRICS */
2660
2661/*
2662 * Dump the system.
2663 */
2664void
2665dumpsys(void)
2666{
2667	dumpsync_t *ds = &dumpsync;
2668	taskq_t *livetaskq = NULL;
2669	pfn_t pfn;
2670	pgcnt_t bitnum;
2671	proc_t *p;
2672	helper_t *hp, *hpend = &dumpcfg.helper[dumpcfg.nhelper];
2673	cbuf_t *cp;
2674	pid_t npids, pidx;
2675	char *content;
2676	char *buf;
2677	size_t size;
2678	int save_dump_clevel;
2679	dumpmlw_t mlw;
2680	dumpcsize_t datatag;
2681	dumpdatahdr_t datahdr;
2682
2683	if (dumpvp == NULL || dumphdr == NULL) {
2684		uprintf("skipping system dump - no dump device configured\n");
2685		if (panicstr) {
2686			dumpcfg.helpers_wanted = 0;
2687			dumpsys_spinunlock(&dumpcfg.helper_lock);
2688		}
2689		return;
2690	}
2691	dumpbuf.cur = dumpbuf.start;
2692
2693	/* clear the sync variables */
2694	ASSERT(dumpcfg.nhelper > 0);
2695	bzero(ds, sizeof (*ds));
2696	ds->dumpcpu = CPU->cpu_id;
2697
2698	/*
2699	 * Calculate the starting block for dump.  If we're dumping on a
2700	 * swap device, start 1/5 of the way in; otherwise, start at the
2701	 * beginning.  And never use the first page -- it may be a disk label.
2702	 */
2703	if (dumpvp->v_flag & VISSWAP)
2704		dumphdr->dump_start = P2ROUNDUP(dumpvp_size / 5, DUMP_OFFSET);
2705	else
2706		dumphdr->dump_start = DUMP_OFFSET;
2707
2708	dumphdr->dump_flags = DF_VALID | DF_COMPLETE | DF_LIVE | DF_COMPRESSED;
2709	dumphdr->dump_crashtime = gethrestime_sec();
2710	dumphdr->dump_npages = 0;
2711	dumphdr->dump_nvtop = 0;
2712	bzero(dumpcfg.bitmap, BT_SIZEOFMAP(dumpcfg.bitmapsize));
2713	dump_timeleft = dump_timeout;
2714
2715	if (panicstr) {
2716		dumphdr->dump_flags &= ~DF_LIVE;
2717		(void) VOP_DUMPCTL(dumpvp, DUMP_FREE, NULL, NULL);
2718		(void) VOP_DUMPCTL(dumpvp, DUMP_ALLOC, NULL, NULL);
2719		(void) vsnprintf(dumphdr->dump_panicstring, DUMP_PANICSIZE,
2720		    panicstr, panicargs);
2721
2722	}
2723
2724	if (dump_conflags & DUMP_ALL)
2725		content = "all";
2726	else if (dump_conflags & DUMP_CURPROC)
2727		content = "kernel + curproc";
2728	else
2729		content = "kernel";
2730	uprintf("dumping to %s, offset %lld, content: %s\n", dumppath,
2731	    dumphdr->dump_start, content);
2732
2733	/* Make sure nodename is current */
2734	bcopy(utsname.nodename, dumphdr->dump_utsname.nodename, SYS_NMLN);
2735
2736	/*
2737	 * If this is a live dump, try to open a VCHR vnode for better
2738	 * performance. We must take care to flush the buffer cache
2739	 * first.
2740	 */
2741	if (!panicstr) {
2742		vnode_t *cdev_vp, *cmn_cdev_vp;
2743
2744		ASSERT(dumpbuf.cdev_vp == NULL);
2745		cdev_vp = makespecvp(VTOS(dumpvp)->s_dev, VCHR);
2746		if (cdev_vp != NULL) {
2747			cmn_cdev_vp = common_specvp(cdev_vp);
2748			if (VOP_OPEN(&cmn_cdev_vp, FREAD | FWRITE, kcred, NULL)
2749			    == 0) {
2750				if (vn_has_cached_data(dumpvp))
2751					(void) pvn_vplist_dirty(dumpvp, 0, NULL,
2752					    B_INVAL | B_TRUNC, kcred);
2753				dumpbuf.cdev_vp = cmn_cdev_vp;
2754			} else {
2755				VN_RELE(cdev_vp);
2756			}
2757		}
2758	}
2759
2760	/*
2761	 * Store a hires timestamp so we can look it up during debugging.
2762	 */
2763	lbolt_debug_entry();
2764
2765	/*
2766	 * Leave room for the message and ereport save areas and terminal dump
2767	 * header.
2768	 */
2769	dumpbuf.vp_limit = dumpvp_size - DUMP_LOGSIZE - DUMP_OFFSET -
2770	    DUMP_ERPTSIZE;
2771
2772	/*
2773	 * Write out the symbol table.  It's no longer compressed,
2774	 * so its 'size' and 'csize' are equal.
2775	 */
2776	dumpbuf.vp_off = dumphdr->dump_ksyms = dumphdr->dump_start + PAGESIZE;
2777	dumphdr->dump_ksyms_size = dumphdr->dump_ksyms_csize =
2778	    ksyms_snapshot(dumpvp_ksyms_write, NULL, LONG_MAX);
2779
2780	/*
2781	 * Write out the translation map.
2782	 */
2783	dumphdr->dump_map = dumpvp_flush();
2784	dump_as(&kas);
2785	dumphdr->dump_nvtop += dump_plat_addr();
2786
2787	/*
2788	 * call into hat, which may have unmapped pages that also need to
2789	 * be in the dump
2790	 */
2791	hat_dump();
2792
2793	if (dump_conflags & DUMP_ALL) {
2794		mutex_enter(&pidlock);
2795
2796		for (npids = 0, p = practive; p != NULL; p = p->p_next)
2797			dumpcfg.pids[npids++] = p->p_pid;
2798
2799		mutex_exit(&pidlock);
2800
2801		for (pidx = 0; pidx < npids; pidx++)
2802			(void) dump_process(dumpcfg.pids[pidx]);
2803
2804		dump_init_memlist_walker(&mlw);
2805		for (bitnum = 0; bitnum < dumpcfg.bitmapsize; bitnum++) {
2806			dump_timeleft = dump_timeout;
2807			pfn = dump_bitnum_to_pfn(bitnum, &mlw);
2808			/*
2809			 * Some hypervisors do not have all pages available to
2810			 * be accessed by the guest OS.  Check for page
2811			 * accessibility.
2812			 */
2813			if (plat_hold_page(pfn, PLAT_HOLD_NO_LOCK, NULL) !=
2814			    PLAT_HOLD_OK)
2815				continue;
2816			BT_SET(dumpcfg.bitmap, bitnum);
2817		}
2818		dumphdr->dump_npages = dumpcfg.bitmapsize;
2819		dumphdr->dump_flags |= DF_ALL;
2820
2821	} else if (dump_conflags & DUMP_CURPROC) {
2822		/*
2823		 * Determine which pid is to be dumped.  If we're panicking, we
2824		 * dump the process associated with panic_thread (if any).  If
2825		 * this is a live dump, we dump the process associated with
2826		 * curthread.
2827		 */
2828		npids = 0;
2829		if (panicstr) {
2830			if (panic_thread != NULL &&
2831			    panic_thread->t_procp != NULL &&
2832			    panic_thread->t_procp != &p0) {
2833				dumpcfg.pids[npids++] =
2834				    panic_thread->t_procp->p_pid;
2835			}
2836		} else {
2837			dumpcfg.pids[npids++] = curthread->t_procp->p_pid;
2838		}
2839
2840		if (npids && dump_process(dumpcfg.pids[0]) == 0)
2841			dumphdr->dump_flags |= DF_CURPROC;
2842		else
2843			dumphdr->dump_flags |= DF_KERNEL;
2844
2845	} else {
2846		dumphdr->dump_flags |= DF_KERNEL;
2847	}
2848
2849	dumphdr->dump_hashmask = (1 << highbit(dumphdr->dump_nvtop - 1)) - 1;
2850
2851	/*
2852	 * Write out the pfn table.
2853	 */
2854	dumphdr->dump_pfn = dumpvp_flush();
2855	dump_init_memlist_walker(&mlw);
2856	for (bitnum = 0; bitnum < dumpcfg.bitmapsize; bitnum++) {
2857		dump_timeleft = dump_timeout;
2858		if (!BT_TEST(dumpcfg.bitmap, bitnum))
2859			continue;
2860		pfn = dump_bitnum_to_pfn(bitnum, &mlw);
2861		ASSERT(pfn != PFN_INVALID);
2862		dumpvp_write(&pfn, sizeof (pfn_t));
2863	}
2864	dump_plat_pfn();
2865
2866	/*
2867	 * Write out all the pages.
2868	 * Map pages, copy them handling UEs, compress, and write them out.
2869	 * Cooperate with any helpers running on CPUs in panic_idle().
2870	 */
2871	dumphdr->dump_data = dumpvp_flush();
2872
2873	bzero(dumpcfg.helpermap, BT_SIZEOFMAP(NCPU));
2874	ds->live = dumpcfg.clevel > 0 &&
2875	    (dumphdr->dump_flags & DF_LIVE) != 0;
2876
2877	save_dump_clevel = dumpcfg.clevel;
2878	if (panicstr)
2879		dumpsys_get_maxmem();
2880	else if (dumpcfg.clevel >= DUMP_CLEVEL_BZIP2)
2881		dumpcfg.clevel = DUMP_CLEVEL_LZJB;
2882
2883	dumpcfg.nhelper_used = 0;
2884	for (hp = dumpcfg.helper; hp != hpend; hp++) {
2885		if (hp->page == NULL) {
2886			hp->helper = DONEHELPER;
2887			continue;
2888		}
2889		++dumpcfg.nhelper_used;
2890		hp->helper = FREEHELPER;
2891		hp->taskqid = 0;
2892		hp->ds = ds;
2893		bzero(&hp->perpage, sizeof (hp->perpage));
2894		if (dumpcfg.clevel >= DUMP_CLEVEL_BZIP2)
2895			(void) BZ2_bzCompressReset(&hp->bzstream);
2896	}
2897
2898	CQ_OPEN(freebufq);
2899	CQ_OPEN(helperq);
2900
2901	dumpcfg.ncbuf_used = 0;
2902	for (cp = dumpcfg.cbuf; cp != &dumpcfg.cbuf[dumpcfg.ncbuf]; cp++) {
2903		if (cp->buf != NULL) {
2904			CQ_PUT(freebufq, cp, CBUF_FREEBUF);
2905			++dumpcfg.ncbuf_used;
2906		}
2907	}
2908
2909	for (cp = dumpcfg.cmap; cp != &dumpcfg.cmap[dumpcfg.ncmap]; cp++)
2910		CQ_PUT(mainq, cp, CBUF_FREEMAP);
2911
2912	ds->start = gethrtime();
2913	ds->iowaitts = ds->start;
2914
2915	/* start helpers */
2916	if (ds->live) {
2917		int n = dumpcfg.nhelper_used;
2918		int pri = MINCLSYSPRI - 25;
2919
2920		livetaskq = taskq_create("LiveDump", n, pri, n, n,
2921		    TASKQ_PREPOPULATE);
2922		for (hp = dumpcfg.helper; hp != hpend; hp++) {
2923			if (hp->page == NULL)
2924				continue;
2925			hp->helper = hp - dumpcfg.helper;
2926			hp->taskqid = taskq_dispatch(livetaskq,
2927			    dumpsys_live_helper, (void *)hp, TQ_NOSLEEP);
2928		}
2929
2930	} else {
2931		if (panicstr)
2932			kmem_dump_begin();
2933		dumpcfg.helpers_wanted = dumpcfg.clevel > 0;
2934		dumpsys_spinunlock(&dumpcfg.helper_lock);
2935	}
2936
2937	/* run main task */
2938	dumpsys_main_task(ds);
2939
2940	ds->elapsed = gethrtime() - ds->start;
2941	if (ds->elapsed < 1)
2942		ds->elapsed = 1;
2943
2944	if (livetaskq != NULL)
2945		taskq_destroy(livetaskq);
2946
2947	if (ds->neednl) {
2948		uprintf("\n");
2949		ds->neednl = 0;
2950	}
2951
2952	/* record actual pages dumped */
2953	dumphdr->dump_npages = ds->npages;
2954
2955	/* platform-specific data */
2956	dumphdr->dump_npages += dump_plat_data(dumpcfg.cbuf[0].buf);
2957
2958	/* note any errors by clearing DF_COMPLETE */
2959	if (dump_ioerr || ds->npages < dumphdr->dump_npages)
2960		dumphdr->dump_flags &= ~DF_COMPLETE;
2961
2962	/* end of stream blocks */
2963	datatag = 0;
2964	dumpvp_write(&datatag, sizeof (datatag));
2965
2966	bzero(&datahdr, sizeof (datahdr));
2967
2968	/* buffer for metrics */
2969	buf = dumpcfg.cbuf[0].buf;
2970	size = MIN(dumpcfg.cbuf[0].size, DUMP_OFFSET - sizeof (dumphdr_t) -
2971	    sizeof (dumpdatahdr_t));
2972
2973	/* finish the kmem intercepts, collect kmem verbose info */
2974	if (panicstr) {
2975		datahdr.dump_metrics = kmem_dump_finish(buf, size);
2976		buf += datahdr.dump_metrics;
2977		size -= datahdr.dump_metrics;
2978	}
2979
2980	/* record in the header whether this is a fault-management panic */
2981	if (panicstr)
2982		dumphdr->dump_fm_panic = is_fm_panic();
2983
2984	/* compression info in data header */
2985	datahdr.dump_datahdr_magic = DUMP_DATAHDR_MAGIC;
2986	datahdr.dump_datahdr_version = DUMP_DATAHDR_VERSION;
2987	datahdr.dump_maxcsize = CBUF_SIZE;
2988	datahdr.dump_maxrange = CBUF_MAPSIZE / PAGESIZE;
2989	datahdr.dump_nstreams = dumpcfg.nhelper_used;
2990	datahdr.dump_clevel = dumpcfg.clevel;
2991#ifdef COLLECT_METRICS
2992	if (dump_metrics_on)
2993		datahdr.dump_metrics += dumpsys_metrics(ds, buf, size);
2994#endif
2995	datahdr.dump_data_csize = dumpvp_flush() - dumphdr->dump_data;
2996
2997	/*
2998	 * Write out the initial and terminal dump headers.
2999	 */
3000	dumpbuf.vp_off = dumphdr->dump_start;
3001	dumpvp_write(dumphdr, sizeof (dumphdr_t));
3002	(void) dumpvp_flush();
3003
3004	dumpbuf.vp_limit = dumpvp_size;
3005	dumpbuf.vp_off = dumpbuf.vp_limit - DUMP_OFFSET;
3006	dumpvp_write(dumphdr, sizeof (dumphdr_t));
3007	dumpvp_write(&datahdr, sizeof (dumpdatahdr_t));
3008	dumpvp_write(dumpcfg.cbuf[0].buf, datahdr.dump_metrics);
3009
3010	(void) dumpvp_flush();
3011
3012	uprintf("\r%3d%% done: %llu pages dumped, ",
3013	    ds->percent_done, (u_longlong_t)ds->npages);
3014
3015	if (dump_ioerr == 0) {
3016		uprintf("dump succeeded\n");
3017	} else {
3018		uprintf("dump failed: error %d\n", dump_ioerr);
3019#ifdef DEBUG
3020		if (panicstr)
3021			debug_enter("dump failed");
3022#endif
3023	}
3024
3025	/*
3026	 * Write out all undelivered messages.  This has to be the *last*
3027	 * thing we do because the dump process itself emits messages.
3028	 */
3029	if (panicstr) {
3030		dump_summary();
3031		dump_ereports();
3032		dump_messages();
3033	}
3034
3035	delay(2 * hz);	/* let people see the 'done' message */
3036	dump_timeleft = 0;
3037	dump_ioerr = 0;
3038
3039	/* restore settings after live dump completes */
3040	if (!panicstr) {
3041		dumpcfg.clevel = save_dump_clevel;
3042
3043		/* release any VCHR open of the dump device */
3044		if (dumpbuf.cdev_vp != NULL) {
3045			(void) VOP_CLOSE(dumpbuf.cdev_vp, FREAD | FWRITE, 1, 0,
3046			    kcred, NULL);
3047			VN_RELE(dumpbuf.cdev_vp);
3048			dumpbuf.cdev_vp = NULL;
3049		}
3050	}
3051}
3052
3053/*
3054 * This function is called whenever the memory size, as represented
3055 * by the phys_install list, changes.
3056 */
3057void
3058dump_resize()
3059{
3060	mutex_enter(&dump_lock);
3061	dumphdr_init();
3062	dumpbuf_resize();
3063	dump_update_clevel();
3064	mutex_exit(&dump_lock);
3065}
3066
3067/*
3068 * This function allows for dynamic resizing of a dump area. It assumes that
3069 * the underlying device has update its appropriate size(9P).
3070 */
3071int
3072dumpvp_resize()
3073{
3074	int error;
3075	vattr_t vattr;
3076
3077	mutex_enter(&dump_lock);
3078	vattr.va_mask = AT_SIZE;
3079	if ((error = VOP_GETATTR(dumpvp, &vattr, 0, kcred, NULL)) != 0) {
3080		mutex_exit(&dump_lock);
3081		return (error);
3082	}
3083
3084	if (error == 0 && vattr.va_size < 2 * DUMP_LOGSIZE + DUMP_ERPTSIZE) {
3085		mutex_exit(&dump_lock);
3086		return (ENOSPC);
3087	}
3088
3089	dumpvp_size = vattr.va_size & -DUMP_OFFSET;
3090	mutex_exit(&dump_lock);
3091	return (0);
3092}
3093
3094int
3095dump_set_uuid(const char *uuidstr)
3096{
3097	const char *ptr;
3098	int i;
3099
3100	if (uuidstr == NULL || strnlen(uuidstr, 36 + 1) != 36)
3101		return (EINVAL);
3102
3103	/* uuid_parse is not common code so check manually */
3104	for (i = 0, ptr = uuidstr; i < 36; i++, ptr++) {
3105		switch (i) {
3106		case 8:
3107		case 13:
3108		case 18:
3109		case 23:
3110			if (*ptr != '-')
3111				return (EINVAL);
3112			break;
3113
3114		default:
3115			if (!isxdigit(*ptr))
3116				return (EINVAL);
3117			break;
3118		}
3119	}
3120
3121	if (dump_osimage_uuid[0] != '\0')
3122		return (EALREADY);
3123
3124	(void) strncpy(dump_osimage_uuid, uuidstr, 36 + 1);
3125
3126	cmn_err(CE_CONT, "?This illumos instance has UUID %s\n",
3127	    dump_osimage_uuid);
3128
3129	return (0);
3130}
3131
3132const char *
3133dump_get_uuid(void)
3134{
3135	return (dump_osimage_uuid[0] != '\0' ? dump_osimage_uuid : "");
3136}
3137